java.io.OutputStream Scala Examples

package com.webank.wedatasphere.linkis.entranceclient.context

import java.io.{InputStream, OutputStream}

import com.webank.wedatasphere.linkis.entrance.log._
import com.webank.wedatasphere.linkis.scheduler.queue.Job
import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration
import org.apache.commons.io.input.NullInputStream
import org.apache.commons.io.output.NullOutputStream


class ClientLogManager extends CacheLogManager {
  override def getLogReader(execId: String): LogReader = {
    new CacheLogReader("", ServerConfiguration.BDP_SERVER_ENCODING.getValue,
      new Cache(10), "") {
      override def getInputStream: InputStream = new NullInputStream(0)
    }
  }

  override def createLogWriter(job: Job): LogWriter = {
    new NullCacheLogWriter(ServerConfiguration.BDP_SERVER_ENCODING.getValue,
      new Cache(20))
  }
  class NullCacheLogWriter(charset:String,
                           sharedCache:Cache,
                           override protected val outputStream: OutputStream = new NullOutputStream)
    extends CacheLogWriter("", charset, sharedCache, "")
} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.lang.annotation.Annotation
import java.lang.reflect.Type
import java.nio.charset.StandardCharsets
import java.text.SimpleDateFormat
import java.util.{Calendar, SimpleTimeZone}
import javax.ws.rs.Produces
import javax.ws.rs.core.{MediaType, MultivaluedMap}
import javax.ws.rs.ext.{MessageBodyWriter, Provider}

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}


@Provider
@Produces(Array(MediaType.APPLICATION_JSON))
private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{

  val mapper = new ObjectMapper() {
    override def writeValueAsString(t: Any): String = {
      super.writeValueAsString(t)
    }
  }
  mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
  mapper.enable(SerializationFeature.INDENT_OUTPUT)
  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)

  override def isWriteable(
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Boolean = {
      true
  }

  override def writeTo(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType,
      multivaluedMap: MultivaluedMap[String, AnyRef],
      outputStream: OutputStream): Unit = {
    t match {
      case ErrorWrapper(err) => outputStream.write(err.getBytes(StandardCharsets.UTF_8))
      case _ => mapper.writeValue(outputStream, t)
    }
  }

  override def getSize(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Long = {
    -1L
  }
}

private[spark] object JacksonMessageWriter {
  def makeISODateFormat: SimpleDateFormat = {
    val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'")
    val cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"))
    iso8601.setCalendar(cal)
    iso8601
  }
} 

package com.github.shadowsocks

import java.io.{IOException, InputStream, OutputStream}
import java.lang.System.currentTimeMillis
import java.util.concurrent.Semaphore

import android.util.Log

import scala.collection.JavaConversions._


class GuardedProcess(cmd: Seq[String]) extends Process {
  private val TAG = classOf[GuardedProcess].getSimpleName

  @volatile private var guardThread: Thread = _
  @volatile private var isDestroyed: Boolean = _
  @volatile private var process: Process = _
  @volatile private var isRestart = false

  def start(onRestartCallback: () => Unit = null): GuardedProcess = {
    val semaphore = new Semaphore(1)
    semaphore.acquire
    @volatile var ioException: IOException = null

    guardThread = new Thread(() => {
      try {
        var callback: () => Unit = null
        while (!isDestroyed) {
          Log.i(TAG, "start process: " + cmd)
          val startTime = currentTimeMillis

          process = new ProcessBuilder(cmd).redirectErrorStream(true).start

          if (callback == null) callback = onRestartCallback else callback()

          semaphore.release
          process.waitFor

          if (isRestart) {
            isRestart = false
          } else {
            if (currentTimeMillis - startTime < 1000) {
              Log.w(TAG, "process exit too fast, stop guard: " + cmd)
              isDestroyed = true
            }
          }
        }
      } catch {
        case ignored: InterruptedException =>
          Log.i(TAG, "thread interrupt, destroy process: " + cmd)
          process.destroy()
        case e: IOException => ioException = e
      } finally semaphore.release
    }, "GuardThread-" + cmd)

    guardThread.start()
    semaphore.acquire

    if (ioException != null) {
      throw ioException
    }

    this
  }

  def destroy() {
    isDestroyed = true
    guardThread.interrupt()
    process.destroy()
    try guardThread.join() catch {
      case ignored: InterruptedException =>
    }
  }

  def restart() {
    isRestart = true
    process.destroy()
  }

  def exitValue: Int = throw new UnsupportedOperationException
  def getErrorStream: InputStream = throw new UnsupportedOperationException
  def getInputStream: InputStream = throw new UnsupportedOperationException
  def getOutputStream: OutputStream = throw new UnsupportedOperationException

  @throws(classOf[InterruptedException])
  def waitFor = {
    guardThread.join()
    0
  }
} 

package org.apache.spark.streaming.util

import java.io.OutputStream
import java.util.concurrent.TimeUnit._

import scala.annotation.tailrec

import org.apache.spark.internal.Logging

private[streaming]
class RateLimitedOutputStream(out: OutputStream, desiredBytesPerSec: Int)
  extends OutputStream
  with Logging {

  require(desiredBytesPerSec > 0)

  private val SYNC_INTERVAL = NANOSECONDS.convert(10, SECONDS)
  private val CHUNK_SIZE = 8192
  private var lastSyncTime = System.nanoTime
  private var bytesWrittenSinceSync = 0L

  override def write(b: Int) {
    waitToWrite(1)
    out.write(b)
  }

  override def write(bytes: Array[Byte]) {
    write(bytes, 0, bytes.length)
  }

  @tailrec
  override final def write(bytes: Array[Byte], offset: Int, length: Int) {
    val writeSize = math.min(length - offset, CHUNK_SIZE)
    if (writeSize > 0) {
      waitToWrite(writeSize)
      out.write(bytes, offset, writeSize)
      write(bytes, offset + writeSize, length)
    }
  }

  override def flush() {
    out.flush()
  }

  override def close() {
    out.close()
  }

  @tailrec
  private def waitToWrite(numBytes: Int) {
    val now = System.nanoTime
    val elapsedNanosecs = math.max(now - lastSyncTime, 1)
    val rate = bytesWrittenSinceSync.toDouble * 1000000000 / elapsedNanosecs
    if (rate < desiredBytesPerSec) {
      // It's okay to write; just update some variables and return
      bytesWrittenSinceSync += numBytes
      if (now > lastSyncTime + SYNC_INTERVAL) {
        // Sync interval has passed; let's resync
        lastSyncTime = now
        bytesWrittenSinceSync = numBytes
      }
    } else {
      // Calculate how much time we should sleep to bring ourselves to the desired rate.
      val targetTimeInMillis = bytesWrittenSinceSync * 1000 / desiredBytesPerSec
      val elapsedTimeInMillis = elapsedNanosecs / 1000000
      val sleepTimeInMillis = targetTimeInMillis - elapsedTimeInMillis
      if (sleepTimeInMillis > 0) {
        logTrace("Natural rate is " + rate + " per second but desired rate is " +
          desiredBytesPerSec + ", sleeping for " + sleepTimeInMillis + " ms to compensate.")
        Thread.sleep(sleepTimeInMillis)
      }
      waitToWrite(numBytes)
    }
  }
} 

package org.apache.spark.security

import java.io.{InputStream, OutputStream}
import java.util.Properties
import javax.crypto.spec.{IvParameterSpec, SecretKeySpec}

import org.apache.commons.crypto.random._
import org.apache.commons.crypto.stream._
import org.apache.hadoop.io.Text

import org.apache.spark.SparkConf
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.internal.Logging
import org.apache.spark.internal.config._


  private[this] def createInitializationVector(properties: Properties): Array[Byte] = {
    val iv = new Array[Byte](IV_LENGTH_IN_BYTES)
    val initialIVStart = System.currentTimeMillis()
    CryptoRandomFactory.getCryptoRandom(properties).nextBytes(iv)
    val initialIVFinish = System.currentTimeMillis()
    val initialIVTime = initialIVFinish - initialIVStart
    if (initialIVTime > 2000) {
      logWarning(s"It costs ${initialIVTime} milliseconds to create the Initialization Vector " +
        s"used by CryptoStream")
    }
    iv
  }
} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.util.zip.ZipOutputStream
import javax.ws.rs.{GET, Produces}
import javax.ws.rs.core.{MediaType, Response, StreamingOutput}

import scala.util.control.NonFatal

import org.apache.spark.SparkConf
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.internal.Logging

@Produces(Array(MediaType.APPLICATION_OCTET_STREAM))
private[v1] class EventLogDownloadResource(
    val uIRoot: UIRoot,
    val appId: String,
    val attemptId: Option[String]) extends Logging {
  val conf = SparkHadoopUtil.get.newConfiguration(new SparkConf)

  @GET
  def getEventLogs(): Response = {
    try {
      val fileName = {
        attemptId match {
          case Some(id) => s"eventLogs-$appId-$id.zip"
          case None => s"eventLogs-$appId.zip"
        }
      }

      val stream = new StreamingOutput {
        override def write(output: OutputStream): Unit = {
          val zipStream = new ZipOutputStream(output)
          try {
            uIRoot.writeEventLogs(appId, attemptId, zipStream)
          } finally {
            zipStream.close()
          }

        }
      }

      Response.ok(stream)
        .header("Content-Disposition", s"attachment; filename=$fileName")
        .header("Content-Type", MediaType.APPLICATION_OCTET_STREAM)
        .build()
    } catch {
      case NonFatal(e) =>
        Response.serverError()
          .entity(s"Event logs are not available for app: $appId.")
          .status(Response.Status.SERVICE_UNAVAILABLE)
          .build()
    }
  }
} 

package org.apache.spark.util.io

import java.io.OutputStream
import java.nio.ByteBuffer

import scala.collection.mutable.ArrayBuffer

import org.apache.spark.storage.StorageUtils


  private[this] var position = chunkSize
  private[this] var _size = 0
  private[this] var closed: Boolean = false

  def size: Long = _size

  override def close(): Unit = {
    if (!closed) {
      super.close()
      closed = true
    }
  }

  override def write(b: Int): Unit = {
    require(!closed, "cannot write to a closed ChunkedByteBufferOutputStream")
    allocateNewChunkIfNeeded()
    chunks(lastChunkIndex).put(b.toByte)
    position += 1
    _size += 1
  }

  override def write(bytes: Array[Byte], off: Int, len: Int): Unit = {
    require(!closed, "cannot write to a closed ChunkedByteBufferOutputStream")
    var written = 0
    while (written < len) {
      allocateNewChunkIfNeeded()
      val thisBatch = math.min(chunkSize - position, len - written)
      chunks(lastChunkIndex).put(bytes, written + off, thisBatch)
      written += thisBatch
      position += thisBatch
    }
    _size += len
  }

  @inline
  private def allocateNewChunkIfNeeded(): Unit = {
    if (position == chunkSize) {
      chunks += allocator(chunkSize)
      lastChunkIndex += 1
      position = 0
    }
  }

  def toChunkedByteBuffer: ChunkedByteBuffer = {
    require(closed, "cannot call toChunkedByteBuffer() unless close() has been called")
    require(!toChunkedByteBufferWasCalled, "toChunkedByteBuffer() can only be called once")
    toChunkedByteBufferWasCalled = true
    if (lastChunkIndex == -1) {
      new ChunkedByteBuffer(Array.empty[ByteBuffer])
    } else {
      // Copy the first n-1 chunks to the output, and then create an array that fits the last chunk.
      // An alternative would have been returning an array of ByteBuffers, with the last buffer
      // bounded to only the last chunk's position. However, given our use case in Spark (to put
      // the chunks in block manager), only limiting the view bound of the buffer would still
      // require the block manager to store the whole chunk.
      val ret = new Array[ByteBuffer](chunks.size)
      for (i <- 0 until chunks.size - 1) {
        ret(i) = chunks(i)
        ret(i).flip()
      }
      if (position == chunkSize) {
        ret(lastChunkIndex) = chunks(lastChunkIndex)
        ret(lastChunkIndex).flip()
      } else {
        ret(lastChunkIndex) = allocator(position)
        chunks(lastChunkIndex).flip()
        ret(lastChunkIndex).put(chunks(lastChunkIndex))
        ret(lastChunkIndex).flip()
        StorageUtils.dispose(chunks(lastChunkIndex))
      }
      new ChunkedByteBuffer(ret)
    }
  }
} 

package com.intel.analytics.bigdl.utils.tf

import java.io.OutputStream
import java.nio.{ByteBuffer, ByteOrder}

import com.intel.analytics.bigdl.utils.Crc32

class TFRecordWriter(out: OutputStream) {

  private def toByteArrayAsLong(data: Long): Array[Byte] = {
    val buff = new Array[Byte](8)
    val bb = ByteBuffer.wrap(buff)
    bb.order(ByteOrder.LITTLE_ENDIAN)
    bb.putLong(data)
    buff
  }

  private def toByteArrayAsInt(data: Int): Array[Byte] = {
    val buff = new Array[Byte](4)
    val bb = ByteBuffer.wrap(buff)
    bb.order(ByteOrder.LITTLE_ENDIAN)
    bb.putInt(data)
    buff
  }

  def write(record: Array[Byte], offset: Int, length: Int): Unit = {
    val len = toByteArrayAsLong(length)
    out.write(len)
    out.write(toByteArrayAsInt(Crc32.maskedCRC32(len).toInt))
    out.write(record, offset, length)
    out.write(toByteArrayAsInt(Crc32.maskedCRC32(record, offset, length).toInt))
  }

  def write(record: Array[Byte]): Unit = {
    write(record, 0, record.length)
  }
} 

package org.archive.archivespark.sparkling.io

import java.io.{InputStream, OutputStream}

trait TypedInOut[A] extends Serializable {
  trait TypedInOutWriter {
    def stream: OutputStream
    def write(record: A)
    def flush(): Unit
    def close(): Unit
  }

  def out(stream: OutputStream): TypedInOutWriter
  def in(stream: InputStream): Iterator[A]
}

object TypedInOut {
  def apply[A, O](writer: OutputStream => O, reader: InputStream => Iterator[A])(writeRecord: (A, O) => Unit, flushOut: O => Unit, closeOut: O => Unit): TypedInOut[A] = new TypedInOut[A] {
    override def out(outStream: OutputStream): TypedInOutWriter = new TypedInOutWriter {
      override val stream: OutputStream = outStream
      private val out = writer(stream)
      override def write(record: A): Unit = writeRecord(record, out)
      override def flush(): Unit = flushOut(out)
      override def close(): Unit = closeOut(out)
    }

    override def in(inStream: InputStream): Iterator[A] = reader(inStream)
  }

  implicit val stringInOut: TypedInOut[String] = TypedInOut(IOUtil.print(_), IOUtil.lines(_))(
    (r, o) => o.println(r),
    _.flush(),
    _.close()
  )

  def toStringInOut[A](toString: A => String, fromString: String => A): TypedInOut[A] = TypedInOut(IOUtil.print(_), IOUtil.lines(_).map(fromString))(
    (r, o) => o.println(toString(r)),
    _.flush(),
    _.close()
  )
} 

package org.archive.archivespark.sparkling.io

import java.io.{FileInputStream, FileOutputStream, OutputStream}

import org.apache.hadoop.fs.Path
import org.archive.archivespark.sparkling.logging.{Log, LogContext}

import scala.util.Try

class HdfsFileWriter private(filename: String, append: Boolean, replication: Short) extends OutputStream {
  implicit val logContext: LogContext = LogContext(this)

  private val file = IOUtil.tmpFile

  Log.info("Writing to temporary local file " + file.getCanonicalPath + " (" + filename + ")...")

  val out = new FileOutputStream(file)

  override def close(): Unit = {
    Try { out.close() }
    Log.info("Copying from temporary file " + file.getCanonicalPath + " to " + filename + "...")
    if (append) {
      val in = new FileInputStream(file)
      val appendOut = HdfsIO.fs.append(new Path(filename))
      IOUtil.copy(in, appendOut)
      appendOut.close()
      in.close()
      file.delete()
    } else HdfsIO.copyFromLocal(file.getCanonicalPath, filename, move = true, overwrite = true, replication)
    Log.info("Done. (" + filename + ")")
  }

  override def write(b: Int): Unit = out.write(b)
  override def write(b: Array[Byte]): Unit = out.write(b)
  override def write(b: Array[Byte], off: Int, len: Int): Unit = out.write(b, off, len)
  override def flush(): Unit = out.flush()
}

object HdfsFileWriter {
  def apply(filename: String, overwrite: Boolean = false, append: Boolean = false, replication: Short = 0): HdfsFileWriter = {
    if (!overwrite && !append) HdfsIO.ensureNewFile(filename)
    new HdfsFileWriter(filename, append, replication)
  }
} 

package com.webank.wedatasphere.linkis.engine.pipeline.executor

import java.io.OutputStream

import com.webank.wedatasphere.linkis.common.io.FsPath
import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
import com.webank.wedatasphere.linkis.engine.pipeline.OutputStreamCache
import com.webank.wedatasphere.linkis.engine.pipeline.constant.PipeLineConstant._
import com.webank.wedatasphere.linkis.engine.pipeline.conversions.FsConvertions._
import com.webank.wedatasphere.linkis.engine.pipeline.exception.PipeLineErrorException
import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteResponse
import com.webank.wedatasphere.linkis.storage.FSFactory
import com.webank.wedatasphere.linkis.storage.excel.ExcelFsWriter
import com.webank.wedatasphere.linkis.storage.source.FileSource
import org.apache.commons.io.IOUtils



class ExcelExecutor extends PipeLineExecutor {
  override def execute(sourcePath: String, destPath: String, engineExecutorContext: EngineExecutorContext): ExecuteResponse = {
    if (!FileSource.isResultSet(sourcePath)) {
      throw new PipeLineErrorException(70005, "不是结果集文件")
    }
    val sourceFsPath = new FsPath(sourcePath)
    val destFsPath = new FsPath(s"$destPath.xlsx")
    val sourceFs = FSFactory.getFs(sourceFsPath)
    sourceFs.init(null)
    val destFs = FSFactory.getFs(destFsPath)
    destFs.init(null)
    val fileSource = FileSource.create(sourceFsPath, sourceFs)
    if (!FileSource.isTableResultSet(fileSource)) {
      throw new PipeLineErrorException(70005, "只有table类型的结果集才能转为excel")
    }
    var nullValue = options.getOrDefault(PIPELINE_OUTPUT_SHUFFLE_NULL_TYPE, "NULL")
    if (BLANK.equalsIgnoreCase(nullValue)) nullValue = ""
    val outputStream: OutputStream = destFs.write(destFsPath, options.get(PIPELINE_OUTPUT_ISOVERWRITE).toBoolean)
    val excelFsWriter = ExcelFsWriter.getExcelFsWriter(DEFAULTC_HARSET, DEFAULT_SHEETNAME, DEFAULT_DATEFORMATE, outputStream)
    import scala.collection.JavaConversions._
    OutputStreamCache.osCache += engineExecutorContext.getJobId.get -> outputStream
    fileSource.addParams("nullValue", nullValue).write(excelFsWriter)
    IOUtils.closeQuietly(excelFsWriter)
    IOUtils.closeQuietly(fileSource)
    IOUtils.closeQuietly(sourceFs)
    IOUtils.closeQuietly(destFs)
    super.execute(sourcePath, destPath, engineExecutorContext)
  }

  override def Kind: String = "excel"
}

object ExcelExecutor {
  val excelExecutor = new ExcelExecutor

  def getInstance: PipeLineExecutor = excelExecutor
} 

package com.webank.wedatasphere.linkis.engine.pipeline.executor

import java.io.OutputStream

import com.webank.wedatasphere.linkis.common.io.FsPath
import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
import com.webank.wedatasphere.linkis.engine.pipeline.OutputStreamCache
import com.webank.wedatasphere.linkis.engine.pipeline.constant.PipeLineConstant._
import com.webank.wedatasphere.linkis.engine.pipeline.conversions.FsConvertions._
import com.webank.wedatasphere.linkis.engine.pipeline.exception.PipeLineErrorException
import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteResponse
import com.webank.wedatasphere.linkis.server._
import com.webank.wedatasphere.linkis.storage.FSFactory
import com.webank.wedatasphere.linkis.storage.csv.CSVFsWriter
import com.webank.wedatasphere.linkis.storage.source.FileSource
import org.apache.commons.io.IOUtils



class CSVExecutor extends PipeLineExecutor {

  override def execute(sourcePath: String, destPath: String, engineExecutorContext: EngineExecutorContext): ExecuteResponse = {
    if (!FileSource.isResultSet(sourcePath)) {
      throw new PipeLineErrorException(70005, "Not a result set file（不是结果集文件）")
    }
    val sourceFsPath = new FsPath(sourcePath)
    val destFsPath = new FsPath(s"$destPath.$Kind")
    val sourceFs = FSFactory.getFs(sourceFsPath)
    sourceFs.init(null)
    val destFs = FSFactory.getFs(destFsPath)
    destFs.init(null)
    val fileSource = FileSource.create(sourceFsPath, sourceFs)
    if (!FileSource.isTableResultSet(fileSource)) {
      throw new PipeLineErrorException(70005, "只有table类型的结果集才能转为csv")
    }
    var nullValue = options.getOrDefault(PIPELINE_OUTPUT_SHUFFLE_NULL_TYPE, "NULL")
    if (BLANK.equalsIgnoreCase(nullValue)) nullValue = ""
    val outputStream: OutputStream = destFs.write(destFsPath, options.get(PIPELINE_OUTPUT_ISOVERWRITE).toBoolean)
    OutputStreamCache.osCache += engineExecutorContext.getJobId.get -> outputStream
    val cSVFsWriter = CSVFsWriter.getCSVFSWriter(options.get(PIPELINE_OUTPUT_CHARSET), options.get(PIPELINE_FIELD_SPLIT), outputStream)
    fileSource.addParams("nullValue", nullValue).write(cSVFsWriter)
    IOUtils.closeQuietly(cSVFsWriter)
    IOUtils.closeQuietly(fileSource)
    IOUtils.closeQuietly(sourceFs)
    IOUtils.closeQuietly(destFs)
    super.execute(sourcePath, destPath, engineExecutorContext)
  }

  override def Kind: String = "csv"

}

object CSVExecutor {
  val csvExecutor = new CSVExecutor

  def getInstance: PipeLineExecutor = csvExecutor
} 

package com.webank.wedatasphere.linkis.entrance.log

import java.io.{Closeable, Flushable, OutputStream}
import java.util

import com.webank.wedatasphere.linkis.common.io.FsPath
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.entrance.exception.EntranceErrorException
import com.webank.wedatasphere.linkis.storage.FSFactory
import com.webank.wedatasphere.linkis.storage.utils.FileSystemUtils
import org.apache.commons.io.IOUtils
import org.apache.commons.lang.StringUtils


abstract class LogWriter(charset: String) extends Closeable with Flushable with Logging {

  private var firstWrite = true

  protected val outputStream: OutputStream

  def write(msg: String): Unit = synchronized {
    val log = if (!firstWrite) "\n" + msg else {
      firstWrite = false
      msg
    }
    Utils.tryQuietly({
      outputStream.write(log.getBytes(charset))
      outputStream.flush()
    }, t => {
      warn("error when write query log to outputStream.", t)
      info(msg)
    })
  }



  def flush(): Unit = Utils.tryQuietly(outputStream.flush(),  t => {
    warn("Error encounters when flush log,", t)
  })


  def close(): Unit = {
    flush()
    if (outputStream != null) IOUtils.closeQuietly(outputStream)

  }
}

abstract class AbstractLogWriter(logPath: String,
                                 user: String,
                                 charset: String) extends LogWriter(charset) {
  if(StringUtils.isBlank(logPath)) throw new EntranceErrorException(20301, "logPath cannot be empty.")
  protected val fileSystem = FSFactory.getFs(new FsPath(logPath))
  fileSystem.init(new util.HashMap[String, String]())

  protected val outputStream: OutputStream = {
    FileSystemUtils.createNewFile(new FsPath(logPath), true)
    fileSystem.write(new FsPath(logPath), true)
  }

  override def close(): Unit = {
    super.close()
    if (fileSystem != null) Utils.tryQuietly(fileSystem.close(), t => {
      warn("Error encounters when closing fileSystem", t)
    })
  }
} 

package com.webank.wedatasphere.linkis.engine.rs

import java.io.OutputStream

import com.webank.wedatasphere.linkis.common.io.resultset.ResultSetWriter
import com.webank.wedatasphere.linkis.common.io.{MetaData, Record}
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
import com.webank.wedatasphere.linkis.storage.LineRecord

import scala.collection.mutable.ArrayBuffer



class RsOutputStream extends OutputStream with Logging{
  private val line = ArrayBuffer[Byte]()
  private var isReady = false
  private var writer: ResultSetWriter[_ <: MetaData, _ <: Record] = _
  override def write(b: Int) = if(isReady) synchronized {
    if(writer != null) {
      if (b == '\n') {
        val outStr = new String(line.toArray,"UTF-8")
        writer.addRecord(new LineRecord(outStr))
        //info("output line:" + outStr)
        line.clear()
      } else line += b.toByte
    }else{
       warn("writer is null")
    }
  }

  def reset(engineExecutorContext: EngineExecutorContext) = {
    writer = engineExecutorContext.createDefaultResultSetWriter()
    writer.addMetaData(null)
  }

  def ready() = isReady = true

  override def flush(): Unit = if(writer != null && line.nonEmpty) {
    val outStr = new String(line.toArray,"UTF-8")
    writer.addRecord(new LineRecord(outStr))
    //info("flush line:" + outStr)
    line.clear()
  }

  override def toString = if(writer != null) writer.toString() else null

  override def close() = if(writer != null) {
    flush()
    writer.close()
    writer = null
  }
} 

package com.webank.wedatasphere.linkis.jobhistory.util

import java.io.{InputStream, OutputStream}
import java.util.Date

import com.webank.wedatasphere.linkis.common.conf.CommonVars
import com.webank.wedatasphere.linkis.common.io.FsPath
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.jobhistory.entity.QueryTask
import com.webank.wedatasphere.linkis.protocol.query.RequestInsertTask
import com.webank.wedatasphere.linkis.storage.FSFactory
import com.webank.wedatasphere.linkis.storage.fs.FileSystem
import com.webank.wedatasphere.linkis.storage.utils.{FileSystemUtils, StorageUtils}
import org.apache.commons.io.IOUtils
import org.apache.commons.lang.time.DateFormatUtils


object QueryUtils extends Logging {

  private val CODE_STORE_PREFIX = CommonVars("bdp.dataworkcloud.query.store.prefix", "hdfs:///tmp/bdp-ide/")
  private val CODE_STORE_SUFFIX = CommonVars("bdp.dataworkcloud.query.store.suffix", "")
  private val CHARSET = "utf-8"
  private val CODE_SPLIT = ";"
  private val LENGTH_SPLIT = "#"

  def storeExecutionCode(requestInsertTask: RequestInsertTask): Unit = {
    if (requestInsertTask.getExecutionCode.length < 60000) return
    val user: String = requestInsertTask.getUmUser
    val path: String = getCodeStorePath(user)
    val fsPath: FsPath = new FsPath(path)
    val fileSystem = FSFactory.getFsByProxyUser(fsPath, user).asInstanceOf[FileSystem]
    fileSystem.init(null)
    var os: OutputStream = null
    var position = 0L
    val codeBytes = requestInsertTask.getExecutionCode.getBytes(CHARSET)
    path.intern() synchronized {
      Utils.tryFinally {
        if (!fileSystem.exists(fsPath)) FileSystemUtils.createNewFile(fsPath, user, true)
        os = fileSystem.write(fsPath, false)
        position = fileSystem.get(path).getLength
        IOUtils.write(codeBytes, os)
      } {
        IOUtils.closeQuietly(os)
        if (fileSystem != null) fileSystem.close()
      }
    }
    val length = codeBytes.length
    requestInsertTask.setExecutionCode(path + CODE_SPLIT + position + LENGTH_SPLIT + length)
  }

  def exchangeExecutionCode(queryTask: QueryTask): Unit = {
    import scala.util.control.Breaks._
    if (queryTask.getExecutionCode == null || !queryTask.getExecutionCode.startsWith(StorageUtils.HDFS_SCHEMA)) return
    val codePath = queryTask.getExecutionCode
    val path = codePath.substring(0, codePath.lastIndexOf(CODE_SPLIT))
    val codeInfo = codePath.substring(codePath.lastIndexOf(CODE_SPLIT) + 1)
    val infos: Array[String] = codeInfo.split(LENGTH_SPLIT)
    val position = infos(0).toInt
    var lengthLeft = infos(1).toInt
    val tub = new Array[Byte](1024)
    val executionCode: StringBuilder = new StringBuilder
    val fsPath: FsPath = new FsPath(path)
    val fileSystem = FSFactory.getFsByProxyUser(fsPath, queryTask.getUmUser).asInstanceOf[FileSystem]
    fileSystem.init(null)
    var is: InputStream = null
    if (!fileSystem.exists(fsPath)) return
    Utils.tryFinally {
      is = fileSystem.read(fsPath)
      if (position > 0) is.skip(position)
      breakable {
        while (lengthLeft > 0) {
          val readed = is.read(tub)
          val useful = Math.min(readed, lengthLeft)
          if (useful < 0) break()
          lengthLeft -= useful
          executionCode.append(new String(tub, 0, useful, CHARSET))
        }
      }
    } {
      if (fileSystem != null) fileSystem.close()
      IOUtils.closeQuietly(is)
    }
    queryTask.setExecutionCode(executionCode.toString())
  }

  private def getCodeStorePath(user: String): String = {
    val date: String = DateFormatUtils.format(new Date, "yyyyMMdd")
    s"${CODE_STORE_PREFIX.getValue}${user}${CODE_STORE_SUFFIX.getValue}/executionCode/${date}/_scripts"
  }
} 

package com.webank.wedatasphere.linkis.storage.script.writer

import java.io.{ByteArrayInputStream, IOException, InputStream, OutputStream}
import java.util

import com.webank.wedatasphere.linkis.common.io.{FsPath, MetaData, Record}
import com.webank.wedatasphere.linkis.storage.LineRecord
import com.webank.wedatasphere.linkis.storage.script.{Compaction, ScriptFsWriter, ScriptMetaData}
import com.webank.wedatasphere.linkis.storage.utils.{StorageConfiguration, StorageUtils}
import org.apache.commons.io.IOUtils


class StorageScriptFsWriter(val path: FsPath, val charset: String, outputStream: OutputStream = null) extends ScriptFsWriter {

  private val stringBuilder = new StringBuilder

  @scala.throws[IOException]
  override def addMetaData(metaData: MetaData): Unit = {
    val compactions = Compaction.listCompactions().filter(p => p.belongTo(StorageUtils.pathToSuffix(path.getPath)))
    val metadataLine = new util.ArrayList[String]()
    if (compactions.length > 0) {
      metaData.asInstanceOf[ScriptMetaData].getMetaData.map(compactions(0).compact).foreach(metadataLine.add)
      if (outputStream != null) {
        IOUtils.writeLines(metadataLine, "\n", outputStream, charset)
      } else {
        import scala.collection.JavaConversions._
        metadataLine.foreach(m => stringBuilder.append(s"$m\n"))
      }
    }
  }

  @scala.throws[IOException]
  override def addRecord(record: Record): Unit = {
    //转成LineRecord而不是TableRecord是为了兼容非Table类型的结果集写到本类中
    val scriptRecord = record.asInstanceOf[LineRecord]
    if (outputStream != null) {
      IOUtils.write(scriptRecord.getLine, outputStream, charset)
    } else {
      stringBuilder.append(scriptRecord.getLine)
    }
  }

  override def close(): Unit = {
    IOUtils.closeQuietly(outputStream)
  }

  override def flush(): Unit = if (outputStream != null) outputStream.flush()

  def getInputStream(): InputStream = {
    new ByteArrayInputStream(stringBuilder.toString().getBytes(StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue))
  }

} 

package com.example.streaming

import java.io.{ InputStream, OutputStream, FileInputStream, FileOutputStream }

import cats.implicits._
import cats.effect.{ IO, IOApp, Blocker, ExitCode }

import fs2.{ Stream, Chunk, Pure }

import dev.tauri.seals.scodec.StreamCodecs._

object Main extends IOApp {

  sealed trait Color
  final case object Brown extends Color
  final case object Grey extends Color

  sealed trait Animal
  final case class Elephant(name: String, tuskLength: Float) extends Animal
  final case class Quokka(name: String, color: Color = Brown) extends Animal
  final case class Quagga(name: String, speed: Double) extends Animal

  def transform(from: InputStream, to: OutputStream)(f: Animal => Stream[Pure, Animal]): IO[Unit] = {
    Blocker[IO].use { blocker =>
      val input = fs2.io.readInputStream(
        IO.pure(from),
        chunkSize = 1024,
        blocker = blocker
      )
      val sIn: Stream[IO, Animal] = input.through(streamDecoderFromReified[Animal].toPipeByte[IO]).flatMap(f)
      val sOut: Stream[IO, Unit] = streamEncoderFromReified[Animal].encode(sIn).flatMap { bv =>
        Stream.chunk(Chunk.bytes(bv.bytes.toArray))
      }.through(fs2.io.writeOutputStream(
        IO.pure(to),
        blocker = blocker,
        closeAfterUse = true
      ))
      sOut.compile.drain
    }
  }

  val transformer: Animal => Stream[Pure, Animal] = {
    case Elephant(n, tl) => Stream(Elephant(n, tl + 17))
    case Quokka(n, Brown) => Stream(Quokka(n, Grey))
    case q @ Quokka(_, _) => Stream(q)
    case Quagga(_, _) => Stream.empty
  }

  override def run(args: List[String]): IO[ExitCode] = {
    val (from, to) = args match {
      case List(from, to, _*) =>
        (from, to)
      case List(from) =>
        (from, "out.bin")
      case _ =>
        ("in.bin", "out.bin")
    }

    val task = transform(new FileInputStream(from), new FileOutputStream(to))(transformer)
    task.as(ExitCode.Success)
  }
} 

package java.util.logging

import java.io.OutputStream
import java.nio.charset.Charset

class StreamHandler(private[this] var out: OutputStream,
    private[this] val formatter: Formatter) extends Handler {

  // Defaults defined on javadocs
  setLevel(Level.INFO)
  setFilter(null)

  if (formatter == null) setFormatter(new SimpleFormatter())
  else setFormatter(formatter)

  // Required by javadoc but it is unspecified what to do if formatter is null
  def this() = this(null, null)

  private[this] var headWritten: Boolean = false

  private def encodingOrDefault: Charset =
    if (getEncoding == null) Charset.defaultCharset()
    else Charset.forName(getEncoding)

  protected def setOutputStream(out: OutputStream): Unit = {
    // Required by javadocs
    if (out != null && formatter != null) {
      out.write(formatter.getTail(this).getBytes(encodingOrDefault))
      flush()
      out.close()
    }
    this.headWritten = false
    this.out = out
  }

  // Mentioned as part of StreamHandler javadocs but it doesn't specify behavior
  override def setEncoding(encoding: String): Unit = super.setEncoding(encoding)

  private def write(c: Formatter => String): Unit = {
    // The javadocs don't specify what to do if the formatter is null
    if (out != null && formatter != null) {
      out.write(c(formatter).getBytes(encodingOrDefault))
      flush()
    }
  }

  private[logging] def writeHeader(): Unit = {
    if (!headWritten) {
      write(_.getHead(this))
      headWritten = true
    }
  }

  private[logging] def writeTail(): Unit = write(_.getTail(this))

  override def publish(record: LogRecord): Unit = {
    writeHeader()
    // The javadocs don't specify what to do if the formatter is null
    if (out != null && formatter != null && isLoggable(record)) {
      out.write(formatter.format(record).getBytes(encodingOrDefault))
      flush()
    }
  }

  override def isLoggable(record:LogRecord): Boolean =
    out != null && record != null && super.isLoggable(record)

  override def flush(): Unit = if (out != null) out.flush()

  override def close(): Unit = {
    if (out != null) {
      // Required by javadocs
      writeHeader()
      writeTail()
      flush()
      out.close()
    }
  }
} 

package utils.io

import java.io.{ByteArrayOutputStream, OutputStream, PrintStream}

class CaptureOutputStream extends OutputStream {
  val data = new ByteArrayOutputStream()

  override def write(b: Int): Unit = data.write(b)
  override def write(b: Array[Byte]): Unit = data.write(b)
  override def write(b: Array[Byte], off: Int, len: Int): Unit = data.write(b,off,len)

  def dump: String = new java.lang.String(data.toByteArray, java.nio.charset.StandardCharsets.UTF_8)
}

class CaptureStream(__out: CaptureOutputStream, paired: PrintStream) extends PrintStream(__out) {
  def this(paired: PrintStream) = this(new CaptureOutputStream(), paired)
  def dump: String = __out.dump
  //TODO[5]: For some reason this duplicates the printing
  //override def print(s: String): Unit = { paired.print(s); super.print(s) }
  //override def println(s: String): Unit = { paired.println(s); super.println(s) }
} 

package io.eels.component.avro

import java.io.OutputStream
import java.util.concurrent.atomic.AtomicInteger

import io.eels.Row
import io.eels.schema.StructType
import org.apache.avro.file.DataFileWriter
import org.apache.avro.generic
import org.apache.avro.generic.GenericRecord

class AvroWriter(structType: StructType, out: OutputStream) {
  
  private val schema = AvroSchemaFns.toAvroSchema(structType)
  private val datumWriter = new generic.GenericDatumWriter[GenericRecord](schema)
  private val dataFileWriter = new DataFileWriter[GenericRecord](datumWriter)
  private val serializer = new RowSerializer(schema)
  private val _records = new AtomicInteger(0)

  dataFileWriter.create(schema, out)

  def write(row: Row): Unit = {
    val record = serializer.serialize(row)
    dataFileWriter.append(record)
    _records.incrementAndGet()
  }

  def records: Int = _records.get()

  def close(): Unit = {
    dataFileWriter.flush()
    dataFileWriter.close()
  }
} 

package io.eels.component.csv

import java.io.{OutputStream, Writer}

import com.univocity.parsers.csv.{CsvParser, CsvParserSettings, CsvWriter, CsvWriterSettings}

object CsvSupport {

  def createParser(format: CsvFormat,
                   ignoreLeadingWhitespaces: Boolean = true,
                   ignoreTrailingWhitespaces: Boolean = true,
                   skipEmptyLines: Boolean = true,
                   emptyCellValue: String = null,
                   nullValue: String = null,
                   skipRows: Option[Long] = None,
                   selectedColumns: Seq[String] = Seq.empty): CsvParser = {
    val settings = new CsvParserSettings()
    settings.getFormat.setDelimiter(format.delimiter)
    settings.getFormat.setQuote(format.quoteChar)
    settings.getFormat.setQuoteEscape(format.quoteEscape)
    settings.setLineSeparatorDetectionEnabled(true)
    // this is always true as we will fetch the headers ourselves by reading first row
    settings.setHeaderExtractionEnabled(false)
    settings.setIgnoreLeadingWhitespaces(ignoreLeadingWhitespaces)
    settings.setIgnoreTrailingWhitespaces(ignoreTrailingWhitespaces)
    settings.setSkipEmptyLines(skipEmptyLines)
    settings.setCommentCollectionEnabled(true)
    settings.setEmptyValue(emptyCellValue)
    settings.setNullValue(nullValue)
    settings.setMaxCharsPerColumn(-1)
    settings.setMaxColumns(2048)
    settings.setReadInputOnSeparateThread(false)
    skipRows.foreach(settings.setNumberOfRowsToSkip)
    selectedColumns.headOption.foreach(_ => settings.selectFields(selectedColumns: _*))
    new com.univocity.parsers.csv.CsvParser(settings)
  }

  def writerSettings(format: CsvFormat,
                     ignoreLeadingWhitespaces: Boolean,
                     ignoreTrailingWhitespaces: Boolean): CsvWriterSettings = {
    val settings = new CsvWriterSettings()
    settings.getFormat.setDelimiter(format.delimiter)
    settings.getFormat.setQuote(format.quoteChar)
    settings.getFormat.setQuoteEscape(format.quoteEscape)
    // we will handle header writing ourselves
    settings.setHeaderWritingEnabled(false)
    settings.setIgnoreLeadingWhitespaces(ignoreLeadingWhitespaces)
    settings.setIgnoreTrailingWhitespaces(ignoreTrailingWhitespaces)
    settings
  }

  def createWriter(writer: Writer,
                   format: CsvFormat,
                   ignoreLeadingWhitespaces: Boolean,
                   ignoreTrailingWhitespaces: Boolean): CsvWriter = {
    new CsvWriter(writer, writerSettings(format, ignoreLeadingWhitespaces, ignoreTrailingWhitespaces))
  }

  def createWriter(output: OutputStream,
                   format: CsvFormat,
                   ignoreLeadingWhitespaces: Boolean,
                   ignoreTrailingWhitespaces: Boolean): CsvWriter = {
    new CsvWriter(output, writerSettings(format, ignoreLeadingWhitespaces, ignoreTrailingWhitespaces))
  }
} 

package blobstore
package sftp

import java.util.Date

import com.jcraft.jsch._
import cats.instances.option._

import scala.util.Try
import java.io.OutputStream

import cats.Traverse
import cats.effect.{Blocker, ConcurrentEffect, ContextShift, IO, Resource}
import cats.effect.concurrent.{MVar, Semaphore}
import fs2.concurrent.Queue

final class SftpStore[F[_]](
  absRoot: String,
  session: Session,
  blocker: Blocker,
  mVar: MVar[F, ChannelSftp],
  semaphore: Option[Semaphore[F]],
  connectTimeout: Int
)(implicit F: ConcurrentEffect[F], CS: ContextShift[F]) extends Store[F] {
  import implicits._

  import Path.SEP

  private val openChannel: F[ChannelSftp] = {
    val openF = blocker.delay{
      val ch = session.openChannel("sftp").asInstanceOf[ChannelSftp]
      ch.connect(connectTimeout)
      ch
    }
    semaphore.fold(openF){s =>
      F.ifM(s.tryAcquire)(openF, getChannel)
    }
  }

  private val getChannel = F.flatMap(mVar.tryTake) {
    case Some(channel) => F.pure(channel)
    case None => openChannel
  }

  private def channelResource: Resource[F, ChannelSftp] = Resource.make{
    getChannel
  }{
    case ch if ch.isClosed => F.unit
    case ch => F.ifM(mVar.tryPut(ch))(F.unit, SftpStore.closeChannel(semaphore, blocker)(ch))
  }

  
  def apply[F[_]](
    absRoot: String,
    fa: F[Session],
    blocker: Blocker,
    maxChannels: Option[Long] = None,
    connectTimeout: Int = 10000
  )(implicit F: ConcurrentEffect[F], CS: ContextShift[F]): fs2.Stream[F, SftpStore[F]] =
    if (maxChannels.exists(_ < 1)) {
      fs2.Stream.raiseError[F](new IllegalArgumentException(s"maxChannels must be >= 1"))
    } else {
      for {
        session <- fs2.Stream.bracket(fa)(session => F.delay(session.disconnect()))
        semaphore <- fs2.Stream.eval(Traverse[Option].sequence(maxChannels.map(Semaphore.apply[F])))
        mVar <- fs2.Stream.bracket(MVar.empty[F, ChannelSftp])(mVar => F.flatMap(mVar.tryTake)(_.fold(F.unit)(closeChannel[F](semaphore, blocker))))
      } yield new SftpStore[F](absRoot, session, blocker, mVar, semaphore, connectTimeout)
    }

  private def closeChannel[F[_]](semaphore: Option[Semaphore[F]], blocker: Blocker)(ch: ChannelSftp)(implicit F: ConcurrentEffect[F], CS: ContextShift[F]): F[Unit] =
    F.productR(semaphore.fold(F.unit)(_.release))(blocker.delay(ch.disconnect()))
} 

import java.io.OutputStream
import java.nio.file.Files

import cats.effect.{ContextShift, Sync, Blocker}
import fs2.{Pipe, Pull, Stream}
import cats.implicits._

package object blobstore {
  protected[blobstore] def _writeAllToOutputStream1[F[_]](in: Stream[F, Byte], out: OutputStream, blocker: Blocker)(
    implicit F: Sync[F], CS: ContextShift[F]): Pull[F, Nothing, Unit] = {
    in.pull.uncons.flatMap {
      case None => Pull.done
      case Some((hd, tl)) => Pull.eval[F, Unit](blocker.delay(out.write(hd.toArray))) >> _writeAllToOutputStream1(tl, out, blocker)
    }
  }

  protected[blobstore] def bufferToDisk[F[_]](chunkSize: Int, blocker: Blocker)(implicit F: Sync[F], CS: ContextShift[F])
  : Pipe[F, Byte, (Long, Stream[F, Byte])] = {
    in => Stream.bracket(F.delay(Files.createTempFile("bufferToDisk", ".bin")))(
      p => F.delay(p.toFile.delete).void).flatMap { p =>
        in.through(fs2.io.file.writeAll(p, blocker)).drain ++
        Stream.emit((p.toFile.length, fs2.io.file.readAll(p, blocker, chunkSize)))
    }
  }

} 

package it.agilelab.darwin.manager.util

import java.io.OutputStream
import java.nio.{ByteBuffer, ByteOrder}

import it.agilelab.darwin.common.LONG_SIZE


private[darwin] object ByteArrayUtils {

  implicit class EnrichedLong(val l: Long) extends AnyVal {
    
    def writeToStream(os: OutputStream, endianness: ByteOrder): Unit = {
      endianness match {
        case ByteOrder.BIG_ENDIAN =>
          os.write((l >>> 56).asInstanceOf[Int])
          os.write((l >>> 48).asInstanceOf[Int])
          os.write((l >>> 40).asInstanceOf[Int])
          os.write((l >>> 32).asInstanceOf[Int])
          os.write((l >>> 24).asInstanceOf[Int])
          os.write((l >>> 16).asInstanceOf[Int])
          os.write((l >>> 8).asInstanceOf[Int])
          os.write((l >>> 0).asInstanceOf[Int])
        case ByteOrder.LITTLE_ENDIAN =>
          os.write((l >>> 0).asInstanceOf[Int])
          os.write((l >>> 8).asInstanceOf[Int])
          os.write((l >>> 16).asInstanceOf[Int])
          os.write((l >>> 24).asInstanceOf[Int])
          os.write((l >>> 32).asInstanceOf[Int])
          os.write((l >>> 40).asInstanceOf[Int])
          os.write((l >>> 48).asInstanceOf[Int])
          os.write((l >>> 56).asInstanceOf[Int])
      }
    }
  }

  def arrayEquals(b1: Array[Byte], b2: Array[Byte], start1: Int, start2: Int, length: Int): Boolean = {
    require(length > 0, "length must be positive")
    var i = start1
    var j = start2
    var areEqual = true
    while (areEqual && i < start1 + length) {
      if (b1(i) != b2(j)) {
        areEqual = false
      }
      i += 1
      j += 1
    }
    areEqual
  }


} 

package org.apache.spark.sql.execution.datasources

import java.io.{InputStream, OutputStream, OutputStreamWriter}
import java.nio.charset.{Charset, StandardCharsets}

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.hadoop.io.compress._
import org.apache.hadoop.mapreduce.JobContext
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
import org.apache.hadoop.util.ReflectionUtils

import org.apache.spark.TaskContext

object CodecStreams {
  private def getDecompressionCodec(config: Configuration, file: Path): Option[CompressionCodec] = {
    val compressionCodecs = new CompressionCodecFactory(config)
    Option(compressionCodecs.getCodec(file))
  }

  def createInputStream(config: Configuration, file: Path): InputStream = {
    val fs = file.getFileSystem(config)
    val inputStream: InputStream = fs.open(file)

    getDecompressionCodec(config, file)
      .map(codec => codec.createInputStream(inputStream))
      .getOrElse(inputStream)
  }

  
  def getCompressionExtension(context: JobContext): String = {
    getCompressionCodec(context)
      .map(_.getDefaultExtension)
      .getOrElse("")
  }
} 

package org.apache.spark.sql.execution.streaming


import java.io.{InputStream, OutputStream}
import java.nio.charset.StandardCharsets._

import scala.io.{Source => IOSource}

import org.apache.spark.sql.SparkSession


class OffsetSeqLog(sparkSession: SparkSession, path: String)
  extends HDFSMetadataLog[OffsetSeq](sparkSession, path) {

  override protected def deserialize(in: InputStream): OffsetSeq = {
    // called inside a try-finally where the underlying stream is closed in the caller
    def parseOffset(value: String): Offset = value match {
      case OffsetSeqLog.SERIALIZED_VOID_OFFSET => null
      case json => SerializedOffset(json)
    }
    val lines = IOSource.fromInputStream(in, UTF_8.name()).getLines()
    if (!lines.hasNext) {
      throw new IllegalStateException("Incomplete log file")
    }

    val version = parseVersion(lines.next(), OffsetSeqLog.VERSION)

    // read metadata
    val metadata = lines.next().trim match {
      case "" => None
      case md => Some(md)
    }
    OffsetSeq.fill(metadata, lines.map(parseOffset).toArray: _*)
  }

  override protected def serialize(offsetSeq: OffsetSeq, out: OutputStream): Unit = {
    // called inside a try-finally where the underlying stream is closed in the caller
    out.write(("v" + OffsetSeqLog.VERSION).getBytes(UTF_8))

    // write metadata
    out.write('\n')
    out.write(offsetSeq.metadata.map(_.json).getOrElse("").getBytes(UTF_8))

    // write offsets, one per line
    offsetSeq.offsets.map(_.map(_.json)).foreach { offset =>
      out.write('\n')
      offset match {
        case Some(json: String) => out.write(json.getBytes(UTF_8))
        case None => out.write(OffsetSeqLog.SERIALIZED_VOID_OFFSET.getBytes(UTF_8))
      }
    }
  }
}

object OffsetSeqLog {
  private[streaming] val VERSION = 1
  private val SERIALIZED_VOID_OFFSET = "-"
} 

package org.apache.spark.sql.execution.streaming

import java.io.{InputStream, OutputStream}
import java.nio.charset.StandardCharsets._

import scala.io.{Source => IOSource}

import org.json4s.NoTypeHints
import org.json4s.jackson.Serialization

import org.apache.spark.sql.SparkSession


class CommitLog(sparkSession: SparkSession, path: String)
  extends HDFSMetadataLog[CommitMetadata](sparkSession, path) {

  import CommitLog._

  override protected def deserialize(in: InputStream): CommitMetadata = {
    // called inside a try-finally where the underlying stream is closed in the caller
    val lines = IOSource.fromInputStream(in, UTF_8.name()).getLines()
    if (!lines.hasNext) {
      throw new IllegalStateException("Incomplete log file in the offset commit log")
    }
    parseVersion(lines.next.trim, VERSION)
    val metadataJson = if (lines.hasNext) lines.next else EMPTY_JSON
    CommitMetadata(metadataJson)
  }

  override protected def serialize(metadata: CommitMetadata, out: OutputStream): Unit = {
    // called inside a try-finally where the underlying stream is closed in the caller
    out.write(s"v${VERSION}".getBytes(UTF_8))
    out.write('\n')

    // write metadata
    out.write(metadata.json.getBytes(UTF_8))
  }
}

object CommitLog {
  private val VERSION = 1
  private val EMPTY_JSON = "{}"
}


case class CommitMetadata(nextBatchWatermarkMs: Long = 0) {
  def json: String = Serialization.write(this)(CommitMetadata.format)
}

object CommitMetadata {
  implicit val format = Serialization.formats(NoTypeHints)

  def apply(json: String): CommitMetadata = Serialization.read[CommitMetadata](json)
} 

package com.orendainx.trucking.nifi.processors

import java.io.{InputStream, OutputStream}
import java.nio.charset.StandardCharsets
import java.util.concurrent.atomic.AtomicReference
import java.util.Scanner

import com.orendainx.trucking.commons.models.{EnrichedTruckData, TruckData}
import com.orendainx.trucking.enrichment.WeatherAPI
import org.apache.nifi.annotation.behavior._
import org.apache.nifi.annotation.documentation.{CapabilityDescription, Tags}
import org.apache.nifi.components.PropertyDescriptor
import org.apache.nifi.logging.ComponentLog
import org.apache.nifi.processor.io.InputStreamCallback
import org.apache.nifi.processor.io.OutputStreamCallback
import org.apache.nifi.processor._

import scala.collection.JavaConverters._


@Tags(Array("trucking", "data", "event", "enrich", "iot"))
@CapabilityDescription("Enriches simulated truck sensor data. Find the master project and its code, documentation and corresponding tutorials at: https://github.com/orendain/trucking-iot")
@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
@TriggerSerially
@WritesAttributes(Array(
  new WritesAttribute(attribute = "dataType", description = "The class name of the resulting enriched data type.")
))
class EnrichTruckData extends AbstractProcessor {

  private var log: ComponentLog = _
  private val RelSuccess = new Relationship.Builder().name("success").description("All generated data is routed to this relationship.").build

  override def init(context: ProcessorInitializationContext): Unit = {
    log = context.getLogger
  }

  override def onTrigger(context: ProcessContext, session: ProcessSession): Unit = {

    var flowFile = session.get
    log.debug(s"Flowfile received: $flowFile")

    // Convert the entire stream of bytes from the flow file into a string
    val content = new AtomicReference[String]
    session.read(flowFile, new InputStreamCallback {
      override def process(inputStream: InputStream) = {
        val scanner = new Scanner(inputStream).useDelimiter("\\A")
        val result = if (scanner.hasNext()) scanner.next() else ""
        log.debug(s"Parsed content: $result")
        content.set(result)
      }
    })

    // Form a TruckData object from content, then creating an EnrichedTruckData object by making the appropriate
    // calls to WeatherAPI
    val truckData = TruckData.fromCSV(content.get())
    val enrichedTruckData = EnrichedTruckData(truckData, WeatherAPI.default.getFog(truckData.eventType),
      WeatherAPI.default.getRain(truckData.eventType), WeatherAPI.default.getWind(truckData.eventType))

    log.debug(s"EnrichedData generated: $enrichedTruckData")

    // Add the new data type as a flow file attribute
    flowFile = session.putAttribute(flowFile, "dataType", enrichedTruckData.getClass.getSimpleName)

    // Replace the flow file, writing in the new content
    flowFile = session.write(flowFile, new OutputStreamCallback {
      override def process(outputStream: OutputStream) =
        outputStream.write(enrichedTruckData.toCSV.getBytes(StandardCharsets.UTF_8))
    })

    // TODO: document what this does
    session.getProvenanceReporter.route(flowFile, RelSuccess)
    session.transfer(flowFile, RelSuccess)
    session.commit()
  }

  // Define properties and relationships
  override def getSupportedPropertyDescriptors: java.util.List[PropertyDescriptor] = List.empty[PropertyDescriptor].asJava

  override def getRelationships: java.util.Set[Relationship] = Set(RelSuccess).asJava
} 

package com.stefansavev.core.serialization

import java.io.{InputStream, OutputStream}


object StreamExtensions{

  implicit class IntSerializerExt(outputStream: OutputStream) {
    def writeInt(value: Int): Unit = {
      IntSerializer.write(outputStream, value)
    }
  }

  implicit class IntDeSerializerExt(inputStream: InputStream) {
    def readInt(): Int = {
      IntSerializer.read(inputStream)
    }
  }

  implicit class DoubleArraySerializerExt(outputStream: OutputStream) {
    def writeDoubleArray(values: Array[Double]): Unit = {
      DoubleArraySerializer.write(outputStream, values)
    }
  }

  implicit class DoubleArrayDeSerializerExt(inputStream: InputStream) {
    def readDoubleArray(): Array[Double] = {
      DoubleArraySerializer.read(inputStream)
    }
  }

} 

package com.stefansavev.core.serialization

import java.io.{InputStream, OutputStream}

import com.stefansavev.core.serialization.PrimitiveTypeSerializers.TypedIntSerializer

import scala.reflect.ClassTag

object GenericArraySerializer {

  
  //
  class GenericArraySerializer[T: ClassTag](ev: T <:< AnyRef, nestedSerializer: TypedSerializer[T]) extends TypedSerializer[Array[T]] {
    val clazz = scala.reflect.classTag[T].runtimeClass
    if (clazz.equals(Int.getClass) || clazz.equals(Double.getClass)) {
      throw new IllegalStateException("GenericArraySerializer should not be applied to primitive types")
    }

    def toBinary(outputStream: OutputStream, input: Array[T]): Unit = {
      TypedIntSerializer.toBinary(outputStream, input.length)
      var i = 0
      while (i < input.length) {
        nestedSerializer.toBinary(outputStream, input(i))
        i += 1
      }
    }

    def fromBinary(inputStream: InputStream): Array[T] = {
      val len = TypedIntSerializer.fromBinary(inputStream)
      val output = Array.ofDim[T](len)
      var i = 0
      while (i < len) {
        output(i) = nestedSerializer.fromBinary(inputStream)
        i += 1
      }
      output
    }

    def name: String = s"GenericArraySerializer(${nestedSerializer.name})"
  }

  implicit def genericArraySerializer[T: ClassTag](implicit ev: T <:< AnyRef, nestedSerializer: TypedSerializer[T]): GenericArraySerializer[T] = {
    new GenericArraySerializer[T](ev, nestedSerializer)
  }
} 

package com.stefansavev.core.serialization

import java.io.{InputStream, OutputStream}

object IsoSerializers {
  class IsoSerializer[A, B](iso: Iso[A, B], serB: TypedSerializer[B]) extends TypedSerializer[A]{
    def toBinary(outputStream: OutputStream, input: A): Unit = {
      serB.toBinary(outputStream, iso.from(input))
    }

    def fromBinary(inputStream: InputStream): A = {
      iso.to(serB.fromBinary(inputStream))
    }

    def name: String = s"IsoSerializer(via = ${serB.name})"
  }

  implicit def isoSerializer[A, B](implicit iso: Iso[A, B], serB: TypedSerializer[B]): IsoSerializer[A, B] = {
    new IsoSerializer[A, B](iso, serB)
  }
} 

package com.stefansavev.core.string2id

import java.io.{InputStream, OutputStream}

import com.stefansavev.core.serialization.{StringSerializer, IntSerializer, TypedSerializer}
import com.stefansavev.core.serialization.StringSerializer

object String2IdHasherSerialization {

  implicit object String2IdHasherSerializer extends TypedSerializer[String2IdHasher] {
    def toBinary(outputStream: OutputStream, string2Id: String2IdHasher): Unit = {
      if (string2Id == null){
        throw new IllegalStateException("string2IdHasher cannot be null")
      }
      val settings = string2Id.getSettings()
      IntSerializer.write(outputStream, settings.maxValues)
      IntSerializer.write(outputStream, settings.avgStringLen)
      IntSerializer.write(outputStream, settings.toleratedNumberOfCollisions)
      var id = 0
      val numStrings = string2Id.numberOfUniqueStrings()
      IntSerializer.write(outputStream, numStrings)
      while(id < numStrings){
        val str = string2Id.getStringAtInternalIndex(id).get
        StringSerializer.write(outputStream, str)
        id += 1
      }
    }

    def fromBinary(inputStream: InputStream): String2IdHasher = {
      
      val maxValues = IntSerializer.read(inputStream)
      val avgStringLen = IntSerializer.read(inputStream)
      val numCollisions = IntSerializer.read(inputStream)
      val settings = new StringIdHasherSettings(maxValues, avgStringLen, numCollisions)
      val string2IdHasher = new String2IdHasher(settings)
      val numStrings = IntSerializer.read(inputStream)
      var i = 0
      while(i < numStrings){
        val str = StringSerializer.read(inputStream)
        val handle = string2IdHasher.add(str)
        val index = string2IdHasher.getInternalId(handle)
        if (index != i){
          throw new IllegalStateException("Internal error while reading hashed strings")
        }
        i += 1
      }
      string2IdHasher
    }

    def name: String = "String2IdHasherSerializer"
  }


} 

package co.ledger.wallet.core.utils.logs

import java.io.{OutputStream, PrintStream}

import scala.scalajs.js
import js.Dynamic.global

  override def flush(): Unit = if (!flushed) {
    doWriteLine(buffer + LineContEnd)
    buffer = LineContStart
    flushed = true
  }

  override def close(): Unit = ()

  private def doWriteLine(line: String): Unit = {
    Logger.log(logLevel, "Global", line)
  }
}

object LoggerPrintStream {
  private final val LineContEnd: String = "\u21A9"
  private final val LineContStart: String = "\u21AA"

  class DummyOutputStream extends OutputStream {
    def write(c: Int): Unit =
      throw new AssertionError(
        "Should not get in JSConsoleBasedPrintStream.DummyOutputStream")
  }

  def init(): Unit = {
    System.setErr(new LoggerPrintStream("E"))
    System.setOut(new LoggerPrintStream("D"))
  }
} 

package org.apache.spark.sql.execution.streaming.http

import java.nio.ByteBuffer
import org.apache.spark.serializer.SerializerInstance
import org.apache.spark.serializer.DeserializationStream
import org.apache.spark.serializer.SerializationStream
import java.io.OutputStream
import java.io.InputStream
import scala.reflect.ClassTag
import com.fasterxml.jackson.databind.ObjectMapper
import org.apache.spark.SparkConf
import org.apache.spark.serializer.JavaSerializer
import org.apache.spark.serializer.KryoSerializer


object SerializerFactory {
	val DEFAULT = new SerializerFactory {
		override def getSerializerInstance(serializerName: String): SerializerInstance = {
			serializerName.toLowerCase() match {
				case "kryo" ⇒
					new KryoSerializer(new SparkConf()).newInstance();
				case "java" ⇒
					new JavaSerializer(new SparkConf()).newInstance();
				case _ ⇒ throw new InvalidSerializerNameException(serializerName);
			}
		}
	}
}

trait SerializerFactory {
	def getSerializerInstance(serializerName: String): SerializerInstance;
} 

package ml.combust.mleap.runtime.serialization

import java.io.{File, FileOutputStream, OutputStream}
import java.nio.charset.Charset

import ml.combust.mleap.ClassLoaderUtil
import ml.combust.mleap.runtime.frame.LeapFrame
import resource._

import scala.reflect.ClassTag
import scala.util.Try


object FrameWriter {
  def apply[LF <: LeapFrame[LF]](frame: LF,
                                 format: String = BuiltinFormats.json,
                                 clOption: Option[ClassLoader] = None)
                                (implicit ct: ClassTag[LF]): FrameWriter = {
    val cl = clOption.getOrElse(ClassLoaderUtil.findClassLoader(classOf[FrameWriter].getCanonicalName))
    cl.loadClass(s"$format.DefaultFrameWriter").
      getConstructor(classOf[LeapFrame[_]]).
      newInstance(frame).
      asInstanceOf[FrameWriter]
  }
}

trait FrameWriter {
  def toBytes(charset: Charset = BuiltinFormats.charset): Try[Array[Byte]]

  def save(file: File): Try[Any] = save(file, BuiltinFormats.charset)
  def save(file: File, charset: Charset = BuiltinFormats.charset): Try[Any] = {
    (for(out <- managed(new FileOutputStream(file))) yield {
      save(out, charset)
    }).tried.flatMap(identity)
  }

  def save(out: OutputStream): Try[Any] = save(out, BuiltinFormats.charset)
  def save(out: OutputStream, charset: Charset): Try[Any] = {
    toBytes(charset).map(out.write)
  }
} 

package wvlet.airframe.msgpack.spi
import java.io.{InputStream, OutputStream}

import wvlet.airframe.json.{JSON, JSONScanner, JSONSource}
import wvlet.airframe.msgpack.json.{NestedMessagePackBuilder, StreamMessagePackBuilder}


object MessagePack {
  def newBufferPacker: BufferPacker                                      = Compat.newBufferPacker
  def newPacker(out: OutputStream): Packer                               = Compat.newPacker(out)
  def newUnpacker(in: InputStream): Unpacker                             = Compat.newUnpacker(in)
  def newUnpacker(msgpack: Array[Byte]): Unpacker                        = Compat.newUnpacker(msgpack)
  def newUnpacker(msgpack: Array[Byte], offset: Int, len: Int): Unpacker = Compat.newUnpacker(msgpack, offset, len)

  def fromJSON(json: String): MsgPack           = fromJSON(JSONSource.fromString(json))
  def fromJSON(jsonBytes: Array[Byte]): MsgPack = fromJSON(JSONSource.fromBytes(jsonBytes))
  def fromJSON(json: JSONSource): MsgPack = {
    val context = new StreamMessagePackBuilder()
    JSONScanner.scanAny(json, context)
    context.result
  }
} 

package wvlet.airframe.msgpack.spi
import java.io.{InputStream, OutputStream}

import org.msgpack.{core => mj}
import wvlet.airframe.msgpack.impl.{
  BufferPackerImpl,
  PackerImpl,
  PureScalaBufferPacker,
  PureScalaBufferUnpacker,
  UnpackerImpl
}
import wvlet.airframe.msgpack.io.ByteArrayBuffer


object Compat {
  def isScalaJS = false

  def floatToIntBits(v: Float): Int     = java.lang.Float.floatToRawIntBits(v)
  def doubleToLongBits(v: Double): Long = java.lang.Double.doubleToRawLongBits(v)

  def newBufferPacker: BufferPacker = {
    new PureScalaBufferPacker
    //new BufferPackerImpl(mj.MessagePack.newDefaultBufferPacker())
  }

  def newPacker(out: OutputStream): Packer = {
    // TODO: Use pure-scala packer
    // new PureScalaBufferPacker
    new PackerImpl(mj.MessagePack.newDefaultPacker(out))
  }

  def newUnpacker(in: InputStream): Unpacker = {
    new UnpackerImpl(mj.MessagePack.newDefaultUnpacker(in))
  }

  def newUnpacker(msgpack: Array[Byte]): Unpacker = {
    newUnpacker(msgpack, 0, msgpack.length)
  }

  def newUnpacker(msgpack: Array[Byte], offset: Int, len: Int): Unpacker = {
    //new UnpackerImpl(mj.MessagePack.newDefaultUnpacker(msgpack, offset, len))
    // Use pure-scala unpacker
    new PureScalaBufferUnpacker(ByteArrayBuffer.fromArray(msgpack, offset, len))
  }
} 

package com.amazon.milan.tools

import java.io.{OutputStream, OutputStreamWriter}
import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets
import java.nio.file.Files

import com.amazon.milan.application.{Application, ApplicationConfiguration, ApplicationInstance}
import com.amazon.milan.lang._
import com.amazon.milan.testing.applications._
import com.amazon.milan.{Id, SemanticVersion}
import org.junit.Assert._
import org.junit.Test


object TestCompileApplicationInstance {

  case class Record(recordId: String, i: Int)

  class Provider extends ApplicationInstanceProvider {
    override def getApplicationInstance(params: List[(String, String)]): ApplicationInstance = {
      val input = Stream.of[Record]
      val graph = new StreamGraph(input)
      val config = new ApplicationConfiguration
      config.setListSource(input, Record("1", 1))

      val instanceId = params.find(_._1 == "instanceId").get._2
      val appId = params.find(_._1 == "appId").get._2

      new ApplicationInstance(
        instanceId,
        new Application(appId, graph, SemanticVersion.ZERO),
        config)
    }
  }

  class Compiler extends ApplicationInstanceCompiler {
    override def compile(applicationInstance: ApplicationInstance,
                         params: List[(String, String)],
                         output: OutputStream): Unit = {
      val writer = new OutputStreamWriter(output)
      val testParam = params.find(_._1 == "test").get._2
      writer.write(testParam)
      writer.write(applicationInstance.toJsonString)
      writer.close()
    }
  }

}

@Test
class TestCompileApplicationInstance {
  @Test
  def test_CompileApplicationInstance_Main_SendsProviderAndCompilerParameters(): Unit = {

    val tempFile = Files.createTempFile("TestCompileApplicationInstance", ".scala")
    Files.deleteIfExists(tempFile)

    val appId = Id.newId()
    val instanceId = Id.newId()
    val testValue = Id.newId()

    try {
      val args = Array(
        "--provider",
        "com.amazon.milan.tools.TestCompileApplicationInstance.Provider",
        "--compiler",
        "com.amazon.milan.tools.TestCompileApplicationInstance.Compiler",
        "--package",
        "generated",
        "--output",
        tempFile.toString,
        s"-PinstanceId=$instanceId",
        s"-PappId=$appId",
        s"-Ctest=$testValue"
      )
      CompileApplicationInstance.main(args)

      val fileContents = StandardCharsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(tempFile))).toString
      assertTrue(fileContents.contains(appId))
      assertTrue(fileContents.contains(instanceId))
      assertTrue(fileContents.contains(testValue))
    }
    finally {
      Files.deleteIfExists(tempFile)
    }
  }
} 

package com.amazon.milan.dataformats

import java.io.OutputStream
import java.nio.charset.StandardCharsets

import com.amazon.milan.HashUtil
import com.amazon.milan.serialization.{JavaTypeFactory, MilanObjectMapper}
import com.amazon.milan.typeutil.TypeDescriptor
import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize}

import scala.collection.JavaConverters._



@JsonSerialize
@JsonDeserialize
class JsonDataOutputFormat[T: TypeDescriptor] extends DataOutputFormat[T] {
  @transient private lazy val objectMapper = new MilanObjectMapper()
  @transient private lazy val javaType = new JavaTypeFactory(this.objectMapper.getTypeFactory).makeJavaType(this.recordTypeDescriptor)
  @transient private lazy val hashCodeValue = HashUtil.combineHashCodes(this.recordTypeDescriptor.hashCode())
  @transient private lazy val writer = this.objectMapper.writerFor(this.javaType)
  @transient private lazy val newLine = "\n".getBytes(StandardCharsets.UTF_8)

  private var recordTypeDescriptor = implicitly[TypeDescriptor[T]]

  override def getGenericArguments: List[TypeDescriptor[_]] =
    List(implicitly[TypeDescriptor[T]])

  override def setGenericArguments(genericArgs: List[TypeDescriptor[_]]): Unit = {
    this.recordTypeDescriptor = genericArgs.head.asInstanceOf[TypeDescriptor[T]]
  }

  override def writeValue(value: T, outputStream: OutputStream): Unit = {
    this.writer.writeValue(outputStream, value)
    outputStream.write(this.newLine)
  }

  override def writeValues(values: TraversableOnce[T], outputStream: OutputStream): Unit = {
    this.writer
      .withRootValueSeparator("\n")
      .writeValues(outputStream)
      .writeAll(values.toIterable.asJava)
    outputStream.write(this.newLine)
  }

  override def hashCode(): Int = this.hashCodeValue

  override def equals(obj: Any): Boolean = {
    obj match {
      case o: JsonDataOutputFormat[T] =>
        this.recordTypeDescriptor.equals(o.recordTypeDescriptor)

      case _ =>
        false
    }
  }
} 

package com.amazon.milan.compiler.flink.generator

import java.io.{ByteArrayOutputStream, OutputStream}
import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Path, StandardOpenOption}

import com.amazon.milan.application.{Application, ApplicationConfiguration, ApplicationInstance}
import com.amazon.milan.compiler.flink.internal.FlinkTypeEmitter
import com.amazon.milan.lang.StreamGraph
import com.amazon.milan.program.{Cycle, StreamExpression}
import com.amazon.milan.{Id, SemanticVersion}
import com.typesafe.scalalogging.Logger
import org.slf4j.LoggerFactory


case class GeneratorConfig(preventGenericTypeInformation: Boolean = false)


object FlinkGenerator {
  val default = new FlinkGenerator(GeneratorConfig())
}


class FlinkGenerator(classLoader: ClassLoader, generatorConfig: GeneratorConfig) {

  private val generatorTypeLifter = new FlinkTypeLifter(new FlinkTypeEmitter, this.generatorConfig.preventGenericTypeInformation)

  private val logger = Logger(LoggerFactory.getLogger(getClass))

  def this(generatorConfig: GeneratorConfig) {
    this(getClass.getClassLoader, generatorConfig)
  }

  def generateScala(graph: StreamGraph,
                    appConfig: ApplicationConfiguration,
                    packageName: String,
                    className: String): String = {
    val application = new Application(Id.newId(), graph, SemanticVersion.ZERO)
    val instance = new ApplicationInstance(Id.newId(), application, appConfig)
    this.generateScala(instance, packageName, className)
  }

  def generateScala(instance: ApplicationInstance,
                    outputPath: Path,
                    packageName: String,
                    className: String): Unit = {
    val scalaCode = this.generateScala(instance, packageName, className)
    val contents = scalaCode.getBytes(StandardCharsets.UTF_8)
    Files.write(outputPath, contents, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)
  }

  def generateScala(instance: ApplicationInstance,
                    packageName: String,
                    className: String): String = {
    val output = new ByteArrayOutputStream()
    this.generateScala(instance, output, packageName, className)

    output.flush()
    StandardCharsets.UTF_8.decode(ByteBuffer.wrap(output.toByteArray)).toString
  }

  def generateScala(instance: ApplicationInstance,
                    output: OutputStream,
                    packageName: String,
                    className: String): Unit = {
    val finalGraph = instance.application.graph.getDereferencedGraph
    finalGraph.typeCheckGraph()

    val outputs = new GeneratorOutputs(this.generatorTypeLifter)
    val context = GeneratorContext.createEmpty(instance.instanceDefinitionId, finalGraph, instance.config, outputs, this.generatorTypeLifter)

    // Ensure that every data stream is generated.
    finalGraph
      .getStreams
      .foreach(stream => this.ensureStreamIsGenerated(context, stream))

    // Close any cycles.
    finalGraph
      .getStreams
      .filter(_.isInstanceOf[Cycle])
      .map(_.asInstanceOf[Cycle])
      .foreach(context.closeCycle)

    // Add all sinks at the end.
    instance.config.dataSinks.foreach(sink => context.generateSink(sink))

    val generated = context.output.generateScala(packageName, className)
    output.write(generated.getBytes(StandardCharsets.UTF_8))
  }

  private def ensureStreamIsGenerated(context: GeneratorContext,
                                      stream: StreamExpression): Unit = {
    context.getOrGenerateDataStream(stream)
  }
} 

package com.amazon.milan.compiler.flink.runtime

import java.io.OutputStream
import java.nio.file.{Files, Paths, StandardOpenOption}

import com.amazon.milan.dataformats.DataOutputFormat
import org.apache.flink.streaming.api.functions.sink.SinkFunction


class FileSinkFunction[T](path: String, dataFormat: DataOutputFormat[T]) extends SinkFunction[T] {
  @transient private lazy val outputStream = this.openOutputStream()

  override def invoke(value: T, context: SinkFunction.Context[_]): Unit = {
    this.dataFormat.writeValue(value, this.outputStream)
    this.outputStream.flush()
  }

  private def openOutputStream(): OutputStream = {
    Files.newOutputStream(Paths.get(this.path), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)
  }
} 

package com.amazon.milan.compiler

import java.io.OutputStream
import java.nio.charset.StandardCharsets

import com.amazon.milan.typeutil.TypeDescriptor

import _root_.scala.language.implicitConversions


package object scala {

  implicit class MilanScalaCompilerStringExtensions(s: String) {
    def strip: String =
      this.s.stripMargin.stripLineEnd.stripPrefix("\n")

    def indent(level: Int): String = {
      val prefix = Seq.tabulate(level)(_ => "  ").mkString
      this.s.lines.map(line => prefix + line).mkString("\n")
    }

    def indentTail(level: Int): String = {
      val prefix = Seq.tabulate(level)(_ => "  ").mkString
      this.s.lines.zipWithIndex.map {
        case (line, index) =>
          if (index == 0) {
            line
          }
          else {
            prefix + line
          }
      }.mkString("\n")
    }

    def getUtf8Bytes: Array[Byte] =
      this.s.getBytes(StandardCharsets.UTF_8)
  }

  implicit class MilanScalaCompilerTypeDescriptorExtensions[_](t: TypeDescriptor[_]) {
    def toTerm: ClassName = {
      if (t.isTuple && t.genericArguments.isEmpty) {
        ClassName("Product")
      }
      else {
        ClassName(t.fullName)
      }
    }
  }

  implicit class MilanScalaOutputStreamExtensions(outputStream: OutputStream) {
    def writeUtf8(s: String): Unit = {
      this.outputStream.write(s.getUtf8Bytes)
    }
  }

} 

package org.apache.spark.shuffle

import java.io.{InputStream, OutputStream}
import java.util.Comparator

import org.apache.spark.internal.Logging

import scala.util.control.NonFatal

object SplashUtils extends Logging {
  def withResources[T <: AutoCloseable, V](r: => T)(f: T => V): V = {
    val resource: T = r
    require(resource != null, "resource is null")
    var exception: Throwable = null
    try {
      f(resource)
    } catch {
      case NonFatal(e) =>
        exception = e
        throw e
      case e: Throwable =>
        logError("fatal error received.", e)
        throw e
    } finally {
      closeAndAddSuppressed(exception, resource)
    }
  }

  private def closeAndAddSuppressed(e: Throwable,
      resource: AutoCloseable): Unit = {
    if (e != null) {
      try {
        resource.close()
      } catch {
        case NonFatal(suppressed) =>
          e.addSuppressed(suppressed)
      }
    } else {
      resource.close()
    }
  }

  
class SplashHashComparator[K] extends Comparator[K] {
  def compare(key1: K, key2: K): Int = {
    val hash1 = SplashUtils.hash(key1)
    val hash2 = SplashUtils.hash(key2)
    if (hash1 < hash2) -1 else if (hash1 == hash2) 0 else 1
  }
}


class SplashSpillableIterator[T](var upstream: Iterator[T],
    val spillInMemoryIterator: Iterator[T] => SpilledFile,
    val getNextUpstream: SpilledFile => Iterator[T])
    extends Iterator[T] with Logging {
  private val spillLock = new Object
  private var spilledFileOpt: Option[SpilledFile] = None
  private var cur: T = readNext()

  def spill(): Option[SpilledFile] = spillLock.synchronized {
    spilledFileOpt match {
      case Some(_) =>
        // has spilled, return None
        None
      case None =>
        // never spilled, now spilling
        val spilledFile = spillInMemoryIterator(upstream)
        spilledFileOpt = Some(spilledFile)
        spilledFileOpt
    }
  }

  def readNext(): T = spillLock.synchronized {
    spilledFileOpt match {
      case Some(spilledFile) =>
        upstream = getNextUpstream(spilledFile)
        spilledFileOpt = None
      case None =>
      // do nothing
    }
    if (upstream.hasNext) {
      upstream.next()
    } else {
      null.asInstanceOf[T]
    }
  }

  override def hasNext: Boolean = cur != null

  override def next(): T = {
    val ret = cur
    cur = readNext()
    ret
  }
} 

package scala.meta.jsonrpc

import java.io.InputStream
import java.io.OutputStream
import monix.execution.Cancelable


final class InputOutput(val in: InputStream, val out: OutputStream)
    extends Cancelable {
  override def cancel(): Unit = {
    Cancelable.cancelAll(
      List(
        Cancelable(() => in.close()),
        Cancelable(() => out.close())
      )
    )
  }
} 

package scala.meta.jsonrpc

import java.io.ByteArrayOutputStream
import java.io.OutputStream
import java.io.OutputStreamWriter
import java.io.PrintWriter
import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets
import scala.concurrent.Future
import io.circe.syntax._
import monix.execution.Ack
import monix.reactive.Observer
import scribe.LoggerSupport


  def write(msg: Message): Future[Ack] = lock.synchronized {
    baos.reset()
    val json = msg.asJson
    val protocol = BaseProtocolMessage.fromJson(json)
    logger.trace(s" --> $json")
    val byteBuffer = MessageWriter.write(protocol, baos, headerOut)
    out.onNext(byteBuffer)
  }
}

object MessageWriter {

  def headerWriter(out: OutputStream): PrintWriter = {
    new PrintWriter(new OutputStreamWriter(out, StandardCharsets.US_ASCII))
  }

  def write(message: BaseProtocolMessage): ByteBuffer = {
    val out = new ByteArrayOutputStream()
    val header = headerWriter(out)
    write(message, out, header)
  }

  def write(
      message: BaseProtocolMessage,
      out: ByteArrayOutputStream,
      headerOut: PrintWriter
  ): ByteBuffer = {
    message.header.foreach {
      case (key, value) =>
        headerOut.write(key)
        headerOut.write(": ")
        headerOut.write(value)
        headerOut.write("\r\n")
    }
    headerOut.write("\r\n")
    out.write(message.content)
    out.flush()
    val buffer = ByteBuffer.wrap(out.toByteArray, 0, out.size())
    buffer
  }
} 

package scala.meta.jsonrpc

import java.io.IOException
import java.io.OutputStream
import java.nio.ByteBuffer
import monix.execution.Ack
import monix.execution.Cancelable
import monix.execution.Scheduler
import monix.reactive.Observable
import monix.reactive.Observer
import scribe.LoggerSupport

object MonixEnrichments {

  
  class ObservableCurrentValue[+A](obs: Observable[A])(implicit s: Scheduler)
      extends (() => A)
      with Cancelable {
    private var value: Any = _
    private val cancelable = obs.foreach(newValue => value = newValue)
    override def apply(): A = {
      if (value == null) {
        throw new NoSuchElementException(
          "Reading from empty Observable, consider using MulticastStrategy.behavior(initialValue)"
        )
      } else {
        value.asInstanceOf[A]
      }
    }
    override def cancel(): Unit = cancelable.cancel()
  }

  implicit class XtensionObservable[A](val obs: Observable[A]) extends AnyVal {

    def focus[B: cats.Eq](f: A => B): Observable[B] =
      obs.distinctUntilChangedByKey(f).map(f)

    def toFunction0()(implicit s: Scheduler): () => A =
      toObservableCurrentValue()

    def toObservableCurrentValue()(
        implicit s: Scheduler
    ): ObservableCurrentValue[A] =
      new ObservableCurrentValue[A](obs)
  }

  implicit class XtensionObserverCompanion[A](val `_`: Observer.type)
      extends AnyVal {
    def fromOutputStream(
        out: OutputStream,
        logger: LoggerSupport
    ): Observer.Sync[ByteBuffer] = {
      new Observer.Sync[ByteBuffer] {
        private[this] var isClosed: Boolean = false
        override def onNext(elem: ByteBuffer): Ack = {
          if (isClosed) Ack.Stop
          else {
            try {
              while (elem.hasRemaining) out.write(elem.get())
              out.flush()
              Ack.Continue
            } catch {
              case _: IOException =>
                logger.error("OutputStream closed!")
                isClosed = true
                Ack.Stop
            }
          }
        }
        override def onError(ex: Throwable): Unit = ()
        override def onComplete(): Unit = out.close()
      }
    }
  }

} 

package scala.meta.jsonrpc

import cats.syntax.either._
import io.circe.Decoder
import io.circe.Encoder
import io.circe.syntax._
import java.io.OutputStream
import java.nio.ByteBuffer
import monix.eval.Callback
import monix.eval.Task
import monix.execution.Ack
import monix.execution.Cancelable
import monix.execution.atomic.Atomic
import monix.execution.atomic.AtomicInt
import monix.reactive.Observer
import scala.collection.concurrent.TrieMap
import scala.concurrent.Future
import scala.concurrent.duration.Duration
import MonixEnrichments._
import scribe.LoggerSupport

class LanguageClient(out: Observer[ByteBuffer], logger: LoggerSupport)
    extends JsonRpcClient {
  def this(out: OutputStream, logger: LoggerSupport) =
    this(Observer.fromOutputStream(out, logger), logger)
  private val writer = new MessageWriter(out, logger)
  private val counter: AtomicInt = Atomic(1)
  private val activeServerRequests =
    TrieMap.empty[RequestId, Callback[Response]]
  def notify[A: Encoder](method: String, notification: A): Future[Ack] =
    writer.write(Notification(method, Some(notification.asJson)))
  def serverRespond(response: Response): Future[Ack] = response match {
    case Response.Empty => Ack.Continue
    case x: Response.Success => writer.write(x)
    case x: Response.Error =>
      logger.error(s"Response error: $x")
      writer.write(x)
  }
  def clientRespond(response: Response): Unit =
    for {
      id <- response match {
        case Response.Empty => None
        case Response.Success(_, requestId) => Some(requestId)
        case Response.Error(_, requestId) => Some(requestId)
      }
      callback <- activeServerRequests.get(id).orElse {
        logger.error(s"Response to unknown request: $response")
        None
      }
    } {
      activeServerRequests.remove(id)
      callback.onSuccess(response)
    }

  def request[A: Encoder, B: Decoder](
      method: String,
      request: A
  ): Task[Either[Response.Error, B]] = {
    val nextId = RequestId(counter.incrementAndGet())
    val response = Task.create[Response] { (out, cb) =>
      val scheduled = out.scheduleOnce(Duration(0, "s")) {
        val json = Request(method, Some(request.asJson), nextId)
        activeServerRequests.put(nextId, cb)
        writer.write(json)
      }
      Cancelable { () =>
        scheduled.cancel()
        this.notify("$/cancelRequest", CancelParams(nextId.value))
      }
    }
    response.map {
      case Response.Empty =>
        Left(
          Response.invalidParams(
            s"Got empty response for request $request",
            nextId
          )
        )
      case err: Response.Error =>
        Left(err)
      case Response.Success(result, _) =>
        result.as[B].leftMap { err =>
          Response.invalidParams(err.toString, nextId)
        }
    }
  }
}

object LanguageClient {
  def fromOutputStream(out: OutputStream, logger: LoggerSupport) =
    new LanguageClient(Observer.fromOutputStream(out, logger), logger)
} 

import java.io.OutputStream
import java.nio.file.Files

import cats.effect.{Blocker, Concurrent, ContextShift, Resource, Sync}
import fs2.{Chunk, Hotswap, Pipe, Pull, RaiseThrowable, Stream}
import cats.implicits._

package object blobstore {
  protected[blobstore] def _writeAllToOutputStream1[F[_]](in: Stream[F, Byte], out: OutputStream, blocker: Blocker)(
    implicit F: Sync[F],
    CS: ContextShift[F]
  ): Pull[F, Nothing, Unit] = {
    in.pull.uncons.flatMap {
      case None => Pull.done
      case Some((hd, tl)) =>
        Pull.eval[F, Unit](blocker.delay(out.write(hd.toArray))) >> _writeAllToOutputStream1(tl, out, blocker)
    }
  }

  protected[blobstore] def bufferToDisk[F[_]](
    chunkSize: Int,
    blocker: Blocker
  )(implicit F: Sync[F], CS: ContextShift[F]): Pipe[F, Byte, (Long, Stream[F, Byte])] = { in =>
    Stream.bracket(F.delay(Files.createTempFile("bufferToDisk", ".bin")))(p => F.delay(p.toFile.delete).void).flatMap {
      p =>
        in.through(fs2.io.file.writeAll(p, blocker)).drain ++
          Stream.emit((p.toFile.length, fs2.io.file.readAll(p, blocker, chunkSize)))
    }
  }

  private[blobstore] def putRotateBase[F[_]: Concurrent, T](
    limit: Long,
    openNewFile: Resource[F, T]
  )(consume: T => Chunk[Byte] => F[Unit]): Pipe[F, Byte, Unit] = { in =>
    Stream
      .resource(Hotswap(openNewFile))
      .flatMap {
        case (hotswap, newFile) =>
          goRotate(limit, 0L, in, newFile, hotswap, openNewFile)(
            consume = consumer => bytes => Pull.eval(consume(consumer)(bytes)).as(consumer),
            extract = Stream.emit
          ).stream
      }
  }

  private[blobstore] def goRotate[F[_]: RaiseThrowable, A, B](
    limit: Long,
    acc: Long,
    s: Stream[F, Byte],
    consumer: B,
    hotswap: Hotswap[F, A],
    resource: Resource[F, A]
  )(
    consume: B => Chunk[Byte] => Pull[F, Unit, B],
    extract: A => Stream[F, B]
  ): Pull[F, Unit, Unit] = {
    val toWrite = (limit - acc).min(Int.MaxValue.toLong).toInt
    s.pull.unconsLimit(toWrite).flatMap {
      case Some((hd, tl)) =>
        val newAcc = acc + hd.size
        consume(consumer)(hd).flatMap { consumer =>
          if (newAcc >= limit) {
            Pull
              .eval(hotswap.swap(resource))
              .flatMap(a => extract(a).pull.headOrError)
              .flatMap(nc => goRotate(limit, 0L, tl, nc, hotswap, resource)(consume, extract))
          } else {
            goRotate(limit, newAcc, tl, consumer, hotswap, resource)(consume, extract)
          }
        }
      case None => Pull.done
    }
  }

} 

package org.apache.spark.sql.execution.datasources.oap.index.impl

import java.io.OutputStream

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path

import org.apache.spark.sql.execution.datasources.oap.index.IndexFileWriter

private[index] case class IndexFileWriterImpl(
    configuration: Configuration,
    indexPath: Path) extends IndexFileWriter {

  protected override val os: OutputStream =
    indexPath.getFileSystem(configuration).create(indexPath, true)

  // Give RecordWriter a chance which file it's writing to.
  override def getName: String = indexPath.toString

  override def tempRowIdWriter: IndexFileWriter = {
    val tempFileName = new Path(indexPath.getParent, indexPath.getName + ".id")
    IndexFileWriterImpl(configuration, tempFileName)
  }

  override def writeRowId(tempWriter: IndexFileWriter): Unit = {
    val path = new Path(tempWriter.getName)
    val is = path.getFileSystem(configuration).open(path)
    val length = path.getFileSystem(configuration).getFileStatus(path).getLen
    val bufSize = configuration.getInt("io.file.buffer.size", 4096)
    val bytes = new Array[Byte](bufSize)
    var remaining = length
    while (remaining > 0) {
      val readSize = math.min(bufSize, remaining).toInt
      is.readFully(bytes, 0, readSize)
      os.write(bytes, 0, readSize)
      remaining -= readSize
    }
    is.close()
    path.getFileSystem(configuration).delete(path, false)
  }
} 

package com.decodified.scalassh

import annotation.tailrec
import java.io.{ ByteArrayOutputStream, OutputStream, InputStream }

final class StreamCopier(bufferSize: Int = 4096) {
  private val buffer = new Array[Byte](bufferSize)

  @tailrec
  def copy(in: InputStream, out: OutputStream) {
    val bytes = in.read(buffer)
    if (bytes > 0) {
      out.write(buffer, 0, bytes)
      copy(in, out)
    } else {
      in.close()
      out.close()
    }
  }

  def emptyToString(inputStream: InputStream, charset: String = "UTF8") = {
    new String(emptyToByteArray(inputStream), charset)
  }

  def emptyToByteArray(inputStream: InputStream) = {
    val output = new ByteArrayOutputStream()
    copy(inputStream, output)
    output.toByteArray
  }
} 

package org.apache.gearpump.jarstore.dfs

import java.io.{InputStream, OutputStream}
import org.apache.gearpump.util.Constants
import org.apache.gearpump.jarstore.JarStore
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import com.typesafe.config.Config
import org.apache.hadoop.fs.permission.{FsAction, FsPermission}


  override def getFile(fileName: String): InputStream = {
    val filePath = new Path(rootPath, fileName)
    val fs = filePath.getFileSystem(new Configuration())
    fs.open(filePath)
  }

  private def createDirIfNotExists(path: Path): Unit = {
    val fs = path.getFileSystem(new Configuration())
    if (!fs.exists(path)) {
      fs.mkdirs(path, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))
    }
  }
} 

package org.apache.gearpump.experiments.yarn.glue

import java.io.{InputStream, OutputStream}
import java.net.ConnectException

import org.apache.gearpump.util.LogUtil
import org.apache.hadoop.fs.Path

import scala.util.{Failure, Success, Try}

class FileSystem(yarnConfig: YarnConfig) {

  private val conf = yarnConfig.conf
  private val fs = org.apache.hadoop.fs.FileSystem.get(conf)

  private def LOG = LogUtil.getLogger(getClass)

  def open(file: String): InputStream = exceptionHandler {
    val path = new Path(file)
    fs.open(path)
  }

  def create(file: String): OutputStream = exceptionHandler {
    val path = new Path(file)
    fs.create(path)
  }

  def exists(file: String): Boolean = exceptionHandler {
    val path = new Path(file)
    fs.exists(path)
  }

  def name: String = {
    fs.getUri.toString
  }

  def getHomeDirectory: String = {
    fs.getHomeDirectory.toString
  }

  private def exceptionHandler[T](call: => T): T = {
    val callTry = Try(call)
    callTry match {
      case Success(v) => v
      case Failure(ex) =>
        if (ex.isInstanceOf[ConnectException]) {
          LOG.error("Please check whether we connect to the right HDFS file system, " +
            "current file system is $name." + "\n. Please copy all configs under " +
            "$HADOOP_HOME/etc/hadoop into conf/yarnconf directory of Gearpump package, " +
            "so that we can use the right File system.", ex)
        }
        throw ex
    }
  }
} 

package org.apache.toree.kernel.protocol.v5.stream

import java.io.OutputStream
import java.nio.charset.Charset

import org.apache.toree.kernel.protocol.v5.content.StreamContent
import org.apache.toree.kernel.protocol.v5.{SystemActorType, MessageType, KMBuilder}
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.utils.{LogLike, ScheduledTaskManager}
import scala.collection.mutable.ListBuffer
import KernelOutputStream._

object KernelOutputStream {
  val DefaultStreamType = "stdout"
  val DefaultSendEmptyOutput = false
}


  override def write(b: Int): Unit = internalBytes.synchronized {
    // Begin periodic flushing if this is a new set of bytes
    enableAutoFlush()

    internalBytes += b.toByte
  }
} 

package org.apache.toree.kernel.protocol.v5.relay

import java.io.OutputStream
import akka.actor.Actor
import akka.pattern._
import akka.util.Timeout
import org.apache.toree.interpreter.{ExecuteAborted, ExecuteError, ExecuteFailure, ExecuteOutput}
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.content._
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5.magic.MagicParser
import org.apache.toree.plugins.PluginManager
import org.apache.toree.utils.LogLike
import scala.concurrent.Future
import scala.concurrent.duration._
import org.apache.toree.plugins.NewOutputStream

case class ExecuteRequestRelay(
  actorLoader: ActorLoader,
  pluginManager: PluginManager,
  magicParser: MagicParser
)
  extends Actor with LogLike
{
  import context._
  implicit val timeout = Timeout(21474835.seconds)

  
  private def packageFutureResponse(
    future: Future[Either[ExecuteOutput, ExecuteFailure]]
  ): Future[(ExecuteReply, ExecuteResult)] = future.map { value =>
    if (value.isLeft) {
      val data = value.left.get
      (
        ExecuteReplyOk(1, Some(Payloads()), Some(UserExpressions())),
        ExecuteResult(1, data, Metadata())
      )
    } else {
      failureMatch(value.right.get)
    }
  }

  override def receive: Receive = {
    case (executeRequest: ExecuteRequest, parentMessage: KernelMessage,
      outputStream: OutputStream) =>
      val interpreterActor = actorLoader.load(SystemActorType.Interpreter)

      // Store our old sender so we don't lose it in the callback
      // NOTE: Should point back to our KernelMessageRelay
      val oldSender = sender()

      // Sets the outputStream for this particular ExecuteRequest
      import org.apache.toree.plugins.Implicits._
      pluginManager.fireEventFirstResult(
        NewOutputStream,
        "outputStream" -> outputStream
      )

      // Parse the code for magics before sending it to the interpreter and
      // pipe the response to sender
      (magicParser.parse(executeRequest.code) match {
        case Left(code) =>
          val parsedRequest =
            (executeRequest.copy(code = code), parentMessage, outputStream)
          val interpreterFuture = (interpreterActor ? parsedRequest)
            .mapTo[Either[ExecuteOutput, ExecuteFailure]]
          packageFutureResponse(interpreterFuture)

        case Right(error) =>
          val failure = ExecuteError("Error parsing magics!", error, Nil)
          Future { failureMatch(failure) }
      }) pipeTo oldSender
  }
} 

package org.apache.toree.kernel.api

import java.io.{InputStream, OutputStream}

import org.apache.toree.kernel.protocol.v5
import org.apache.toree.kernel.protocol.v5.{KMBuilder, KernelMessage}
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5.stream.{KernelOutputStream, KernelInputStream}
import com.typesafe.config.Config


  override def newKernelOutputStream(
    streamType: String = KernelOutputStream.DefaultStreamType,
    sendEmptyOutput: Boolean = config.getBoolean("send_empty_output")
  ): OutputStream = {
    new v5.stream.KernelOutputStream(
      actorLoader,
      kmBuilder,
      org.apache.toree.global.ScheduledTaskManager.instance,
      streamType = streamType,
      sendEmptyOutput = sendEmptyOutput
    )
  }
} 

package org.apache.toree.magic.builtin

import java.io.OutputStream
import java.net.URL

import org.apache.toree.interpreter.Interpreter
import org.apache.toree.magic.dependencies.{IncludeOutputStream, IncludeInterpreter}
import org.apache.toree.magic.{CellMagic, LineMagic}
import org.apache.spark.SparkContext
import org.scalatest.{Matchers, FunSpec}
import org.scalatest.mock.MockitoSugar

import org.mockito.Mockito._
import org.mockito.Matchers._

class TestLSMagic(sc: SparkContext, intp: Interpreter, os: OutputStream)
  extends LSMagic
  with IncludeInterpreter
  with IncludeOutputStream
  {
    override val interpreter: Interpreter = intp
    override val outputStream: OutputStream = os
  }

class LSMagicSpec extends FunSpec with Matchers with MockitoSugar {
  describe("LSMagic") {

    describe("#execute") {
      it("should call println with a magics message") {
        val lsm = spy(new TestLSMagic(
          mock[SparkContext], mock[Interpreter], mock[OutputStream])
        )
        val classList = new BuiltinLoader().loadClasses()
        lsm.execute("")
        verify(lsm).magicNames("%", classOf[LineMagic], classList)
        verify(lsm).magicNames("%%", classOf[CellMagic], classList)
      }
    }

    describe("#magicNames") {
      it("should filter classnames by interface") {
        val prefix = "%"
        val interface = classOf[LineMagic]
        val classes : List[Class[_]] = List(classOf[LSMagic], classOf[Integer])
        val lsm = new TestLSMagic(
          mock[SparkContext], mock[Interpreter], mock[OutputStream])
        lsm.magicNames(prefix, interface, classes).length should be(1)
      }
      it("should prepend prefix to each name"){
        val prefix = "%"
        val className = classOf[LSMagic].getSimpleName
        val interface = classOf[LineMagic]
        val expected = s"${prefix}${className}"
        val classes : List[Class[_]] = List(classOf[LSMagic], classOf[Integer])
        val lsm = new TestLSMagic(
          mock[SparkContext], mock[Interpreter], mock[OutputStream])
        lsm.magicNames(prefix, interface, classes) should be(List(expected))
      }
    }

  }

} 

package org.apache.toree.global

import java.io.{InputStream, OutputStream, PrintStream}


  def withStreams[T](thunk: => T): T = {
    init(_inputStream, _outputStream, _errorStream)

    val returnValue = Console.withIn(_inputStream) {
      Console.withOut(_outputStream) {
        Console.withErr(_errorStream) {
          thunk
        }
      }
    }

    reset()

    returnValue
  }
} 

package org.apache.toree.utils

import joptsimple.{OptionSpec, OptionParser}
import scala.collection.JavaConverters._
import scala.language.implicitConversions
import java.io.{PrintStream, OutputStream}

trait ArgumentParsingSupport {
  protected lazy val parser = new OptionParser()
  private var options: joptsimple.OptionSet = _
  parser.allowsUnrecognizedOptions()

  
  def parseArgs(args: String, delimiter: String = " ") = {
    options = parser.parse(args.split(delimiter): _*)

    options.nonOptionArguments().asScala.map(_.toString)
  }

  def printHelp(outputStream: OutputStream, usage: String) = {
    val printStream = new PrintStream(outputStream)

    printStream.println(s"Usage: $usage\n")
    parser.printHelpOn(outputStream)
  }

  implicit def has[T](spec: OptionSpec[T]): Boolean = {
    require(options != null, "Arguments not parsed yet!")
    options.has(spec)
  }

  implicit def get[T](spec: OptionSpec[T]): Option[T] = {
    require(options != null, "Arguments not parsed yet!")
    Some(options.valueOf(spec)).filter(_ != null)
  }

  // NOTE: Cannot be implicit as conflicts with get
  def getAll[T](spec: OptionSpec[T]): Option[List[T]] = {
    require(options != null, "Arguments not parsed yet!")
    Some(options.valuesOf(spec).asScala.toList).filter(_ != null)
  }
} 

package org.apache.toree.utils

import java.io.OutputStream

import org.scalatest.mock.MockitoSugar
import org.scalatest.{BeforeAndAfter, Matchers, FunSpec}
import org.mockito.Matchers._
import org.mockito.Mockito._

class MultiOutputStreamSpec
  extends FunSpec with Matchers with MockitoSugar with BeforeAndAfter {

  describe("MultiOutputStream") {
    val listOfMockOutputStreams = List(mock[OutputStream], mock[OutputStream])
    val multiOutputStream = MultiOutputStream(listOfMockOutputStreams)

    describe("#close") {
      it("should call #close on all internal output streams") {
        multiOutputStream.close()

        listOfMockOutputStreams.foreach(mockOutputStream => verify(mockOutputStream).close())
      }
    }

    describe("#flush") {
      it("should call #flush on all internal output streams") {
        multiOutputStream.flush()

        listOfMockOutputStreams.foreach(mockOutputStream => verify(mockOutputStream).flush())
      }
    }

    describe("#write(int)") {
      it("should call #write(int) on all internal output streams") {
        multiOutputStream.write(anyInt())

        listOfMockOutputStreams.foreach(
          mockOutputStream => verify(mockOutputStream).write(anyInt()))
      }
    }
    describe("#write(byte[])") {
      it("should call #write(byte[]) on all internal output streams") {
        multiOutputStream.write(any[Array[Byte]])

        listOfMockOutputStreams.foreach(
          mockOutputStream => verify(mockOutputStream).write(any[Array[Byte]]))
      }
    }

    describe("#write(byte[], int, int)") {
      it("should call #write(byte[], int, int) on all internal output streams") {
        multiOutputStream.write(any[Array[Byte]], anyInt(), anyInt())

        listOfMockOutputStreams.foreach(
          mockOutputStream =>
            verify(mockOutputStream).write(any[Array[Byte]], anyInt(), anyInt()))
      }
    }
  }
} 

package org.apache.toree.utils

import java.io.OutputStream

import org.scalatest.mock.MockitoSugar
import org.mockito.Mockito._
import org.mockito.Matchers._
import org.scalatest.{Matchers, FunSpec}

class ConditionalOutputStreamSpec extends FunSpec with Matchers with MockitoSugar {
  describe("ConditionalOutputStream") {
    describe("#()") {
      it("should throw an exception if the output stream is null") {
        intercept[IllegalArgumentException] {
          new ConditionalOutputStream(null, true)
        }
      }
    }

    describe("#write") {
      it("should call the underlying write if the condition is true") {
        val mockOutputStream = mock[OutputStream]
        val conditionalOutputStream =
          new ConditionalOutputStream(mockOutputStream, true)

        val expected = 101
        conditionalOutputStream.write(expected)

        verify(mockOutputStream).write(expected)
      }

      it("should call the underlying write if the condition becomes true") {
        val mockOutputStream = mock[OutputStream]
        var condition = false

        val conditionalOutputStream =
          new ConditionalOutputStream(mockOutputStream, condition)

        condition = true

        val expected = 101
        conditionalOutputStream.write(expected)

        verify(mockOutputStream).write(expected)
      }

      it("should not call the underlying write if the condition is false") {
        val mockOutputStream = mock[OutputStream]
        val conditionalOutputStream =
          new ConditionalOutputStream(mockOutputStream, false)

        val expected = 101
        conditionalOutputStream.write(expected)

        verify(mockOutputStream, never()).write(any[Byte])
      }

      it("should not call the underlying write if the condition becomes false") {
        val mockOutputStream = mock[OutputStream]
        var condition = true

        val conditionalOutputStream =
          new ConditionalOutputStream(mockOutputStream, condition)

        condition = false

        val expected = 101
        conditionalOutputStream.write(expected)

        verify(mockOutputStream, never()).write(any[Byte])
      }
    }
  }
} 

package org.apache.spark.sql.execution.streaming


import java.io.{InputStream, OutputStream}
import java.nio.charset.StandardCharsets._

import scala.io.{Source => IOSource}

import org.apache.spark.sql.SparkSession


class OffsetSeqLog(sparkSession: SparkSession, path: String)
  extends HDFSMetadataLog[OffsetSeq](sparkSession, path) {

  override protected def deserialize(in: InputStream): OffsetSeq = {
    // called inside a try-finally where the underlying stream is closed in the caller
    def parseOffset(value: String): Offset = value match {
      case OffsetSeqLog.SERIALIZED_VOID_OFFSET => null
      case json => SerializedOffset(json)
    }
    val lines = IOSource.fromInputStream(in, UTF_8.name()).getLines()
    if (!lines.hasNext) {
      throw new IllegalStateException("Incomplete log file")
    }
    val version = lines.next()
    if (version != OffsetSeqLog.VERSION) {
      throw new IllegalStateException(s"Unknown log version: ${version}")
    }

    // read metadata
    val metadata = lines.next().trim match {
      case "" => None
      case md => Some(md)
    }
    OffsetSeq.fill(metadata, lines.map(parseOffset).toArray: _*)
  }

  override protected def serialize(offsetSeq: OffsetSeq, out: OutputStream): Unit = {
    // called inside a try-finally where the underlying stream is closed in the caller
    out.write(OffsetSeqLog.VERSION.getBytes(UTF_8))

    // write metadata
    out.write('\n')
    out.write(offsetSeq.metadata.map(_.json).getOrElse("").getBytes(UTF_8))

    // write offsets, one per line
    offsetSeq.offsets.map(_.map(_.json)).foreach { offset =>
      out.write('\n')
      offset match {
        case Some(json: String) => out.write(json.getBytes(UTF_8))
        case None => out.write(OffsetSeqLog.SERIALIZED_VOID_OFFSET.getBytes(UTF_8))
      }
    }
  }
}

object OffsetSeqLog {
  private val VERSION = "v1"
  private val SERIALIZED_VOID_OFFSET = "-"
} 

package org.apache.spark.streaming.util

import java.io.OutputStream
import java.util.concurrent.TimeUnit._

import scala.annotation.tailrec

import org.apache.spark.internal.Logging

private[streaming]
class RateLimitedOutputStream(out: OutputStream, desiredBytesPerSec: Int)
  extends OutputStream
  with Logging {

  require(desiredBytesPerSec > 0)

  private val SYNC_INTERVAL = NANOSECONDS.convert(10, SECONDS)
  private val CHUNK_SIZE = 8192
  private var lastSyncTime = System.nanoTime
  private var bytesWrittenSinceSync = 0L

  override def write(b: Int) {
    waitToWrite(1)
    out.write(b)
  }

  override def write(bytes: Array[Byte]) {
    write(bytes, 0, bytes.length)
  }

  @tailrec
  override final def write(bytes: Array[Byte], offset: Int, length: Int) {
    val writeSize = math.min(length - offset, CHUNK_SIZE)
    if (writeSize > 0) {
      waitToWrite(writeSize)
      out.write(bytes, offset, writeSize)
      write(bytes, offset + writeSize, length)
    }
  }

  override def flush() {
    out.flush()
  }

  override def close() {
    out.close()
  }

  @tailrec
  private def waitToWrite(numBytes: Int) {
    val now = System.nanoTime
    val elapsedNanosecs = math.max(now - lastSyncTime, 1)
    val rate = bytesWrittenSinceSync.toDouble * 1000000000 / elapsedNanosecs
    if (rate < desiredBytesPerSec) {
      // It's okay to write; just update some variables and return
      bytesWrittenSinceSync += numBytes
      if (now > lastSyncTime + SYNC_INTERVAL) {
        // Sync interval has passed; let's resync
        lastSyncTime = now
        bytesWrittenSinceSync = numBytes
      }
    } else {
      // Calculate how much time we should sleep to bring ourselves to the desired rate.
      val targetTimeInMillis = bytesWrittenSinceSync * 1000 / desiredBytesPerSec
      val elapsedTimeInMillis = elapsedNanosecs / 1000000
      val sleepTimeInMillis = targetTimeInMillis - elapsedTimeInMillis
      if (sleepTimeInMillis > 0) {
        logTrace("Natural rate is " + rate + " per second but desired rate is " +
          desiredBytesPerSec + ", sleeping for " + sleepTimeInMillis + " ms to compensate.")
        Thread.sleep(sleepTimeInMillis)
      }
      waitToWrite(numBytes)
    }
  }
} 

package org.apache.spark.security

import java.io.{InputStream, OutputStream}
import java.util.Properties
import javax.crypto.KeyGenerator
import javax.crypto.spec.{IvParameterSpec, SecretKeySpec}

import org.apache.commons.crypto.random._
import org.apache.commons.crypto.stream._

import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.internal.config._


  private[this] def createInitializationVector(properties: Properties): Array[Byte] = {
    val iv = new Array[Byte](IV_LENGTH_IN_BYTES)
    val initialIVStart = System.currentTimeMillis()
    CryptoRandomFactory.getCryptoRandom(properties).nextBytes(iv)
    val initialIVFinish = System.currentTimeMillis()
    val initialIVTime = initialIVFinish - initialIVStart
    if (initialIVTime > 2000) {
      logWarning(s"It costs ${initialIVTime} milliseconds to create the Initialization Vector " +
        s"used by CryptoStream")
    }
    iv
  }
} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.lang.annotation.Annotation
import java.lang.reflect.Type
import java.nio.charset.StandardCharsets
import java.text.SimpleDateFormat
import java.util.{Calendar, Locale, SimpleTimeZone}
import javax.ws.rs.Produces
import javax.ws.rs.core.{MediaType, MultivaluedMap}
import javax.ws.rs.ext.{MessageBodyWriter, Provider}

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}


@Provider
@Produces(Array(MediaType.APPLICATION_JSON))
private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{

  val mapper = new ObjectMapper() {
    override def writeValueAsString(t: Any): String = {
      super.writeValueAsString(t)
    }
  }
  mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
  mapper.enable(SerializationFeature.INDENT_OUTPUT)
  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)

  override def isWriteable(
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Boolean = {
      true
  }

  override def writeTo(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType,
      multivaluedMap: MultivaluedMap[String, AnyRef],
      outputStream: OutputStream): Unit = {
    t match {
      case ErrorWrapper(err) => outputStream.write(err.getBytes(StandardCharsets.UTF_8))
      case _ => mapper.writeValue(outputStream, t)
    }
  }

  override def getSize(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Long = {
    -1L
  }
}

private[spark] object JacksonMessageWriter {
  def makeISODateFormat: SimpleDateFormat = {
    val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'", Locale.US)
    val cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"))
    iso8601.setCalendar(cal)
    iso8601
  }
} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.util.zip.ZipOutputStream
import javax.ws.rs.{GET, Produces}
import javax.ws.rs.core.{MediaType, Response, StreamingOutput}

import scala.util.control.NonFatal

import org.apache.spark.SparkConf
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.internal.Logging

@Produces(Array(MediaType.APPLICATION_OCTET_STREAM))
private[v1] class EventLogDownloadResource(
    val uIRoot: UIRoot,
    val appId: String,
    val attemptId: Option[String]) extends Logging {
  val conf = SparkHadoopUtil.get.newConfiguration(new SparkConf)

  @GET
  def getEventLogs(): Response = {
    try {
      val fileName = {
        attemptId match {
          case Some(id) => s"eventLogs-$appId-$id.zip"
          case None => s"eventLogs-$appId.zip"
        }
      }

      val stream = new StreamingOutput {
        override def write(output: OutputStream): Unit = {
          val zipStream = new ZipOutputStream(output)
          try {
            uIRoot.writeEventLogs(appId, attemptId, zipStream)
          } finally {
            zipStream.close()
          }

        }
      }

      Response.ok(stream)
        .header("Content-Disposition", s"attachment; filename=$fileName")
        .header("Content-Type", MediaType.APPLICATION_OCTET_STREAM)
        .build()
    } catch {
      case NonFatal(e) =>
        Response.serverError()
          .entity(s"Event logs are not available for app: $appId.")
          .status(Response.Status.SERVICE_UNAVAILABLE)
          .build()
    }
  }
} 

package org.apache.spark.util.io

import java.io.OutputStream
import java.nio.ByteBuffer

import scala.collection.mutable.ArrayBuffer

import org.apache.spark.storage.StorageUtils


  private[this] var position = chunkSize
  private[this] var _size = 0
  private[this] var closed: Boolean = false

  def size: Long = _size

  override def close(): Unit = {
    if (!closed) {
      super.close()
      closed = true
    }
  }

  override def write(b: Int): Unit = {
    require(!closed, "cannot write to a closed ChunkedByteBufferOutputStream")
    allocateNewChunkIfNeeded()
    chunks(lastChunkIndex).put(b.toByte)
    position += 1
    _size += 1
  }

  override def write(bytes: Array[Byte], off: Int, len: Int): Unit = {
    require(!closed, "cannot write to a closed ChunkedByteBufferOutputStream")
    var written = 0
    while (written < len) {
      allocateNewChunkIfNeeded()
      val thisBatch = math.min(chunkSize - position, len - written)
      chunks(lastChunkIndex).put(bytes, written + off, thisBatch)
      written += thisBatch
      position += thisBatch
    }
    _size += len
  }

  @inline
  private def allocateNewChunkIfNeeded(): Unit = {
    if (position == chunkSize) {
      chunks += allocator(chunkSize)
      lastChunkIndex += 1
      position = 0
    }
  }

  def toChunkedByteBuffer: ChunkedByteBuffer = {
    require(closed, "cannot call toChunkedByteBuffer() unless close() has been called")
    require(!toChunkedByteBufferWasCalled, "toChunkedByteBuffer() can only be called once")
    toChunkedByteBufferWasCalled = true
    if (lastChunkIndex == -1) {
      new ChunkedByteBuffer(Array.empty[ByteBuffer])
    } else {
      // Copy the first n-1 chunks to the output, and then create an array that fits the last chunk.
      // An alternative would have been returning an array of ByteBuffers, with the last buffer
      // bounded to only the last chunk's position. However, given our use case in Spark (to put
      // the chunks in block manager), only limiting the view bound of the buffer would still
      // require the block manager to store the whole chunk.
      val ret = new Array[ByteBuffer](chunks.size)
      for (i <- 0 until chunks.size - 1) {
        ret(i) = chunks(i)
        ret(i).flip()
      }
      if (position == chunkSize) {
        ret(lastChunkIndex) = chunks(lastChunkIndex)
        ret(lastChunkIndex).flip()
      } else {
        ret(lastChunkIndex) = allocator(position)
        chunks(lastChunkIndex).flip()
        ret(lastChunkIndex).put(chunks(lastChunkIndex))
        ret(lastChunkIndex).flip()
        StorageUtils.dispose(chunks(lastChunkIndex))
      }
      new ChunkedByteBuffer(ret)
    }
  }
} 

package org.apache.spark.util.io

import java.io.OutputStream

import scala.collection.mutable.ArrayBuffer



  private var position = chunkSize

  override def write(b: Int): Unit = {
    allocateNewChunkIfNeeded()
    chunks(lastChunkIndex)(position) = b.toByte
    position += 1
  }

  override def write(bytes: Array[Byte], off: Int, len: Int): Unit = {
    var written = 0
    while (written < len) {
      allocateNewChunkIfNeeded()
      val thisBatch = math.min(chunkSize - position, len - written)
      System.arraycopy(bytes, written + off, chunks(lastChunkIndex), position, thisBatch)
      written += thisBatch
      position += thisBatch
    }
  }

  @inline
  private def allocateNewChunkIfNeeded(): Unit = {
    if (position == chunkSize) {
      chunks += new Array[Byte](chunkSize)
      lastChunkIndex += 1
      position = 0
    }
  }

  def toArrays: Array[Array[Byte]] = {
    if (lastChunkIndex == -1) {
      new Array[Array[Byte]](0)
    } else {
      // Copy the first n-1 chunks to the output, and then create an array that fits the last chunk.
      // An alternative would have been returning an array of ByteBuffers, with the last buffer
      // bounded to only the last chunk's position. However, given our use case in Spark (to put
      // the chunks in block manager), only limiting the view bound of the buffer would still
      // require the block manager to store the whole chunk.
      val ret = new Array[Array[Byte]](chunks.size)
      for (i <- 0 until chunks.size - 1) {
        ret(i) = chunks(i)
      }
      if (position == chunkSize) {
        ret(lastChunkIndex) = chunks(lastChunkIndex)
      } else {
        ret(lastChunkIndex) = new Array[Byte](position)
        System.arraycopy(chunks(lastChunkIndex), 0, ret(lastChunkIndex), 0, position)
      }
      ret
    }
  }
} 

package io.github.mandar2812.dynaml

import java.io.{InputStream, OutputStream, PrintStream}

import ammonite.ops.Path
import ammonite.runtime.Storage
import ammonite.sshd.{SshServer, SshServerConfig}
import ammonite.sshd.util.Environment
import ammonite.util.{Bind, Colors}


class DynaMLSSH(
  sshConfig: SshServerConfig,
  predef: String = "",
  defaultPredef: Boolean = true,
  wd: os.Path = os.pwd,
  replArgs: Seq[Bind[_]] = Nil,
  classLoader: ClassLoader = DynaMLSSH.getClass.getClassLoader) {
  private lazy val sshd = SshServer(
    sshConfig,
    shellServer = DynaMLSSH.runRepl(
      sshConfig.ammoniteHome,
      predef,
      defaultPredef,
      wd,
      replArgs,
      classLoader
    )
  )

  def port                    = sshd.getPort
  def start(): Unit           = sshd.start()
  def stop(): Unit            = sshd.stop()
  def stopImmediately(): Unit = sshd.stop(true)
}

object DynaMLSSH {
  // Actually runs a repl inside of session serving a remote user shell.
  private def runRepl(
    homePath: os.Path,
    predefCode: String,
    defaultPredef: Boolean,
    wd: os.Path,
    replArgs: Seq[Bind[_]],
    replServerClassLoader: ClassLoader
  )(in: InputStream,
    out: OutputStream
  ): Unit = {
    // since sshd server has it's own customised environment,
    // where things like System.out will output to the
    // server's console, we need to prepare individual environment
    // to serve this particular user's session

    Environment.withEnvironment(Environment(replServerClassLoader, in, out)) {
      try {
        DynaML(
          predefCode = predefCode,
          predefFile = None,
          defaultPredef = defaultPredef,
          storageBackend = new Storage.Folder(homePath),
          wd = wd,
          inputStream = in,
          outputStream = out,
          errorStream = out,
          verboseOutput = false,
          remoteLogging = false,
          colors = Colors.Default
        ).run(replArgs: _*)
      } catch {
        case any: Throwable =>
          val sshClientOutput = new PrintStream(out)
          sshClientOutput.println(
            "What a terrible failure, DynaML just blew up!"
          )
          any.printStackTrace(sshClientOutput)
      }
    }
  }
} 

package me.lyh.protobuf.generic

import java.io.{InputStream, ObjectInputStream, ObjectOutputStream, OutputStream}
import java.nio.ByteBuffer
import java.util.{ArrayList => JArrayList, LinkedHashMap => JLinkedHashMap, TreeMap => JTreeMap}

import com.google.protobuf.Descriptors.FieldDescriptor.Type
import com.google.protobuf.{CodedInputStream, WireFormat}

import scala.collection.JavaConverters._

object GenericReader {
  def of(schema: Schema): GenericReader = new GenericReader(schema)
}

class GenericReader(val schema: Schema) extends Serializable {
  def read(buf: Array[Byte]): GenericRecord =
    read(CodedInputStream.newInstance(buf), schema.root)

  def read(buf: ByteBuffer): GenericRecord =
    read(CodedInputStream.newInstance(buf), schema.root)

  def read(input: InputStream): GenericRecord =
    read(CodedInputStream.newInstance(input), schema.root)

  private def read(input: CodedInputStream, messageSchema: MessageSchema): GenericRecord = {
    val map = new JTreeMap[java.lang.Integer, Any]()
    while (!input.isAtEnd) {
      val tag = input.readTag()
      val id = WireFormat.getTagFieldNumber(tag)
      val field = messageSchema.fields(id)

      if (field.label == Label.REPEATED) {
        if (!map.containsKey(id)) {
          map.put(id, new JArrayList[Any]())
        }
        val list = map.get(id).asInstanceOf[java.util.ArrayList[Any]]
        if (field.packed) {
          val bytesIn = CodedInputStream.newInstance(input.readByteBuffer())
          while (!bytesIn.isAtEnd) {
            list.add(readValue(bytesIn, field))
          }
        } else {
          list.add(readValue(input, field))
        }
      } else {
        map.put(id, readValue(input, field))
      }
    }

    val result = new JLinkedHashMap[String, Any]()
    map.asScala.foreach(kv => result.put(messageSchema.fields(kv._1).name, kv._2))
    messageSchema.fields.valuesIterator.foreach { f =>
      if (f.default.isDefined && !result.containsKey(f.name)) {
        result.put(f.name, f.default.get)
      }
    }
    result
  }

  private def readValue(in: CodedInputStream, field: Field): Any = field.`type` match {
    case Type.FLOAT    => in.readFloat()
    case Type.DOUBLE   => in.readDouble()
    case Type.FIXED32  => in.readFixed32()
    case Type.FIXED64  => in.readFixed64()
    case Type.INT32    => in.readInt32()
    case Type.INT64    => in.readInt64()
    case Type.UINT32   => in.readUInt32()
    case Type.UINT64   => in.readUInt64()
    case Type.SFIXED32 => in.readSFixed32()
    case Type.SFIXED64 => in.readSFixed64()
    case Type.SINT32   => in.readSInt32()
    case Type.SINT64   => in.readSInt64()
    case Type.BOOL     => in.readBool()
    case Type.STRING   => in.readString()
    case Type.BYTES    => Base64.encode(in.readByteArray())
    case Type.ENUM     => schema.enums(field.schema.get).values(in.readEnum())
    case Type.MESSAGE =>
      val nestedIn = CodedInputStream.newInstance(in.readByteBuffer())
      read(nestedIn, schema.messages(field.schema.get))
    case Type.GROUP => throw new IllegalArgumentException("Unsupported type: GROUP")
  }

  private def readObject(in: ObjectInputStream): Unit = {
    val schema = Schema.fromJson(in.readUTF())

    val schemaField = getClass.getDeclaredField("schema")
    schemaField.setAccessible(true)
    schemaField.set(this, schema)
  }

  private def writeObject(out: ObjectOutputStream): Unit =
    out.writeUTF(schema.toJson)
} 

package dev.profunktor.fs2rabbit

import java.io.{DataInputStream, DataOutputStream, InputStream, OutputStream}
import java.time.Instant

import com.rabbitmq.client.impl.{ValueReader, ValueWriter}
import dev.profunktor.fs2rabbit.model.AmqpFieldValue._
import dev.profunktor.fs2rabbit.model.{AmqpFieldValue, ShortString}
import org.scalatest.flatspec.AnyFlatSpecLike
import org.scalatest.Assertion
import org.scalatest.matchers.should.Matchers

class AmqpFieldValueSpec extends AnyFlatSpecLike with Matchers with AmqpPropertiesArbitraries {

  it should "convert from and to Java primitive header values" in {
    val intVal    = IntVal(1)
    val longVal   = LongVal(2L)
    val stringVal = StringVal("hey")
    val arrayVal  = ArrayVal(Vector(IntVal(3), IntVal(2), IntVal(1)))

    AmqpFieldValue.unsafeFrom(intVal.toValueWriterCompatibleJava) should be(intVal)
    AmqpFieldValue.unsafeFrom(longVal.toValueWriterCompatibleJava) should be(longVal)
    AmqpFieldValue.unsafeFrom(stringVal.toValueWriterCompatibleJava) should be(stringVal)
    AmqpFieldValue.unsafeFrom("fs2") should be(StringVal("fs2"))
    AmqpFieldValue.unsafeFrom(arrayVal.toValueWriterCompatibleJava) should be(arrayVal)
  }
  it should "preserve the same value after a round-trip through impure and from" in {
    forAll { amqpHeaderVal: AmqpFieldValue =>
      AmqpFieldValue.unsafeFrom(amqpHeaderVal.toValueWriterCompatibleJava) == amqpHeaderVal
    }
  }

  it should "preserve the same values after a round-trip through the Java ValueReader and ValueWriter" in {
    forAll(assertThatValueIsPreservedThroughJavaWriteAndRead _)
  }

  it should "preserve a specific StringVal that previously failed after a round-trip through the Java ValueReader and ValueWriter" in {
    assertThatValueIsPreservedThroughJavaWriteAndRead(StringVal("kyvmqzlbjivLqQFukljghxdowkcmjklgSeybdy"))
  }

  it should "preserve a specific DateVal created from an Instant that has millisecond accuracy after a round-trip through the Java ValueReader and ValueWriter" in {
    val instant   = Instant.parse("4000-11-03T20:17:29.57Z")
    val myDateVal = TimestampVal.from(instant)
    assertThatValueIsPreservedThroughJavaWriteAndRead(myDateVal)
  }

  "DecimalVal" should "reject a BigDecimal of an unscaled value with 33 bits..." in {
    DecimalVal.from(BigDecimal(Int.MaxValue) + BigDecimal(1)) should be(None)
  }
  it should "reject a BigDecimal with a scale over octet size" in {
    DecimalVal.from(new java.math.BigDecimal(java.math.BigInteger.valueOf(12345L), 1000)) should be(None)
  }

  // We need to wrap things in a dummy table because the method that would be
  // great to test with ValueReader, readFieldValue, is private, and so we
  // have to call the next best thing, readTable.
  private def wrapInDummyTable(value: AmqpFieldValue): TableVal =
    TableVal(Map(ShortString.unsafeFrom("dummyKey") -> value))

  private def createWriterFromQueue(outputResults: collection.mutable.Queue[Byte]): ValueWriter =
    new ValueWriter({
      new DataOutputStream({
        new OutputStream {
          override def write(b: Int): Unit =
            outputResults.enqueue(b.toByte)
        }
      })
    })

  private def createReaderFromQueue(input: collection.mutable.Queue[Byte]): ValueReader = {
    val inputStream = new InputStream {
      override def read(): Int =
        try {
          val result = input.dequeue()
          // A signed -> unsigned conversion because bytes by default are
          // converted into signed ints, which is bad when the API of read
          // states that negative numbers indicate EOF...
          0Xff & result.toInt
        } catch {
          case _: NoSuchElementException => -1
        }

      override def available(): Int = {
        val result = input.size
        result
      }
    }
    new ValueReader(new DataInputStream(inputStream))
  }

  private def assertThatValueIsPreservedThroughJavaWriteAndRead(amqpHeaderVal: AmqpFieldValue): Assertion = {
    val outputResultsAsTable = collection.mutable.Queue.empty[Byte]
    val tableWriter          = createWriterFromQueue(outputResultsAsTable)
    tableWriter.writeTable(wrapInDummyTable(amqpHeaderVal).toValueWriterCompatibleJava)

    val reader    = createReaderFromQueue(outputResultsAsTable)
    val readValue = reader.readTable()
    AmqpFieldValue.unsafeFrom(readValue) should be(wrapInDummyTable(amqpHeaderVal))
  }
} 

package io.bullet.borer.output

import java.io.OutputStream

import io.bullet.borer.{ByteAccess, Output}
import io.bullet.borer.Output.ToValueProvider

trait ToOutputStreamOutput {

  implicit object ToOutputStreamProvider extends ToValueProvider[OutputStream] {
    type Out = ToOutputStream

    def apply(outputStream: OutputStream, bufferSize: Int, allowBufferCaching: Boolean) =
      new ToOutputStream(outputStream, bufferSize)
  }

  abstract class ToOutputStreamBase(protected val outputStream: OutputStream, bufferSize: Int) extends Output {
    type Self <: ToOutputStreamBase { type Self = ToOutputStreamBase.this.Self }

    final def writeByte(byte: Byte): Self = {
      outputStream.write(byte.toInt)
      this.asInstanceOf[Self]
    }

    final def writeBytes(a: Byte, b: Byte): Self          = writeByte(a).writeByte(b)
    final def writeBytes(a: Byte, b: Byte, c: Byte): Self = writeByte(a).writeByte(b).writeByte(c)

    final def writeBytes(a: Byte, b: Byte, c: Byte, d: Byte): Self =
      writeByte(a).writeByte(b).writeByte(c).writeByte(d)

    final def writeBytes[Bytes](bytes: Bytes)(implicit byteAccess: ByteAccess[Bytes]): Self = {
      outputStream.write(byteAccess.toByteArray(bytes))
      this.asInstanceOf[Self]
    }
  }

  
  final class ToOutputStream(outputStream: OutputStream, bufferSize: Int)
      extends ToOutputStreamBase(outputStream, bufferSize) {
    type Self   = ToOutputStream
    type Result = OutputStream

    def result() = outputStream
  }
} 

package com.github.gtache.lsp.client.connection

import java.io.{File, IOException, InputStream, OutputStream}
import java.util.Objects

import com.intellij.openapi.diagnostic.Logger
import org.jetbrains.annotations.Nullable


class ProcessStreamConnectionProvider(private var commands: Seq[String], private var workingDir: String) extends StreamConnectionProvider {
  private val LOG: Logger = Logger.getInstance(classOf[ProcessStreamConnectionProvider])
  @Nullable private var process: Process = _

  @throws[IOException]
  override def start(): Unit = {
    if (this.workingDir == null || this.commands == null || this.commands.isEmpty || this.commands.contains(null)) throw new IOException("Unable to start language server: " + this.toString) //$NON-NLS-1$
    val builder = createProcessBuilder
    LOG.info("Starting server process with commands " + commands + " and workingDir " + workingDir)
    this.process = builder.start

    if (!process.isAlive) throw new IOException("Unable to start language server: " + this.toString) else LOG.info("Server process started " + process)
  }

  protected def createProcessBuilder: ProcessBuilder = {
    import scala.collection.JavaConverters._
    val builder = new ProcessBuilder(getCommands.map(s => s.replace("\'", "")).asJava)
    builder.directory(new File(getWorkingDirectory))
    builder
  }

  protected def getCommands: Seq[String] = commands

  def setCommands(commands: Seq[String]): Unit = {
    this.commands = commands
  }

  protected def getWorkingDirectory: String = workingDir

  def setWorkingDirectory(workingDir: String): Unit = {
    this.workingDir = workingDir
  }

  @Nullable override def getInputStream: InputStream = {
    if (process == null) null
    else process.getInputStream
  }

  @Nullable override def getOutputStream: OutputStream = {
    if (process == null) null
    else process.getOutputStream
  }

  @Nullable override def getErrorStream: InputStream = {
    if (process == null) null
    else process.getErrorStream
  }

  override def stop(): Unit = {
    if (process != null) process.destroy()
  }

  override def equals(obj: Any): Boolean = {
    obj match {
      case other: ProcessStreamConnectionProvider =>
        getCommands.size == other.getCommands.size && this.getCommands.toSet == other.getCommands.toSet && this.getWorkingDirectory == other.getWorkingDirectory
      case _ => false
    }

  }

  override def hashCode: Int = {
    Objects.hashCode(this.getCommands) ^ Objects.hashCode(this.getWorkingDirectory)
  }
} 

package com.github.gtache.lsp.client.connection

import java.io.{IOException, InputStream, OutputStream}
import java.net.{ServerSocket, Socket}
import java.util.Objects

import com.intellij.openapi.diagnostic.Logger


class ProcessOverSocketStreamConnectionProvider(commands: Seq[String], workingDir: String, port: Int = 0) extends ProcessStreamConnectionProvider(commands, workingDir) {

  import ProcessOverSocketStreamConnectionProvider._

  private var socket: Socket = _
  private var inputStream: InputStream = _
  private var outputStream: OutputStream = _

  @throws[IOException]
  override def start(): Unit = {
    val serverSocket = new ServerSocket(port)
    val socketThread = new Thread(() => {
      try
        socket = serverSocket.accept
      catch {
        case e: IOException =>
          LOG.error(e)
      } finally try
        serverSocket.close()
      catch {
        case e: IOException =>
          LOG.error(e)
      }
    })
    socketThread.start()
    super.start()
    try {
      socketThread.join(5000)
    }
    catch {
      case e: InterruptedException =>
        LOG.error(e)
    }
    if (socket == null) throw new IOException("Unable to make socket connection: " + toString) //$NON-NLS-1$
    inputStream = socket.getInputStream
    outputStream = socket.getOutputStream
  }

  override def getInputStream: InputStream = inputStream

  override def getOutputStream: OutputStream = outputStream

  override def getErrorStream: InputStream = inputStream

  override def stop(): Unit = {
    super.stop()
    if (socket != null) try
      socket.close()
    catch {
      case e: IOException =>
        LOG.error(e)
    }
  }

  override def hashCode: Int = {
    val result = super.hashCode
    result ^ Objects.hashCode(this.port)
  }
}

object ProcessOverSocketStreamConnectionProvider {
  private val LOG = Logger.getInstance(classOf[ProcessOverSocketStreamConnectionProvider])
} 

package org.codeoverflow.chatoverflow.requirement.service.rcon

import java.io.{DataInputStream, IOException, InputStream, OutputStream}
import java.net.{Socket, SocketException}
import java.nio.{ByteBuffer, ByteOrder}
import java.util.Random

import org.codeoverflow.chatoverflow.WithLogger
import org.codeoverflow.chatoverflow.connector.Connector

class RconConnector(override val sourceIdentifier: String) extends Connector(sourceIdentifier) with WithLogger {
  override protected var requiredCredentialKeys: List[String] = List("password", "address")
  override protected var optionalCredentialKeys: List[String] = List("port")

  private var socket: Socket = _
  private var outputStream: OutputStream = _
  private var inputStream: InputStream = _
  private var requestId: Int = 0

  def sendCommand(command: String): String = {
    logger debug s"Sending $command to RCON"
    requestId += 1
    if (write(2, command.getBytes("ASCII"))) {
      return read()
    }
    null
  }


  
  override def stop(): Boolean = {
    logger info s"Stopped RCON connector to ${credentials.get.getValue("address").get}!"
    socket.close()
    true
  }
} 

package yamrcraft.etlite.writers

import java.io.OutputStream

import org.apache.hadoop.fs.Path
import org.apache.hadoop.io.{BytesWritable, IntWritable, SequenceFile, Text}
import org.json4s.jackson.Serialization
import org.json4s.{DefaultFormats, Formats, ShortTypeHints}
import yamrcraft.etlite.utils.FileUtils


case class ErrorInfo(
  errorType: String,
  errorMsg: Option[String]
)


class ErrorEventWriter(folder: String, jobId: Long, partitionId: Int)
  extends ErrorEventsWriter {

  // incremental record id
  var recordId = 1

  val fs = FileUtils.getFS(folder)

  val seqPath = new Path(folder, s"errors_job${jobId}_part$partitionId.seq")
  if (fs.exists(seqPath)) {
    fs.delete(seqPath, false)
  }

  val metaPath = new Path(folder, s"errors_job${jobId}_part$partitionId.meta.seq")
  if (fs.exists(metaPath)) {
    fs.delete(metaPath, false)
  }

  private var seqWriter: Option[SequenceFile.Writer] = None
  private var metaWriter: Option[SequenceFile.Writer] = None

  implicit val formats =
    new Formats {
      val dateFormat = DefaultFormats.lossless.dateFormat
      override val typeHints = ShortTypeHints(List(classOf[ErrorInfo]))
      override val typeHintFieldName = "type"
    }

  override def write(errorEvent: (Array[Byte], ErrorInfo)) = {

    if (seqWriter.isEmpty) {
      seqWriter = createSequenceFile(seqPath, classOf[IntWritable], classOf[BytesWritable])
      metaWriter = createSequenceFile(metaPath, classOf[IntWritable], classOf[Text])
    }

    val id = new IntWritable(recordId)
    seqWriter.get.append(id, new BytesWritable(errorEvent._1))
    metaWriter.get.append(id, new Text(Serialization.write(errorEvent._2)))

    recordId += 1
  }

  override def commit() = {
    seqWriter.foreach(p => p.close())
    metaWriter.foreach(p => p.close())
  }

  private def createSequenceFile(path: Path, keyClass: Class[_], valueClass: Class[_]) = {
    val optPath = SequenceFile.Writer.file(path)
    val optKey = SequenceFile.Writer.keyClass(keyClass)
    val optVal = SequenceFile.Writer.valueClass(valueClass)
    Some(SequenceFile.createWriter(fs.getConf, optPath, optKey, optVal))
  }
} 

package com.olegych.scastie.sbtscastie

import java.io.{OutputStream, PrintWriter}

import com.olegych.scastie.api._
import org.apache.logging.log4j.core.LogEvent
import org.apache.logging.log4j.core.appender.AbstractAppender
import org.apache.logging.log4j.core.layout.PatternLayout
import org.apache.logging.log4j.message.ObjectMessage
import play.api.libs.json.Json
import sbt.Keys._
import sbt._
import sbt.internal.LogManager.suppressedMessage
import sbt.internal.util.MainAppender.defaultScreen
import sbt.internal.util.{ObjectEvent, TraceEvent}

object RuntimeErrorLogger {
  private object NoOp {
    def apply(): NoOp = {
      def out(in: String): Unit = {
        println(
          Json.stringify(
            Json.toJson[ConsoleOutput](
              ConsoleOutput.SbtOutput(ProcessOutput(in.trim, ProcessOutputType.StdOut, None))
            )
          )
        )
      }

      new NoOp(new OutputStream {
        override def close(): Unit = ()
        override def flush(): Unit = ()
        override def write(b: Array[Byte]): Unit =
          out(new String(b))
        override def write(b: Array[Byte], off: Int, len: Int): Unit =
          out(new String(b, off, len))
        override def write(b: Int): Unit = ()
      })
    }
  }
  private class NoOp(os: OutputStream) extends PrintWriter(os)

  private val clientLogger = new AbstractAppender("sbt-scastie-appender", null, PatternLayout.createDefaultLayout(), true, Array()) {
    def append(event: LogEvent): Unit = {
      //daaamn
      val throwable = Option(event.getThrown).orElse {
        for {
          e <- Option(event.getMessage).collect {
            case e: ObjectMessage => e
          }
          e <- Option(e.getParameter).collect {
            case e: ObjectEvent[_] => e
          }
          e <- Option(e.message).collect {
            case e: TraceEvent => e
          }
          //since worksheet wraps the code in object we unwrap it to display clearer message
          e <- Option(e.message).collect {
            case e: ExceptionInInitializerError if e.getCause != null && e.getCause.getStackTrace.headOption.exists { e =>
                  e.getClassName == Instrumentation.instrumentedObject + "$" && e.getMethodName == "<clinit>"
                } =>
              e.getCause
            case e => e
          }
        } yield e
      }
      throwable.foreach { throwable =>
        val error = RuntimeErrorWrap(RuntimeError.fromThrowable(throwable))
        println(Json.stringify(Json.toJson(error)))
      }
    }
    start()
  }

  val settings: Seq[sbt.Def.Setting[_]] = Seq(
    extraLoggers := { (key: ScopedKey[_]) =>
      Seq(clientLogger)
    },
    showSuccess := false,
    logManager := sbt.internal.LogManager.withLoggers(
      (task, state) => defaultScreen(ConsoleOut.printWriterOut(NoOp()), suppressedMessage(task, state)),
      relay = _ => clientLogger
    )
  )
} 

package ch.epfl.scala.index
package data

import me.tongfei.progressbar.{ProgressBar => PB, ProgressBarStyle}
import java.io.{PrintStream, ByteArrayOutputStream, OutputStream}
import org.slf4j.Logger

object ProgressBar {
  def apply(title: String, count: Int, logger: Logger): ProgressBar = {
    new ProgressBar(
      new PB(title, count, 1000, System.out, ProgressBarStyle.UNICODE_BLOCK),
      logger,
      count
    )
  }
}

class ProgressBar(inner: PB, logger: Logger, count: Int) {
  var c = 0
  var printed = 0

  def start(): Unit = {
    inner.start()
  }

  def step(): Unit = {
    inner.step()
    c += 1
    print()
  }

  def stepBy(n: Int): Unit = {
    inner.stepBy(n)
    c += n
    print()
  }

  def stop(): Unit = {
    inner.stop()
  }

  private def print(): Unit = {
    val pp = ((c.toDouble / count) * 100).toInt

    if (printed < pp) {
      logger.debug(pp + "%")
      printed = pp
    }
  }
} 

package coursier.publish.sonatype.logger

import java.io.{OutputStream, OutputStreamWriter}

import coursier.cache.internal.Terminal.Ansi

final class InteractiveSonatypeLogger(out: OutputStreamWriter, verbosity: Int) extends SonatypeLogger {
  override def listingProfiles(attempt: Int, total: Int): Unit =
    if (verbosity >= 0) {
      val extra =
        if (attempt == 0) ""
        else s" (attempt $attempt / $total)"
      out.write("Listing Sonatype profiles..." + extra)
      out.flush()
    }
  override def listedProfiles(errorOpt: Option[Throwable]): Unit = {
    if (verbosity >= 0) {
      out.clearLine(2)
      out.write('\n')
      out.up(1)
      out.flush()
    }

    val msgOpt =
      if (errorOpt.isEmpty) {
        if (verbosity >= 1)
          Some("Listed Sonatype profiles")
        else
          None
      } else
        Some("Fail to list Sonatype profiles")

    for (msg <- msgOpt) {
      out.write(s"$msg\n")
      out.flush()
    }
  }
}

object InteractiveSonatypeLogger {
  def create(out: OutputStream, verbosity: Int): SonatypeLogger =
    new InteractiveSonatypeLogger(new OutputStreamWriter(out), verbosity)
} 

package coursier.publish.upload.logger

import java.io.{OutputStream, OutputStreamWriter, Writer}

import com.lightbend.emoji.ShortCodes.Defaults.defaultImplicit.emoji
import coursier.publish.fileset.FileSet
import coursier.publish.logging.ProgressLogger
import coursier.publish.upload.Upload

// FIXME Would have been better if dummy was passed by the Upload instance when calling the methods of UploadLogger
final class InteractiveUploadLogger(out: Writer, dummy: Boolean, isLocal: Boolean) extends UploadLogger {

  private val underlying = new ProgressLogger[Object](
    if (isLocal) {
      if (dummy)
        "Would have written"
      else
        "Wrote"
    } else {
      if (dummy)
        "Would have uploaded"
      else
        "Uploaded"
    },
    "files",
    out,
    doneEmoji = emoji("truck").map(_.toString())
  )

  override def uploadingSet(id: Object, fileSet: FileSet): Unit =
    underlying.processingSet(id, Some(fileSet.elements.length))
  override def uploadedSet(id: Object, fileSet: FileSet): Unit =
    underlying.processedSet(id)

  override def uploading(url: String, idOpt: Option[Object], totalOpt: Option[Long]): Unit =
    for (id <- idOpt)
      underlying.processing(url, id)

  override def progress(url: String, idOpt: Option[Object], uploaded: Long, total: Long): Unit =
    for (id <- idOpt)
      underlying.progress(url, id, uploaded, total)
  override def uploaded(url: String, idOpt: Option[Object], errorOpt: Option[Upload.Error]): Unit =
    for (id <- idOpt)
      underlying.processed(url, id, errorOpt.nonEmpty)

  override def start(): Unit =
    underlying.start()
  override def stop(keep: Boolean): Unit =
    underlying.stop(keep)
}

object InteractiveUploadLogger {
  def create(out: OutputStream, dummy: Boolean, isLocal: Boolean): UploadLogger =
    new InteractiveUploadLogger(new OutputStreamWriter(out), dummy, isLocal)
} 

package coursier.publish.checksum.logger

import java.io.{OutputStream, OutputStreamWriter, Writer}

import coursier.publish.checksum.ChecksumType
import coursier.publish.fileset.FileSet
import coursier.publish.logging.ProgressLogger

final class InteractiveChecksumLogger(out: Writer, verbosity: Int) extends ChecksumLogger {

  private val underlying = new ProgressLogger[Object](
    "Computed",
    "checksums",
    out
  )

  override def computingSet(id: Object, fs: FileSet): Unit =
    underlying.processingSet(id, Some(fs.elements.length))
  override def computing(id: Object, type0: ChecksumType, path: String): Unit = {
    if (verbosity >= 2)
      out.write(s"Computing ${type0.name} checksum of ${path.repr}\n")
    underlying.processing(path, id)
  }
  override def computed(id: Object, type0: ChecksumType, path: String, errorOpt: Option[Throwable]): Unit = {
    if (verbosity >= 2)
      out.write(s"Computed ${type0.name} checksum of ${path.repr}\n")
    underlying.processed(path, id, errorOpt.nonEmpty)
  }
  override def computedSet(id: Object, fs: FileSet): Unit =
    underlying.processedSet(id)

  override def start(): Unit =
    underlying.start()
  override def stop(keep: Boolean): Unit =
    underlying.stop(keep)
}

object InteractiveChecksumLogger {
  def create(out: OutputStream, verbosity: Int): InteractiveChecksumLogger =
    new InteractiveChecksumLogger(new OutputStreamWriter(out), verbosity)
} 

package coursier.publish.signing.logger

import java.io.{OutputStream, OutputStreamWriter, Writer}

import coursier.publish.fileset.{FileSet, Path}
import coursier.publish.logging.ProgressLogger

final class InteractiveSignerLogger(out: Writer, verbosity: Int) extends SignerLogger {

  private val underlying = new ProgressLogger[Object](
    "Signed",
    "files",
    out,
    updateOnChange = true,
    doneEmoji = Some("\u270D\uFE0F ")
  )

  override def signing(id: Object, fileSet: FileSet): Unit = {
    underlying.processingSet(id, Some(fileSet.elements.length))
  }
  override def signed(id: Object, fileSet: FileSet): Unit =
    underlying.processedSet(id)

  override def signingElement(id: Object, path: Path): Unit = {
    if (verbosity >= 2)
      out.write(s"Signing ${path.repr}\n")
    underlying.processing(path.repr, id)
  }
  override def signedElement(id: Object, path: Path, excOpt: Option[Throwable]): Unit = {
    if (verbosity >= 2)
      out.write(s"Signed ${path.repr}\n")
    underlying.processed(path.repr, id, excOpt.nonEmpty)
  }

  override def start(): Unit =
    underlying.start()
  override def stop(keep: Boolean): Unit =
    underlying.stop(keep)
}

object InteractiveSignerLogger {
  def create(out: OutputStream, verbosity: Int): SignerLogger =
    new InteractiveSignerLogger(new OutputStreamWriter(out), verbosity)
} 

package coursier.publish.dir.logger

import java.io.{OutputStream, OutputStreamWriter}
import java.nio.file.Path

import com.lightbend.emoji.ShortCodes.Defaults.defaultImplicit.emoji
import coursier.publish.logging.ProgressLogger

final class InteractiveDirLogger(out: OutputStreamWriter, dirName: String, verbosity: Int) extends DirLogger {

  private val underlying = new ProgressLogger[String](
    "Read",
    s"files from $dirName",
    out,
    doneEmoji = emoji("mag").map(_.toString())
  )

  override def reading(dir: Path): Unit =
    underlying.processingSet(dirName, None)
  override def element(dir: Path, file: Path): Unit = {
    underlying.processing(file.toString, dirName)
    underlying.processed(file.toString, dirName, false)
  }
  override def read(dir: Path, elements: Int): Unit =
    underlying.processedSet(dirName)

  override def start(): Unit =
    underlying.start()
  override def stop(keep: Boolean): Unit =
    underlying.stop(keep)
}

object InteractiveDirLogger {
  def create(out: OutputStream, dirName: String, verbosity: Int): DirLogger =
    new InteractiveDirLogger(new OutputStreamWriter(out), dirName, verbosity)
} 

package coursier.launcher

import java.io.{ByteArrayOutputStream, OutputStream}
import java.nio.file.{Files, Path}
import java.util.jar.{Attributes, JarOutputStream, Manifest}

import coursier.launcher.internal.FileUtil

object ManifestJarGenerator extends Generator[Parameters.ManifestJar] {

  def generate(parameters: Parameters.ManifestJar, output: Path): Unit = {

    val cp = parameters.classpath.map(_.toURI.getRawPath).mkString(" ")

    val manifest = new Manifest
    val attr = manifest.getMainAttributes
    attr.put(Attributes.Name.MANIFEST_VERSION, "1.0")
    attr.put(Attributes.Name.CLASS_PATH, cp)
    attr.put(Attributes.Name.MAIN_CLASS, parameters.mainClass)

    val content = {
      val baos = new ByteArrayOutputStream
      val jos = new JarOutputStream(baos, manifest)
      jos.close()
      baos.close()
      baos.toByteArray()
    }

    FileUtil.withOutputStream(output) { os =>

      for (p <- parameters.preambleOpt.map(_.value))
        os.write(p)

      os.write(content)
    }

    FileUtil.tryMakeExecutable(output)
  }
} 

package coursier.launcher.internal

import java.io.{ByteArrayOutputStream, InputStream, OutputStream}
import java.nio.file.attribute.PosixFilePermission
import java.nio.file.{Files, LinkOption, Path}

import scala.collection.JavaConverters._

private[coursier] object FileUtil {

  // Won't be necessary anymore with Java 9
  // (https://docs.oracle.com/javase/9/docs/api/java/io/InputStream.html#readAllBytes--,
  // via https://stackoverflow.com/questions/1264709/convert-inputstream-to-byte-array-in-java/37681322#37681322)
  def readFullyUnsafe(is: InputStream): Array[Byte] = {
    val buffer = new ByteArrayOutputStream
    val data = Array.ofDim[Byte](16384)

    var nRead = 0
    while ({
      nRead = is.read(data, 0, data.length)
      nRead != -1
    })
      buffer.write(data, 0, nRead)

    buffer.flush()
    buffer.toByteArray
  }

  def readFully(is: => InputStream): Array[Byte] = {
    var is0: InputStream = null
    try {
      is0 = is
      readFullyUnsafe(is0)
    } finally {
      if (is0 != null)
        is0.close()
    }
  }

  def withOutputStream[T](path: Path)(f: OutputStream => T): T = {
    var os: OutputStream = null
    try {
      os = Files.newOutputStream(path)
      f(os)
    } finally {
      if (os != null)
        os.close()
    }
  }

  def tryMakeExecutable(path: Path): Boolean = {

    try {
      val perms = Files.getPosixFilePermissions(path).asScala.toSet

      var newPerms = perms
      if (perms(PosixFilePermission.OWNER_READ))
        newPerms += PosixFilePermission.OWNER_EXECUTE
      if (perms(PosixFilePermission.GROUP_READ))
        newPerms += PosixFilePermission.GROUP_EXECUTE
      if (perms(PosixFilePermission.OTHERS_READ))
        newPerms += PosixFilePermission.OTHERS_EXECUTE

      if (newPerms != perms)
        Files.setPosixFilePermissions(
          path,
          newPerms.asJava
        )

      true
    } catch {
      case _: UnsupportedOperationException =>
        false
    }
  }

  def tryHideWindows(path: Path): Boolean =
    Windows.isWindows && {
      try {
        Files.setAttribute(path, "dos:hidden", java.lang.Boolean.TRUE, LinkOption.NOFOLLOW_LINKS)
        true
      } catch {
        case _: UnsupportedOperationException =>
          false
      }
    }

} 

package better.files

import java.io.{OutputStream, Writer}
import java.nio.charset.{Charset, CharsetDecoder, CodingErrorAction}
import java.nio.{ByteBuffer, CharBuffer}

import scala.annotation.tailrec


  private[this] val decoderIn = ByteBuffer.allocate(bufferSize >> 4)

  def this(
      writer: Writer,
      bufferSize: Int = DefaultBufferSize,
      flushImmediately: Boolean = false
  )(implicit
      charset: Charset = DefaultCharset
  ) =
    this(
      writer = writer,
      decoder = charset.newDecoder
        .onMalformedInput(CodingErrorAction.REPLACE)
        .onUnmappableCharacter(CodingErrorAction.REPLACE)
        .replaceWith("?"),
      bufferSize = bufferSize,
      flushImmediately = flushImmediately
    )

  override def write(b: Array[Byte], off: Int, len: Int) = {
    @tailrec def loop(off: Int, len: Int): Unit =
      if (len > 0) {
        val c = decoderIn.remaining min len
        decoderIn.put(b, off, c)
        processInput(endOfInput = false)
        loop(off + c, len - c)
      }
    loop(off, len)
    if (flushImmediately) flushOutput()
  }

  override def write(b: Int) = write(Array(b.toByte))

  override def flush() = {
    flushOutput()
    writer.flush()
  }

  override def close() = {
    processInput(endOfInput = true)
    flushOutput()
    writer.close()
  }

  private[this] def processInput(endOfInput: Boolean) = {
    decoderIn.flip()
    @tailrec def loop(): Unit = {
      val coderResult = decoder.decode(decoderIn, decoderOut, endOfInput)
      if (coderResult.isOverflow) {
        flushOutput()
        loop()
      } else {
        assert(coderResult.isUnderflow, "decoder is configured to replace malformed input and unmappable characters")
      }
    }
    loop()
    decoderIn.compact()
  }

  private[this] def flushOutput(): Unit = {
    val p = decoderOut.position()
    if (p > 0) {
      writer.write(decoderOut.array, 0, p)
      val _ = decoderOut.rewind()
    }
  }
} 

package io.github.howardjohn.lambda

import java.io.{InputStream, OutputStream}
import java.nio.charset.StandardCharsets

import scala.io.Source

object StreamOps {
  implicit class InputStreamOps(val is: InputStream) extends AnyVal {
    def consume(): String = {
      val contents = Source.fromInputStream(is).mkString
      is.close()
      contents
    }
  }

  implicit class OutputStreamOps(val os: OutputStream) extends AnyVal {
    def writeAndClose(contents: String): Unit = {
      os.write(contents.getBytes(StandardCharsets.UTF_8))
      os.close()
    }
  }
} 

package io.github.howardjohn.lambda

import java.io.{InputStream, OutputStream}

import io.github.howardjohn.lambda.ProxyEncoding._
import io.github.howardjohn.lambda.StreamOps._

trait LambdaHandler {
  def handleRequest(request: ProxyRequest): ProxyResponse

  def handle(is: InputStream, os: OutputStream): Unit = {
    val rawInput = is.consume()
    val request = parseRequest(rawInput).fold(
      e => throw e,
      identity
    )
    val rawResponse = handleRequest(request)
    val response = encodeResponse(rawResponse)
    os.writeAndClose(response)
  }
} 

package crossroad0201.aws.flywaylambda

import java.io.{BufferedOutputStream, InputStream, OutputStream, PrintWriter}

import com.amazonaws.regions.{Region, Regions}
import com.amazonaws.services.lambda.runtime.{Context, RequestStreamHandler}
import com.amazonaws.services.s3.{AmazonS3, AmazonS3Client}

import scala.io.{BufferedSource, Codec}
import scala.util.{Failure, Success, Try}

class InvokeMigrationHandler extends RequestStreamHandler with S3MigrationHandlerBase {
  type BucketName = String
  type Prefix = String
  type ConfFileName = String

  override def handleRequest(input: InputStream, output: OutputStream, context: Context): Unit = {
    def parseInput: Try[(BucketName, Prefix, ConfFileName)] = Try {
      import spray.json._
      import DefaultJsonProtocol._

      val json = new BufferedSource(input)(Codec("UTF-8")).mkString
      val jsObj = JsonParser(json).toJson.asJsObject
      jsObj.getFields(
        "bucket_name",
        "prefix"
      ) match {
        case Seq(JsString(b), JsString(p)) => {
          jsObj.getFields(
            "flyway_conf"
          ) match {
            case Seq(JsString(c)) => (b, p, c)
            case _ => (b, p, "flyway.conf")
          }
        }
        case _ => throw new IllegalArgumentException(s"Missing require key [bucketName, prefix]. - $json")
      }
    }

    val logger = context.getLogger

    implicit val s3Client: AmazonS3 = new AmazonS3Client().withRegion(Region.getRegion(Regions.fromName(sys.env("AWS_REGION"))))

    (for {
      i <- parseInput
      _ = { logger.log(s"Flyway migration start. by invoke lambda function(${i._1}, ${i._2}, ${i._3}).") }
      r <- migrate(i._1, i._2, i._3)(context, s3Client)
    } yield r) match {
      case Success(r) =>
        logger.log(r)
        val b = r.getBytes("UTF-8")
        val bout = new BufferedOutputStream(output)
        Stream.continually(bout.write(b))
        bout.flush()
      case Failure(e) =>
        e.printStackTrace()
        val w = new PrintWriter(output)
        w.write(e.toString)
        w.flush()
    }
  }

} 

package com.programmaticallyspeaking.ncd.testing

import ch.qos.logback.classic.spi.ILoggingEvent
import ch.qos.logback.core.UnsynchronizedAppenderBase
import ch.qos.logback.core.encoder.Encoder
import ch.qos.logback.core.status.ErrorStatus
import java.io.{ByteArrayOutputStream, IOException, OutputStream}
import java.nio.charset.StandardCharsets

import com.programmaticallyspeaking.ncd.messaging.{Observable, SerializedSubject}

object MemoryAppender {
  private[MemoryAppender] val logEventSubject = new SerializedSubject[String]

  def logEvents: Observable[String] = logEventSubject
}

class MemoryAppender extends UnsynchronizedAppenderBase[ILoggingEvent] {
  import MemoryAppender._
  private var encoder: Encoder[ILoggingEvent] = _
  private var outputStream = new OutputStream {
    override def write(b: Int): Unit = ???

    override def write(b: Array[Byte]): Unit = {
      val str = new String(b, StandardCharsets.UTF_8)
      logEventSubject.onNext(str)
    }
  }

  override def start(): Unit = {
    try {
      Option(encoder).foreach(_.init(outputStream))
      super.start()
    } catch {
      case e: IOException =>
        started = false
        addStatus(new ErrorStatus("Failed to initialize encoder for appender named [" + name + "].", this, e))
    }
  }

  override protected def append(event: ILoggingEvent): Unit = {
    if (!isStarted) return
    try {
      event.prepareForDeferredProcessing()
      Option(encoder).foreach(_.doEncode(event))
    } catch {
      case ioe: IOException =>
        started = false
        addStatus(new ErrorStatus("IO failure in appender", this, ioe))
    }
  }

  def setEncoder(e: Encoder[ILoggingEvent]): Unit = {
    encoder = e
  }
} 

package org.argus.jawa.core.util

import java.io.{File, FileOutputStream, InputStream, OutputStream}
import java.util.zip.{ZipEntry, ZipFile}

import scala.collection.JavaConverters._


object ZipUtil {
  val BUFSIZE = 4096
  val buffer = new Array[Byte](BUFSIZE)

  def unZip(source: String, targetFolder: String): Boolean = {
    val zipFile = new ZipFile(source)

    unzipAllFile(zipFile.entries.asScala.toList, getZipEntryInputStream(zipFile), new File(targetFolder))
  }

  def getZipEntryInputStream(zipFile: ZipFile)(entry: ZipEntry): InputStream = zipFile.getInputStream(entry)

  def unzipAllFile(entryList: List[ZipEntry], inputGetter: ZipEntry => InputStream, targetFolder: File): Boolean = {

    entryList match {
      case entry :: entries =>

        if (entry.isDirectory)
          new File(targetFolder, entry.getName).mkdirs
        else
          saveFile(inputGetter(entry), new FileOutputStream(new File(targetFolder, entry.getName)))

        unzipAllFile(entries, inputGetter, targetFolder)
      case _ =>
        true
    }

  }

  def saveFile(fis: InputStream, fos: OutputStream): Unit = {
    writeToFile(bufferReader(fis), fos)
    fis.close()
    fos.close()
  }

  def bufferReader(fis: InputStream)(buffer: Array[Byte]): (Int, Array[Byte]) = (fis.read(buffer), buffer)

  def writeToFile(reader: Array[Byte] => (Int, Array[Byte]), fos: OutputStream): Boolean = {
    val (length, data) = reader(buffer)
    if (length >= 0) {
      fos.write(data, 0, length)
      writeToFile(reader, fos)
    } else
      true
  }
} 

package is.hail.utils.richUtils

import java.io.OutputStream

class ByteTrackingOutputStream(base: OutputStream) extends OutputStream {
  var bytesWritten = 0L

  def write(c: Int) {
    bytesWritten += 1
    base.write(c)
  }

  override def write(b: Array[Byte]) {
    base.write(b)
    bytesWritten += b.length
  }

  override def write(b: Array[Byte], off: Int, len: Int) {
    base.write(b, off, len)
    bytesWritten += len
  }

  override def close(): Unit = base.close()
} 

package is.hail.utils

import java.io.{ByteArrayOutputStream, OutputStream}
import java.nio.charset.StandardCharsets

import org.apache.log4j.{Level, Logger}

class LoggerOutputStream(logger: Logger, level: Level) extends OutputStream {
  private val buffer = new ByteArrayOutputStream()

  override def write(b: Int) {
    buffer.write(b)
    if (b == '\n') {
      val line = buffer.toString(StandardCharsets.UTF_8.name())
      level match {
        case Level.TRACE => logger.trace(line)
        case Level.DEBUG => logger.debug(line)
        case Level.INFO  => logger.info(line)
        case Level.WARN  => logger.warn(line)
        case Level.ERROR => logger.error(line)
      }
      buffer.reset()
    }
  }
} 

package is.hail.utils

import java.net.URL
import java.io.OutputStream
import java.io.InputStream
import java.net.HttpURLConnection
import is.hail.utils._
import java.nio.charset.StandardCharsets
import org.apache.commons.io.output.ByteArrayOutputStream


object HTTPClient {
  def post[T](
    url: String,
    contentLength: Int,
    writeBody: OutputStream => Unit,
    readResponse: InputStream => T = (_: InputStream) => (),
    chunkSize: Int = 0
  ): T = {
    val conn = new URL(url).openConnection().asInstanceOf[HttpURLConnection]
    conn.setRequestMethod("POST")
    if (chunkSize > 0)
      conn.setChunkedStreamingMode(chunkSize)
    conn.setDoOutput(true);
    conn.setRequestProperty("Content-Length", Integer.toString(contentLength))
    using(conn.getOutputStream())(writeBody)
    assert(200 <= conn.getResponseCode() && conn.getResponseCode() < 300,
      s"POST ${url} ${conn.getResponseCode()} ${using(conn.getErrorStream())(fullyReadInputStreamAsString)}")
    val result = using(conn.getInputStream())(readResponse)
    conn.disconnect()
    result
  }

  def get[T](
    url: String,
    readResponse: InputStream => T
  ): T = {
    val conn = new URL(url).openConnection().asInstanceOf[HttpURLConnection]
    conn.setRequestMethod("GET")
    assert(200 <= conn.getResponseCode() && conn.getResponseCode() < 300,
      s"GET ${url} ${conn.getResponseCode()} ${using(conn.getErrorStream())(fullyReadInputStreamAsString)}")
    val result = using(conn.getInputStream())(readResponse)
    conn.disconnect()
    result
  }

  def delete(
    url: String,
    readResponse: InputStream => Unit = (_: InputStream) => ()
  ): Unit = {
    val conn = new URL(url).openConnection().asInstanceOf[HttpURLConnection]
    conn.setRequestMethod("DELETE")
    assert(200 <= conn.getResponseCode() && conn.getResponseCode() < 300,
      s"DELETE ${url} ${conn.getResponseCode()} ${using(conn.getErrorStream())(fullyReadInputStreamAsString)}")
    val result = using(conn.getInputStream())(readResponse)
    conn.disconnect()
    result
  }

  private[this] def fullyReadInputStreamAsString(is: InputStream): String =
    using(new ByteArrayOutputStream()) { baos =>
      drainInputStreamToOutputStream(is, baos)
      new String(baos.toByteArray(), StandardCharsets.UTF_8)
    }
} 

package is.hail.io

import java.io.{ByteArrayInputStream, ByteArrayOutputStream, InputStream, OutputStream}

import is.hail.annotations.{Region, RegionValue}
import is.hail.asm4s.{Code, TypeInfo, Value}
import is.hail.expr.ir.{EmitClassBuilder, EmitFunctionBuilder, ExecuteContext, typeToTypeInfo}
import is.hail.types.encoded.EType
import is.hail.types.physical.PType
import is.hail.types.virtual.Type
import is.hail.rvd.RVDContext
import is.hail.sparkextras.ContextRDD
import is.hail.utils.using
import org.apache.spark.rdd.RDD

trait AbstractTypedCodecSpec extends Spec {
  def encodedType: EType
  def encodedVirtualType: Type

  type StagedEncoderF[T] = (Value[Region], Value[T], Value[OutputBuffer]) => Code[Unit]
  type StagedDecoderF[T] = (Value[Region], Value[InputBuffer]) => Code[T]

  def buildEncoder(ctx: ExecuteContext, t: PType): (OutputStream) => Encoder

  def decodedPType(requestedType: Type): PType

  def buildDecoder(ctx: ExecuteContext, requestedType: Type): (PType, (InputStream) => Decoder)

  def encode(ctx: ExecuteContext, t: PType, offset: Long): Array[Byte] = {
    val baos = new ByteArrayOutputStream()
    using(buildEncoder(ctx, t)(baos))(_.writeRegionValue(offset))
    baos.toByteArray
  }

  def decode(ctx: ExecuteContext, requestedType: Type, bytes: Array[Byte], region: Region): (PType, Long) = {
    val bais = new ByteArrayInputStream(bytes)
    val (pt, dec) = buildDecoder(ctx, requestedType)
    (pt, dec(bais).readRegionValue(region))
  }

  def buildCodeInputBuffer(is: Code[InputStream]): Code[InputBuffer]

  def buildCodeOutputBuffer(os: Code[OutputStream]): Code[OutputBuffer]

  def buildEmitDecoderF[T](requestedType: Type, cb: EmitClassBuilder[_]): (PType, StagedDecoderF[T])

  def buildEmitEncoderF[T](t: PType, cb: EmitClassBuilder[_]): StagedEncoderF[T]

  def buildEmitDecoderF[T](requestedType: Type, cb: EmitClassBuilder[_], ti: TypeInfo[T]): (PType, StagedDecoderF[T]) = {
    val (ptype, dec) = buildEmitDecoderF[T](requestedType, cb)
    assert(ti == typeToTypeInfo(requestedType))
    ptype -> dec
  }

  def buildEmitEncoderF[T](t: PType, cb: EmitClassBuilder[_], ti: TypeInfo[T]): StagedEncoderF[T] = {
    assert(ti == typeToTypeInfo(t))
    buildEmitEncoderF[T](t, cb)
  }

  // FIXME: is there a better place for this to live?
  def decodeRDD(ctx: ExecuteContext, requestedType: Type, bytes: RDD[Array[Byte]]): (PType, ContextRDD[Long]) = {
    val (pt, dec) = buildDecoder(ctx, requestedType)
    (pt, ContextRDD.weaken(bytes).cmapPartitions { (ctx, it) =>
      RegionValue.fromBytes(dec, ctx.region, it)
    })
  }

  override def toString: String = super[Spec].toString
} 

package is.hail.io

import java.io.{Closeable, InputStream, OutputStream}

import is.hail.annotations.Memory
import is.hail.utils._

final class DoubleInputBuffer(in: InputStream, bufSize: Int) extends Closeable {
  private val buf = new Array[Byte](bufSize)
  private var end: Int = 0
  private var off: Int = 0

  def close() {
    in.close()
  }

  def readDoubles(to: Array[Double]): Unit = readDoubles(to, 0, to.length)

  def readDoubles(to: Array[Double], toOff0: Int, n0: Int) {
    assert(toOff0 >= 0)
    assert(n0 >= 0)
    assert(toOff0 <= to.length - n0)

    var toOff = toOff0
    var n = n0.toLong

    while (n > 0) {
      if (end == off) {
        val len = math.min(bufSize, n << 3).toInt
        in.readFully(buf, 0, len)
        end = len
        off = 0
      }
      val p = math.min(end - off, n << 3).toInt >>> 3
      assert(p > 0)
      Memory.memcpy(to, toOff, buf, off, p)
      toOff += p
      n -= p
      off += (p << 3)
    }
  }
}

final class DoubleOutputBuffer(out: OutputStream, bufSize: Int) extends Closeable {
  private val buf: Array[Byte] = new Array[Byte](bufSize)
  private var off: Int = 0

  def close() {
    flush()
    out.close()
  }

  def flush() {
    out.write(buf, 0, off)
  }

  def writeDoubles(from: Array[Double]): Unit = writeDoubles(from, 0, from.length)

  def writeDoubles(from: Array[Double], fromOff0: Int, n0: Int) {
    assert(n0 >= 0)
    assert(fromOff0 >= 0)
    assert(fromOff0 <= from.length - n0)
    var fromOff = fromOff0
    var n = n0.toLong

    while (off + (n << 3) > bufSize) {
      val p = (buf.length - off) >>> 3
      Memory.memcpy(buf, off, from, fromOff, p)
      off += (p << 3)
      fromOff += p
      n -= p
      out.write(buf, 0, off)
      off = 0
    }
    Memory.memcpy(buf, off, from, fromOff, n)
    off += (n.toInt << 3)
  }
} 

package lila.openingexplorer

import java.io.{ InputStream, OutputStream }
import chess.format.Uci
import chess.{ Pos, Role }

trait PackHelper {

  protected def writeUint(stream: OutputStream, v: Long) = {
    var value = v
    while (value > 127) {
      stream.write(((value & 127) | 128).toInt)
      value >>= 7
    }
    stream.write((value & 127).toInt)
  }

  protected def readUint(stream: InputStream): Long = {
    var value: Long = 0
    var i: Int      = 0
    var byte: Int   = 0

    do {
      byte = stream.read()
      value |= (byte.toLong & 127) << (7 * i)
      i += 1
    } while ((byte & 128) != 0)

    value
  }

  protected def writeUint16(stream: OutputStream, v: Int) = {
    stream.write(0xff & (v >> 8))
    stream.write(0xff & v)
  }

  protected def readUint16(stream: InputStream): Int =
    stream.read() << 8 | stream.read()

  protected def writeUint48(stream: OutputStream, v: Long) = {
    stream.write((0xff & (v >> 40)).toInt)
    stream.write((0xff & (v >> 32)).toInt)
    stream.write((0xff & (v >> 24)).toInt)
    stream.write((0xff & (v >> 16)).toInt)
    stream.write((0xff & (v >> 8)).toInt)
    stream.write((0xff & v).toInt)
  }

  protected def readUint48(stream: InputStream): Long =
    stream.read.toLong << 40 | stream.read.toLong << 32 |
      stream.read.toLong << 24 | stream.read.toLong << 16 |
      stream.read.toLong << 8 | stream.read.toLong

  protected def writeUci(stream: OutputStream, move: Uci.Move): Unit =
    writeUint16(
      stream,
      Pos.all.indexOf(move.orig) |
        Pos.all.indexOf(move.dest) << 6 |
        move.promotion.fold(0)(r => (Role.allPromotable.indexOf(r)) + 1) << 12
    )

  protected def writeUci(stream: OutputStream, drop: Uci.Drop): Unit = {
    val dest = Pos.all.indexOf(drop.pos)
    writeUint16(stream, dest | dest << 6 | (Role.all.indexOf(drop.role) + 1) << 12)
  }

  protected def writeUci(stream: OutputStream, move: Either[Uci.Move, Uci.Drop]): Unit =
    move.fold(writeUci(stream, _), writeUci(stream, _))

  protected def readUci(stream: InputStream): Either[Uci.Move, Uci.Drop] = {
    val enc  = readUint16(stream)
    val orig = Pos.all(enc & 63)
    val dest = Pos.all((enc >> 6) & 63)
    if (orig == dest) {
      Right(new Uci.Drop(Role.all((enc >> 12) - 1), dest))
    } else {
      val role = if ((enc >> 12) != 0) Some(Role.allPromotable((enc >> 12) - 1)) else None
      Left(new Uci.Move(orig, dest, role))
    }
  }
} 

package lila.openingexplorer

import chess.format.Uci
import java.io.{ InputStream, OutputStream }

case class SubEntry(
    moves: Map[Either[Uci.Move, Uci.Drop], MoveStats],
    gameRefs: List[GameRef]
) extends PackHelper {

  lazy val totalWhite = moves.values.map(_.white).sum
  lazy val totalDraws = moves.values.map(_.draws).sum
  lazy val totalBlack = moves.values.map(_.black).sum

  def totalGames = totalWhite + totalDraws + totalBlack

  def isEmpty = totalGames == 0

  def totalAverageRatingSum = moves.values.map(_.averageRatingSum).sum

  def averageRating: Int =
    if (totalGames == 0) 0 else (totalAverageRatingSum / totalGames).toInt

  def withGameRef(game: GameRef, move: Either[Uci.Move, Uci.Drop]) =
    new SubEntry(
      moves + (move -> moves.getOrElse(move, MoveStats.empty).withGameRef(game)),
      game :: gameRefs
    )

  def withExistingGameRef(game: GameRef) = copy(gameRefs = game :: gameRefs)

  def withoutExistingGameRef(game: GameRef, move: Either[Uci.Move, Uci.Drop]) = {
    val stats =
      moves
        .get(move)
        .map(_.withoutExistingGameRef(game: GameRef))
        .getOrElse(MoveStats.empty)

    new SubEntry(
      if (stats.total > 0) moves + (move -> stats) else moves - move,
      gameRefs.filterNot(_.gameId == game.gameId)
    )
  }

  def writeStats(out: OutputStream) = {
    writeUint(out, moves.size)
    moves.foreach {
      case (move, stats) =>
        writeUci(out, move)
        stats.write(out)
    }
  }

  def write(out: OutputStream) = {
    writeStats(out)

    gameRefs
      .sortWith(_.averageRating > _.averageRating)
      .distinct
      .take(SubEntry.maxTopGames)
      .foreach(_.write(out))
  }
}

object SubEntry extends PackHelper {

  val maxTopGames = 4

  def empty = new SubEntry(Map.empty, List.empty)

  def fromGameRef(game: GameRef, move: Either[Uci.Move, Uci.Drop]) =
    empty.withGameRef(game, move)

  def fromExistingGameRef(game: GameRef) =
    empty.withExistingGameRef(game)

  def readStats(in: InputStream, gameRefs: List[GameRef] = List.empty): SubEntry = {
    var remainingMoves = readUint(in)
    val moves          = scala.collection.mutable.Map.empty[Either[Uci.Move, Uci.Drop], MoveStats]
    while (remainingMoves > 0) {
      moves += (readUci(in) -> MoveStats.read(in))
      remainingMoves -= 1;
    }
    new SubEntry(moves.toMap, gameRefs)
  }

  def read(in: InputStream) = {
    val subEntry = readStats(in)

    val gameRefs = scala.collection.mutable.ListBuffer.empty[GameRef]
    while (in.available > 0) {
      gameRefs += GameRef.read(in)
    }

    subEntry.copy(gameRefs = gameRefs.toList)
  }
} 

package lila.openingexplorer

import java.io.{ InputStream, OutputStream }

import chess.Color

case class MoveStats(
    white: Long,
    draws: Long,
    black: Long,
    averageRatingSum: Long
) extends PackHelper {

  def total = white + draws + black

  def isEmpty = total == 0

  def averageRating: Int =
    if (total == 0) 0 else (averageRatingSum / total).toInt

  def withGameRef(game: GameRef) = {
    val avgRatingSum = averageRatingSum + game.averageRating

    game.winner match {
      case Some(Color.White) =>
        copy(white = white + 1, averageRatingSum = avgRatingSum)
      case Some(Color.Black) =>
        copy(black = black + 1, averageRatingSum = avgRatingSum)
      case None =>
        copy(draws = draws + 1, averageRatingSum = avgRatingSum)
    }
  }

  def withoutExistingGameRef(game: GameRef) = {
    val avgRatingSum = averageRatingSum - game.averageRating

    game.winner match {
      case Some(Color.White) =>
        copy(white = white - 1, averageRatingSum = avgRatingSum)
      case Some(Color.Black) =>
        copy(black = black - 1, averageRatingSum = avgRatingSum)
      case None =>
        copy(draws = draws - 1, averageRatingSum = avgRatingSum)
    }
  }

  def add(other: MoveStats) =
    new MoveStats(
      white + other.white,
      draws + other.draws,
      black + other.black,
      averageRatingSum + other.averageRatingSum
    )

  def write(out: OutputStream) = {
    writeUint(out, white)
    writeUint(out, draws)
    writeUint(out, black)
    writeUint(out, averageRatingSum)
  }
}

object MoveStats extends PackHelper {

  def empty = new MoveStats(0, 0, 0, 0)

  def fromGameRef(game: GameRef) = empty.withGameRef(game)

  def read(in: InputStream) =
    new MoveStats(readUint(in), readUint(in), readUint(in), readUint(in))
} 

package org.apache.spark.sql.execution.streaming


import java.io.{InputStream, OutputStream}
import java.nio.charset.StandardCharsets._

import scala.io.{Source => IOSource}

import org.apache.spark.sql.SparkSession


class OffsetSeqLog(sparkSession: SparkSession, path: String)
  extends HDFSMetadataLog[OffsetSeq](sparkSession, path) {

  override protected def deserialize(in: InputStream): OffsetSeq = {
    // called inside a try-finally where the underlying stream is closed in the caller
    def parseOffset(value: String): Offset = value match {
      case OffsetSeqLog.SERIALIZED_VOID_OFFSET => null
      case json => SerializedOffset(json)
    }
    val lines = IOSource.fromInputStream(in, UTF_8.name()).getLines()
    if (!lines.hasNext) {
      throw new IllegalStateException("Incomplete log file")
    }
    val version = lines.next()
    if (version != OffsetSeqLog.VERSION) {
      throw new IllegalStateException(s"Unknown log version: ${version}")
    }

    // read metadata
    val metadata = lines.next().trim match {
      case "" => None
      case md => Some(md)
    }
    OffsetSeq.fill(metadata, lines.map(parseOffset).toArray: _*)
  }

  override protected def serialize(offsetSeq: OffsetSeq, out: OutputStream): Unit = {
    // called inside a try-finally where the underlying stream is closed in the caller
    out.write(OffsetSeqLog.VERSION.getBytes(UTF_8))

    // write metadata
    out.write('\n')
    out.write(offsetSeq.metadata.map(_.json).getOrElse("").getBytes(UTF_8))

    // write offsets, one per line
    offsetSeq.offsets.map(_.map(_.json)).foreach { offset =>
      out.write('\n')
      offset match {
        case Some(json: String) => out.write(json.getBytes(UTF_8))
        case None => out.write(OffsetSeqLog.SERIALIZED_VOID_OFFSET.getBytes(UTF_8))
      }
    }
  }
}

object OffsetSeqLog {
  private val VERSION = "v1"
  private val SERIALIZED_VOID_OFFSET = "-"
} 

package org.apache.spark.streaming.util

import java.io.OutputStream
import java.util.concurrent.TimeUnit._

import scala.annotation.tailrec

import org.apache.spark.internal.Logging

private[streaming]
class RateLimitedOutputStream(out: OutputStream, desiredBytesPerSec: Int)
  extends OutputStream
  with Logging {

  require(desiredBytesPerSec > 0)

  private val SYNC_INTERVAL = NANOSECONDS.convert(10, SECONDS)
  private val CHUNK_SIZE = 8192
  private var lastSyncTime = System.nanoTime
  private var bytesWrittenSinceSync = 0L

  override def write(b: Int) {
    waitToWrite(1)
    out.write(b)
  }

  override def write(bytes: Array[Byte]) {
    write(bytes, 0, bytes.length)
  }

  @tailrec
  override final def write(bytes: Array[Byte], offset: Int, length: Int) {
    val writeSize = math.min(length - offset, CHUNK_SIZE)
    if (writeSize > 0) {
      waitToWrite(writeSize)
      out.write(bytes, offset, writeSize)
      write(bytes, offset + writeSize, length)
    }
  }

  override def flush() {
    out.flush()
  }

  override def close() {
    out.close()
  }

  @tailrec
  private def waitToWrite(numBytes: Int) {
    val now = System.nanoTime
    val elapsedNanosecs = math.max(now - lastSyncTime, 1)
    val rate = bytesWrittenSinceSync.toDouble * 1000000000 / elapsedNanosecs
    if (rate < desiredBytesPerSec) {
      // It's okay to write; just update some variables and return
      bytesWrittenSinceSync += numBytes
      if (now > lastSyncTime + SYNC_INTERVAL) {
        // Sync interval has passed; let's resync
        lastSyncTime = now
        bytesWrittenSinceSync = numBytes
      }
    } else {
      // Calculate how much time we should sleep to bring ourselves to the desired rate.
      val targetTimeInMillis = bytesWrittenSinceSync * 1000 / desiredBytesPerSec
      val elapsedTimeInMillis = elapsedNanosecs / 1000000
      val sleepTimeInMillis = targetTimeInMillis - elapsedTimeInMillis
      if (sleepTimeInMillis > 0) {
        logTrace("Natural rate is " + rate + " per second but desired rate is " +
          desiredBytesPerSec + ", sleeping for " + sleepTimeInMillis + " ms to compensate.")
        Thread.sleep(sleepTimeInMillis)
      }
      waitToWrite(numBytes)
    }
  }
} 

package org.apache.spark.security

import java.io.{InputStream, OutputStream}
import java.util.Properties
import javax.crypto.KeyGenerator
import javax.crypto.spec.{IvParameterSpec, SecretKeySpec}

import org.apache.commons.crypto.random._
import org.apache.commons.crypto.stream._

import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.internal.config._


  private[this] def createInitializationVector(properties: Properties): Array[Byte] = {
    val iv = new Array[Byte](IV_LENGTH_IN_BYTES)
    val initialIVStart = System.currentTimeMillis()
    CryptoRandomFactory.getCryptoRandom(properties).nextBytes(iv)
    val initialIVFinish = System.currentTimeMillis()
    val initialIVTime = initialIVFinish - initialIVStart
    if (initialIVTime > 2000) {
      logWarning(s"It costs ${initialIVTime} milliseconds to create the Initialization Vector " +
        s"used by CryptoStream")
    }
    iv
  }
} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.lang.annotation.Annotation
import java.lang.reflect.Type
import java.nio.charset.StandardCharsets
import java.text.SimpleDateFormat
import java.util.{Calendar, Locale, SimpleTimeZone}
import javax.ws.rs.Produces
import javax.ws.rs.core.{MediaType, MultivaluedMap}
import javax.ws.rs.ext.{MessageBodyWriter, Provider}

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}


@Provider
@Produces(Array(MediaType.APPLICATION_JSON))
private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{

  val mapper = new ObjectMapper() {
    override def writeValueAsString(t: Any): String = {
      super.writeValueAsString(t)
    }
  }
  mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
  mapper.enable(SerializationFeature.INDENT_OUTPUT)
  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)

  override def isWriteable(
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Boolean = {
      true
  }

  override def writeTo(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType,
      multivaluedMap: MultivaluedMap[String, AnyRef],
      outputStream: OutputStream): Unit = {
    t match {
      case ErrorWrapper(err) => outputStream.write(err.getBytes(StandardCharsets.UTF_8))
      case _ => mapper.writeValue(outputStream, t)
    }
  }

  override def getSize(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Long = {
    -1L
  }
}

private[spark] object JacksonMessageWriter {
  def makeISODateFormat: SimpleDateFormat = {
    val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'", Locale.US)
    val cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"))
    iso8601.setCalendar(cal)
    iso8601
  }
} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.util.zip.ZipOutputStream
import javax.ws.rs.{GET, Produces}
import javax.ws.rs.core.{MediaType, Response, StreamingOutput}

import scala.util.control.NonFatal

import org.apache.spark.SparkConf
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.internal.Logging

@Produces(Array(MediaType.APPLICATION_OCTET_STREAM))
private[v1] class EventLogDownloadResource(
    val uIRoot: UIRoot,
    val appId: String,
    val attemptId: Option[String]) extends Logging {
  val conf = SparkHadoopUtil.get.newConfiguration(new SparkConf)

  @GET
  def getEventLogs(): Response = {
    try {
      val fileName = {
        attemptId match {
          case Some(id) => s"eventLogs-$appId-$id.zip"
          case None => s"eventLogs-$appId.zip"
        }
      }

      val stream = new StreamingOutput {
        override def write(output: OutputStream): Unit = {
          val zipStream = new ZipOutputStream(output)
          try {
            uIRoot.writeEventLogs(appId, attemptId, zipStream)
          } finally {
            zipStream.close()
          }

        }
      }

      Response.ok(stream)
        .header("Content-Disposition", s"attachment; filename=$fileName")
        .header("Content-Type", MediaType.APPLICATION_OCTET_STREAM)
        .build()
    } catch {
      case NonFatal(e) =>
        Response.serverError()
          .entity(s"Event logs are not available for app: $appId.")
          .status(Response.Status.SERVICE_UNAVAILABLE)
          .build()
    }
  }
} 

package org.apache.spark.util.io

import java.io.OutputStream
import java.nio.ByteBuffer

import scala.collection.mutable.ArrayBuffer

import org.apache.spark.storage.StorageUtils


  private[this] var position = chunkSize
  private[this] var _size = 0
  private[this] var closed: Boolean = false

  def size: Long = _size

  override def close(): Unit = {
    if (!closed) {
      super.close()
      closed = true
    }
  }

  override def write(b: Int): Unit = {
    require(!closed, "cannot write to a closed ChunkedByteBufferOutputStream")
    allocateNewChunkIfNeeded()
    chunks(lastChunkIndex).put(b.toByte)
    position += 1
    _size += 1
  }

  override def write(bytes: Array[Byte], off: Int, len: Int): Unit = {
    require(!closed, "cannot write to a closed ChunkedByteBufferOutputStream")
    var written = 0
    while (written < len) {
      allocateNewChunkIfNeeded()
      val thisBatch = math.min(chunkSize - position, len - written)
      chunks(lastChunkIndex).put(bytes, written + off, thisBatch)
      written += thisBatch
      position += thisBatch
    }
    _size += len
  }

  @inline
  private def allocateNewChunkIfNeeded(): Unit = {
    if (position == chunkSize) {
      chunks += allocator(chunkSize)
      lastChunkIndex += 1
      position = 0
    }
  }

  def toChunkedByteBuffer: ChunkedByteBuffer = {
    require(closed, "cannot call toChunkedByteBuffer() unless close() has been called")
    require(!toChunkedByteBufferWasCalled, "toChunkedByteBuffer() can only be called once")
    toChunkedByteBufferWasCalled = true
    if (lastChunkIndex == -1) {
      new ChunkedByteBuffer(Array.empty[ByteBuffer])
    } else {
      // Copy the first n-1 chunks to the output, and then create an array that fits the last chunk.
      // An alternative would have been returning an array of ByteBuffers, with the last buffer
      // bounded to only the last chunk's position. However, given our use case in Spark (to put
      // the chunks in block manager), only limiting the view bound of the buffer would still
      // require the block manager to store the whole chunk.
      val ret = new Array[ByteBuffer](chunks.size)
      for (i <- 0 until chunks.size - 1) {
        ret(i) = chunks(i)
        ret(i).flip()
      }
      if (position == chunkSize) {
        ret(lastChunkIndex) = chunks(lastChunkIndex)
        ret(lastChunkIndex).flip()
      } else {
        ret(lastChunkIndex) = allocator(position)
        chunks(lastChunkIndex).flip()
        ret(lastChunkIndex).put(chunks(lastChunkIndex))
        ret(lastChunkIndex).flip()
        StorageUtils.dispose(chunks(lastChunkIndex))
      }
      new ChunkedByteBuffer(ret)
    }
  }
} 

package org.apache.spark.streaming.util

import scala.annotation.tailrec

import java.io.OutputStream
import java.util.concurrent.TimeUnit._

import org.apache.spark.Logging


private[streaming]
class RateLimitedOutputStream(out: OutputStream, desiredBytesPerSec: Int)
  extends OutputStream
  with Logging {

  require(desiredBytesPerSec > 0)

  private val SYNC_INTERVAL = NANOSECONDS.convert(10, SECONDS)
  private val CHUNK_SIZE = 8192
  private var lastSyncTime = System.nanoTime
  private var bytesWrittenSinceSync = 0L

  override def write(b: Int) {
    waitToWrite(1)
    out.write(b)
  }

  override def write(bytes: Array[Byte]) {
    write(bytes, 0, bytes.length)
  }

  @tailrec
  override final def write(bytes: Array[Byte], offset: Int, length: Int) {
    val writeSize = math.min(length - offset, CHUNK_SIZE)
    if (writeSize > 0) {
      waitToWrite(writeSize)
      out.write(bytes, offset, writeSize)
      write(bytes, offset + writeSize, length)
    }
  }

  override def flush() {
    out.flush()
  }

  override def close() {
    out.close()
  }

  @tailrec
  private def waitToWrite(numBytes: Int) {
    val now = System.nanoTime
    val elapsedNanosecs = math.max(now - lastSyncTime, 1)
    val rate = bytesWrittenSinceSync.toDouble * 1000000000 / elapsedNanosecs
    if (rate < desiredBytesPerSec) {
      // It's okay to write; just update some variables and return
      bytesWrittenSinceSync += numBytes
      if (now > lastSyncTime + SYNC_INTERVAL) {
        // Sync interval has passed; let's resync
        lastSyncTime = now
        bytesWrittenSinceSync = numBytes
      }
    } else {
      // Calculate how much time we should sleep to bring ourselves to the desired rate.
      val targetTimeInMillis = bytesWrittenSinceSync * 1000 / desiredBytesPerSec
      val elapsedTimeInMillis = elapsedNanosecs / 1000000
      val sleepTimeInMillis = targetTimeInMillis - elapsedTimeInMillis
      if (sleepTimeInMillis > 0) {
        logTrace("Natural rate is " + rate + " per second but desired rate is " +
          desiredBytesPerSec + ", sleeping for " + sleepTimeInMillis + " ms to compensate.")
        Thread.sleep(sleepTimeInMillis)
      }
      waitToWrite(numBytes)
    }
  }
} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.lang.annotation.Annotation
import java.lang.reflect.Type
import java.text.SimpleDateFormat
import java.util.{Calendar, SimpleTimeZone}
import javax.ws.rs.Produces
import javax.ws.rs.core.{MediaType, MultivaluedMap}
import javax.ws.rs.ext.{MessageBodyWriter, Provider}

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}


@Provider
@Produces(Array(MediaType.APPLICATION_JSON))
private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{

  val mapper = new ObjectMapper() {
    override def writeValueAsString(t: Any): String = {
      super.writeValueAsString(t)
    }
  }
  mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
  mapper.enable(SerializationFeature.INDENT_OUTPUT)
  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)

  override def isWriteable(
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Boolean = {
      true
  }

  override def writeTo(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType,
      multivaluedMap: MultivaluedMap[String, AnyRef],
      outputStream: OutputStream): Unit = {
    t match {
      case ErrorWrapper(err) => outputStream.write(err.getBytes("utf-8"))
      case _ => mapper.writeValue(outputStream, t)
    }
  }

  override def getSize(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Long = {
    -1L
  }
}

private[spark] object JacksonMessageWriter {
  def makeISODateFormat: SimpleDateFormat = {
    val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'")
    val cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"))
    iso8601.setCalendar(cal)
    iso8601
  }
} 

package org.apache.spark.util.io

import java.io.OutputStream

import scala.collection.mutable.ArrayBuffer



  private var position = chunkSize

  override def write(b: Int): Unit = {
    allocateNewChunkIfNeeded()
    chunks(lastChunkIndex)(position) = b.toByte
    position += 1
  }

  override def write(bytes: Array[Byte], off: Int, len: Int): Unit = {
    var written = 0
    while (written < len) {
      allocateNewChunkIfNeeded()
      val thisBatch = math.min(chunkSize - position, len - written)
      System.arraycopy(bytes, written + off, chunks(lastChunkIndex), position, thisBatch)
      written += thisBatch
      position += thisBatch
    }
  }

  @inline
  private def allocateNewChunkIfNeeded(): Unit = {
    if (position == chunkSize) {
      chunks += new Array[Byte](chunkSize)
      lastChunkIndex += 1
      position = 0
    }
  }

  def toArrays: Array[Array[Byte]] = {
    if (lastChunkIndex == -1) {
      new Array[Array[Byte]](0)
    } else {
      // Copy the first n-1 chunks to the output, and then create an array that fits the last chunk.
      // An alternative would have been returning an array of ByteBuffers, with the last buffer
      // bounded to only the last chunk's position. However, given our use case in Spark (to put
      // the chunks in block manager), only limiting the view bound of the buffer would still
      // require the block manager to store the whole chunk.
      val ret = new Array[Array[Byte]](chunks.size)
      for (i <- 0 until chunks.size - 1) {
        ret(i) = chunks(i)
      }
      if (position == chunkSize) {
        ret(lastChunkIndex) = chunks(lastChunkIndex)
      } else {
        ret(lastChunkIndex) = new Array[Byte](position)
        System.arraycopy(chunks(lastChunkIndex), 0, ret(lastChunkIndex), 0, position)
      }
      ret
    }
  }
} 

package org.scaladebugger.api.utils
import java.io.{InputStream, OutputStream}
import java.util.concurrent.TimeUnit

import scala.util.Try


  @throws[InterruptedException]
  @throws[IllegalArgumentException]
  def waitForLimit(
    timeout: Long,
    unit: TimeUnit = TimeUnit.MILLISECONDS
  ): Boolean = {
    require(unit != null)

    val startTime = System.nanoTime()

    // Calculate time remaining
    val rem = () => unit.toNanos(timeout) - (System.nanoTime() - startTime)

    // While still time remaining, poll the exit value status
    var result: Try[Int] = null
    do {
      result = Try(exitValue())

      // Sleep either 100 milliseconds or the remainder of the available time
      Thread.sleep(Math.min(TimeUnit.NANOSECONDS.toMillis(rem()), 100))
    } while (rem() > 0 && result.isFailure)

    result.isSuccess
  }

  override def exitValue(): Int = process.exitValue()
  override def destroy(): Unit = process.destroy()
  override def waitFor(): Int = process.waitFor()
  override def getOutputStream: OutputStream = process.getOutputStream
  override def getErrorStream: InputStream = process.getErrorStream
  override def getInputStream: InputStream = process.getInputStream
} 

package com.ebay.rtran.maven.report

import java.io.{File, OutputStream}
import java.net.URI
import java.util.Optional

import ch.qos.logback.classic.spi.ILoggingEvent
import com.ebay.rtran.report.api.IReportEventSubscriber

import scala.compat.java8.OptionConverters._
import scala.util.Try


class ArtifactsSummarySubscriber(projectRoot: File) extends IReportEventSubscriber[(URI, String)] {

  private[this] var artifacts = Set.empty[(URI, String)]

  override def filter(event: scala.Any): Optional[(URI, String)] = {
    val artifact = event match {
      case e: ILoggingEvent =>
        e.getMessage match {
          case "Found maven pom {} for artifact {}" =>
            val args = e.getArgumentArray
            Try((projectRoot.toURI.relativize(args(0).asInstanceOf[File].toURI), args(1).toString)).toOption
          case _ => None
        }
      case _ => None
    }

    artifact.asJava
  }

  override def dumpTo(outputStream: OutputStream): Unit = if (artifacts.nonEmpty) {
    val outputTemplate = s"\r### Artifacts\n" +
      s"This upgrade request processed only the ${artifacts.size} Maven project artifacts\n" +
      s"that were referenced directly or indirectly by the project's parent POM:\n\n" +
      s"| No. | POM file | Artifact ID |\n" +
      s"| --- | -------- | ----------- |\n"
    var pomCount = 0
    val content = artifacts.foldLeft(outputTemplate) {(c, artifact) =>
      pomCount += 1
      c + s"| $pomCount | [${artifact._1}](${artifact._1}) | ${artifact._2} |\n"
    }
    outputStream.write(content.getBytes("utf8"))
  }

  override def doAccept(event: (URI, String)): Unit = if (event._1.toString != "pom.xml") artifacts += event

  override val sequence = 1
} 

package com.ebay.rtran.maven.report

import java.io.{File, OutputStream}
import java.net.URI
import java.util.Optional

import ch.qos.logback.classic.spi.ILoggingEvent
import com.ebay.rtran.report.api.IReportEventSubscriber

import scala.compat.java8.OptionConverters._
import scala.util.Try


class MavenDependenciesMappingSubscriber(projectRoot: File) extends IReportEventSubscriber[DependencyMappingEvent] {

  private[this] var details = Map.empty[URI, List[DependencyMappingEvent]]

  override def filter(event: scala.Any): Optional[DependencyMappingEvent] = {
    val artifact = event match {
      case e: ILoggingEvent =>
        if (e.getLoggerName.endsWith("MavenDependenciesMappingRule") && e.getMessage == "{} mapped {} to {} in {}") {
          val args = e.getArgumentArray
          Try(DependencyMappingEvent(
            args(1).asInstanceOf[Set[_]].map(_.toString),
            args(2).asInstanceOf[Set[_]].map(_.toString),
            args(3).asInstanceOf[File]
          )).toOption
        } else None
      case _ => None
    }

    artifact.asJava
  }

  override def dumpTo(outputStream: OutputStream): Unit = if (details.nonEmpty) {
    val outputTemplate =
      """
        |### MavenDependenciesMappingRule
        |The following groups of artifacts were mapped to new ones.
      """.stripMargin
    val content = details.foldLeft(outputTemplate) {(c, detail) =>
      val header = s"\n#### File [${detail._1}](${detail._1})\n|from|to|\n|----|---|\n"
      val body = detail._2.foldLeft(header) {(b, event) =>
        val from = event.from.foldLeft("<ul>") {(f, x) =>
          f + s"<li>$x</li>"
        } + "</ul>"
        val to = event.to.foldLeft("<ul>") {(f, x) =>
          f + s"<li>$x</li>"
        } + "</ul>"
        b + s"| $from | $to |\n"
      }
      c + body
    }
    outputStream.write(content.getBytes("utf8"))
  }

  override def doAccept(event: DependencyMappingEvent): Unit = {
    val relativePomPath = projectRoot.toURI relativize event.pomFile.toURI
    details get relativePomPath match {
      case Some(list) => details += relativePomPath -> (event :: list)
      case None => details += relativePomPath -> List(event)
    }
  }
}

case class DependencyMappingEvent(from: Set[String], to: Set[String], pomFile: File) 

package com.ebay.rtran.maven.report

import java.io.{File, OutputStream}
import java.net.URI
import java.util.Optional

import ch.qos.logback.classic.spi.ILoggingEvent
import com.ebay.rtran.report.api.IReportEventSubscriber

import scala.compat.java8.OptionConverters._
import scala.util.Try


class MavenRemoveDependenciesSubscriber(projectRoot: File) extends IReportEventSubscriber[RemoveDependencyEvent] {

  private[this] var details = Map.empty[URI, List[String]]

  override def filter(event: scala.Any): Optional[RemoveDependencyEvent] = {
    val removeEvent = event match {
      case e: ILoggingEvent =>
        if (e.getLoggerName.endsWith("MavenRemoveDependenciesRule") && e.getMessage == "{} removed dependency {} from {}") {
          val args = e.getArgumentArray
          Try(RemoveDependencyEvent(args(1).toString, args(2).asInstanceOf[File])).toOption
        } else None
      case _ => None
    }

    removeEvent.asJava
  }

  override def dumpTo(outputStream: OutputStream): Unit = if (details.nonEmpty) {
    val outputTemplate =
      """
        |### MavenRemoveDependenciesRule
        |The following artifacts were removed from the POM:
      """.stripMargin
    val content = details.foldLeft(outputTemplate) {(c, detail) =>
      val header = s"\n#### File [${detail._1}](${detail._1})\n|Artifacts|\n|---------|\n"
      c + detail._2.foldLeft(header) {(result, artifact) =>
        result + s"|$artifact|\n"
      }
    }
    outputStream.write(content.getBytes("utf8"))
  }

  override def doAccept(event: RemoveDependencyEvent): Unit = {
    val relativePomPath = projectRoot.toURI relativize event.pomFile.toURI
    details get relativePomPath match {
      case Some(list) => details += relativePomPath -> (event.dependency :: list)
      case None => details += relativePomPath -> List(event.dependency)
    }
  }
}

case class RemoveDependencyEvent(dependency: String, pomFile: File) 

package com.ebay.rtran.maven.report

import java.io.{File, OutputStream}
import java.net.URI
import java.util.Optional

import ch.qos.logback.classic.spi.ILoggingEvent
import com.ebay.rtran.report.api.IReportEventSubscriber

import scala.compat.java8.OptionConverters._
import scala.util.Try


class MavenExcludeDependenciesSubscriber(projectRoot: File) extends IReportEventSubscriber[ExcludeDependencyEvent] {

  private[this] var events = Map.empty[URI, Set[ExcludeDependencyEvent]]

  override def filter(event: scala.Any): Optional[ExcludeDependencyEvent] = {
    val excludeEvent = event match {
      case e: ILoggingEvent =>
        if (e.getLoggerName.endsWith("MavenExcludeDependenciesRule") && e.getMessage == "{} excluded {} from {} in {}") {
          val args = e.getArgumentArray
          Try(ExcludeDependencyEvent(
            args(1).asInstanceOf[Set[_]].map(_.toString),
            args(2).toString,
            args(3).asInstanceOf[File]
          )).toOption
        } else None
      case _ => None
    }

    excludeEvent.asJava
  }

  override def dumpTo(outputStream: OutputStream): Unit = if (events.nonEmpty) {
    val outputTemplate =
      """
        |### MavenExcludeDependenciesRule
        |The following artifacts were excluded:
      """.stripMargin
    val content = events.foldLeft(outputTemplate) {(c, event) =>
      val header = s"\n#### File [${event._1}](${event._1})\n|Artifact|Exclusions|\n|-------|------|\n"
      c + header + event._2.map(e => e.dep -> e.exclusions).toMap.foldLeft("") {(result, entry) =>
        result + s"|${entry._1}|" + entry._2.foldLeft("<ul>")(_ + "<li>" + _ + "</li>") + "</ul>|\n"
      }
    }
    outputStream.write(content.getBytes("utf8"))
  }

  override def doAccept(event: ExcludeDependencyEvent): Unit = {
    val relativePomPath = projectRoot.toURI relativize event.pomFile.toURI
    events get relativePomPath match {
      case Some(set) => events += relativePomPath -> (set + event)
      case None => events += relativePomPath -> Set(event)
    }
  }
}

case class ExcludeDependencyEvent(exclusions: Set[String], dep: String, pomFile: File) 

package com.ebay.rtran.maven.report

import java.io.{File, OutputStream}
import java.net.URI
import java.util.Optional

import ch.qos.logback.classic.spi.ILoggingEvent
import com.ebay.rtran.report.api.IReportEventSubscriber

import scala.compat.java8.OptionConverters._
import scala.util.Try


class MavenAddDependenciesSubscriber(projectRoot: File) extends IReportEventSubscriber[AddDependencyEvent] {

  private[this] var details = Map.empty[URI, List[String]]

  override def filter(event: scala.Any): Optional[AddDependencyEvent] = {
    val artifact = event match {
      case e: ILoggingEvent =>
        if (e.getLoggerName.endsWith("MavenAddDependenciesRule") && e.getMessage == "{} added dependency {} to {}") {
          val args = e.getArgumentArray
          Try(AddDependencyEvent(args(1).toString, args(2).asInstanceOf[File])).toOption
        } else None
      case _ => None
    }

    artifact.asJava
  }

  override def dumpTo(outputStream: OutputStream): Unit = if (details.nonEmpty) {
    val outputTemplate =
      """
        |### MavenAddDependenciesRule
        |The following artifacts were added to the POM:
      """.stripMargin
    val content = details.foldLeft(outputTemplate) {(c, detail) =>
     val header = s"\n#### File [${detail._1}](${detail._1})\n|Artifacts|\n|---------|\n"
     c + detail._2.foldLeft(header) {(result, artifact) =>
       result + s"|$artifact|\n"
     }
    }
    outputStream.write(content.getBytes("utf8"))
  }

  override def doAccept(event: AddDependencyEvent): Unit = {
    val relativePomPath = projectRoot.toURI relativize event.pomFile.toURI
    details get relativePomPath match {
      case Some(list) => details += relativePomPath -> (event.dependency :: list)
      case None => details += relativePomPath -> List(event.dependency)
    }
  }
}

case class AddDependencyEvent(dependency: String, pomFile: File) 

package com.ebay.rtran.maven.report

import java.io.{File, OutputStream}
import java.net.URI
import java.util.Optional

import ch.qos.logback.classic.spi.ILoggingEvent
import com.ebay.rtran.report.api.IReportEventSubscriber

import scala.compat.java8.OptionConverters._
import scala.util.Try


class MavenAddManagedDependenciesSubscriber(projectRoot: File)
  extends IReportEventSubscriber[AddManagedDependencyEvent] {

  private[this] var details = Map.empty[URI, List[String]]

  override def filter(event: scala.Any): Optional[AddManagedDependencyEvent] = {
    val artifact = event match {
      case e: ILoggingEvent =>
        if (e.getLoggerName.endsWith("MavenAddManagedDependenciesRule")
          && e.getMessage == "{} added managed dependency {} to {}") {
          val args = e.getArgumentArray
          Try(AddManagedDependencyEvent(args(1).toString, args(2).asInstanceOf[File])).toOption
        } else None
      case _ => None
    }

    artifact.asJava
  }

  override def dumpTo(outputStream: OutputStream): Unit = if (details.nonEmpty) {
    val outputTemplate =
      """
        |### MavenAddManagedDependenciesRule
        |The following artifacts were added to dependencyManagement of the POM:
      """.stripMargin
    val content = details.foldLeft(outputTemplate) {(c, detail) =>
     val header = s"\n#### File [${detail._1}](${detail._1})\n|Artifacts|\n|---------|\n"
     c + detail._2.foldLeft(header) {(result, artifact) =>
       result + s"|$artifact|\n"
     }
    }
    outputStream.write(content.getBytes("utf8"))
  }

  override def doAccept(event: AddManagedDependencyEvent): Unit = {
    val relativePomPath = projectRoot.toURI relativize event.pomFile.toURI
    details get relativePomPath match {
      case Some(list) => details += relativePomPath -> (event.dependency :: list)
      case None => details += relativePomPath -> List(event.dependency)
    }
  }
}

case class AddManagedDependencyEvent(dependency: String, pomFile: File) 

package com.ebay.rtran.report

import java.io.OutputStream

import ch.qos.logback.classic
import ch.qos.logback.classic.LoggerContext
import com.ebay.rtran.report.api.IReportEventSubscriber
import org.slf4j.{Logger, LoggerFactory}


object Report {

  
  def createReport[T](outputStream: OutputStream,
                      loggerNames: Set[String] = Set.empty,
                      subscribers: List[IReportEventSubscriber[_]] = List.empty)
                     (fn: => T): T = {
    val lc = LoggerFactory.getILoggerFactory.asInstanceOf[classic.LoggerContext]
    val reportAppender = prepareReportAppender(lc, loggerNames)
    reportAppender.startWithSubscribers(subscribers)
    val rootLogger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME).asInstanceOf[classic.Logger]
    rootLogger.addAppender(reportAppender)
    val result = fn
    reportAppender.stopAndSave(outputStream)
    rootLogger.detachAppender(reportAppender)
    result
  }

  private def prepareReportAppender(lc: LoggerContext, loggerNames: Set[String]): ReportEventPublisher = {
    val reportAppender = new ReportEventPublisher
    reportAppender.setContext(lc)
    reportAppender.addFilter(new SameThreadFilter)
    reportAppender.addFilter(new LoggerNameFilter(loggerNames))
    reportAppender
  }

} 

package com.ebay.rtran.report.impl;

import java.io.OutputStream
import java.util.Optional

import ch.qos.logback.classic.spi.ILoggingEvent
import com.ebay.rtran.report.api.IReportEventSubscriber

import scala.compat.java8.OptionConverters._



class ProjectDetailsSubscriber extends IReportEventSubscriber[ProjectDetails]{

  private[this] var projectDetails: Option[ProjectDetails] = None

  override def filter(event: scala.Any): Optional[ProjectDetails] = {
    val details = event match {
      case event1: ILoggingEvent =>
        // Starting upgrade {} project to {}, pom {}
        val NoTaskId = """Starting upgrade (.*) project to (.*), pom (.*) with taskId None""".r
        val HasTaskId = """Starting upgrade (.*) project to (.*), pom (.*) with taskId Some\((.*)\)""".r

        event1.getFormattedMessage match {
          case HasTaskId(stack, targetVersion, pomPath, id) => Some(ProjectDetails(pomPath, stack, targetVersion, Some(id)))
          case NoTaskId(stack, targetVersion, pomPath) => Some(ProjectDetails(pomPath, stack, targetVersion, None))
          case _ => None
        }
      case _ => None
    }
    details.asJava
  }

  override def dumpTo(outputStream: OutputStream): Unit = projectDetails match {
    case Some(ProjectDetails(pathToPom, stack, targetVersion, taskId)) =>
      outputStream.write(outputTemplate(pathToPom, stack, targetVersion, taskId).getBytes("utf8"))
    case None =>
  }

  private def outputTemplate(pathToPom: String,stack: String, targetVersion: String,  taskId: Option[String]) = {
    s"""
       |# $stack project upgrade report
       |## Project details
       |Name | Description
       |---- | -----------
       |Path to project POM |	$pathToPom
       |Target Version |	$targetVersion
       |Upgrade job ID | $taskId
       |Full upgrade log | [link](raptor-upgrade-debug${taskId.map("-" + _) getOrElse ""}.log)
       |Upgrade warnings only log | [link](raptor-upgrade-warn${taskId.map("-" + _) getOrElse ""}.log)
       |
     """.stripMargin
  }

  override def doAccept(event: ProjectDetails): Unit = projectDetails = Some(event)

  override val sequence = 0
}

case class ProjectDetails(pathToPom: String, stack: String, targetVersion: String, taskId: Option[String]) 

package com.ebay.rtran.report.impl

import java.io.OutputStream
import java.util.Optional

import ch.qos.logback.classic.spi.ILoggingEvent
import com.ebay.rtran.report.api.IReportEventSubscriber

import scala.compat.java8.OptionConverters._


class ManualChangesSummarySubscriber extends IReportEventSubscriber[(String, Int)] {

  private[this] var manualChanges = Map.empty[String, Int]

  override def filter(event: scala.Any): Optional[(String, Int)] = {
    val Regex = """Rule (.*) requires (\d+) manual changes""".r
    val info = event match {
      case event1: ILoggingEvent =>
        event1.getFormattedMessage match {
          case Regex(rule, num) => Some((rule, num.toInt))
          case _ => None
        }
      case _ => None
    }

    info.asJava
  }

  override def dumpTo(outputStream: OutputStream): Unit = if (manualChanges.nonEmpty) {
    val outputTemplate = "\r## Manual Changes Required\n\n| Rule | Details |\n| ---- | ----------- |\n"
    val content = manualChanges.foldLeft(outputTemplate) {(c, summary) =>
      c + s"|[${summary._1}](#${summary._1.split("\\.").lastOption getOrElse ""}) | ${summary._2} manual changes required |\n"
    }
    outputStream.write(content.getBytes("utf8"))
  }

  override def doAccept(event: (String, Int)): Unit = manualChanges get event._1 match {
    case Some(num) => manualChanges += event._1 -> (event._2 + num)
    case None => manualChanges += event._1 -> event._2
  }

  override val sequence = 4
} 

package com.ebay.rtran.report.impl

import java.io.OutputStream
import java.util.Optional

import ch.qos.logback.classic.spi.ILoggingEvent
import com.ebay.rtran.report.api.IReportEventSubscriber

import scala.compat.java8.OptionConverters._


class UpgradeSummarySubscriber extends IReportEventSubscriber[(String, Int)] {

  private[this] var ruleSummary = Map.empty[String, Int]

  override def filter(event: scala.Any): Optional[(String, Int)] = {
    val Regex = """Rule (.+) was applied to (\d+).*""".r
    val info = event match {
      case event1: ILoggingEvent =>
        event1.getFormattedMessage match {
          case Regex(rule, num) => Some((rule, num.toInt))
          case _ => None
        }

      case _ => None
    }

    info.asJava
  }

  override def dumpTo(outputStream: OutputStream): Unit = if (ruleSummary.nonEmpty) {
    val outputTemplate = "\r## Summary\n\n| Operation | Details |\n| ---- | ----------- |\n"
    val content = ruleSummary.foldLeft(outputTemplate) {(c, summary) =>
      c + s"|[${summary._1}](#${summary._1.split("\\.").lastOption getOrElse ""}) | impacted ${summary._2} file(s) |\n"
    }
    outputStream.write(content.getBytes("utf8"))
  }

  override def doAccept(event: (String, Int)): Unit = ruleSummary get event._1 match {
    case Some(num) => ruleSummary += event._1 -> (event._2 + num)
    case None => ruleSummary += event._1 -> event._2
  }

  override val sequence = 3
} 

package com.ebay.rtran.report

import java.io.OutputStream

import ch.qos.logback.classic.spi.ILoggingEvent
import ch.qos.logback.core.AppenderBase
import com.ebay.rtran.report.api.IReportEventSubscriber


class ReportEventPublisher extends AppenderBase[ILoggingEvent] {

  private[this] var _subscribers = List.empty[IReportEventSubscriber[_]]

  setName("ReportEventPublisher")

  override def append(eventObject: ILoggingEvent): Unit = {
    _subscribers foreach {_.accept(eventObject)}
  }

  def startWithSubscribers(subscribers: List[IReportEventSubscriber[_]]): Unit = {
    _subscribers = subscribers
    super.start()
  }

  def stopAndSave(outputStream: OutputStream): Unit = {
    _subscribers foreach (_ dumpTo outputStream)
    outputStream.flush()
    outputStream.close()
    super.stop()
  }
} 

package com.ebay.rtran.report

import java.io.{File, FileOutputStream, OutputStream}
import java.util.Optional
import com.typesafe.scalalogging.LazyLogging
import com.ebay.rtran.report.api.IReportEventSubscriber
import org.scalatest.{FlatSpecLike, Matchers}


class ReportTest extends FlatSpecLike with Matchers with LazyLogging {

  "Report" should "accept log event correctly" in {
    val subscriber = new TestSubscriber
    val outputStream = new FileOutputStream("test")
    Report.createReport(outputStream, subscribers = List(subscriber))(testFunc())
    subscriber.getCount should be (3)
    new File("test").delete
  }

  "Report" should "work in concurrent mode" in {
    val subscribers = (1 to 10) map (_ => new TestSubscriber)

    subscribers foreach {sub =>
      new Thread(new TestThread(sub)).start()
    }

    val waitPeriod = 1000

    Thread.sleep(waitPeriod)

    subscribers foreach (_.getCount should be (3))
  }

  class TestThread(subscriber: IReportEventSubscriber[_]) extends Runnable {
    override def run(): Unit = {
      val file = new File(s"test-${System.nanoTime}")
      val outputStream = new FileOutputStream(file)
      Report.createReport(outputStream, subscribers = List(subscriber))(testFunc())
      file.delete
    }
  }

  def testFunc(): Unit = {
    val str = "hello"
    val number = 2000
    logger.info("String {}", str)
    logger.info(s"number ${number + 2}")
    logger.info("String {} number {}", str, number.toString)
  }

  class TestSubscriber extends IReportEventSubscriber[Int] {
    import scala.compat.java8.OptionConverters._

    private var count = 0

    def getCount = count

    override def filter(event: scala.Any): Optional[Int] = Some(1).asJava

    override def dumpTo(outputStream: OutputStream): Unit = {}

    override def doAccept(event: Int): Unit = count += event
  }

} 

package org.apache.spark.streaming.util

import scala.annotation.tailrec

import java.io.OutputStream
import java.util.concurrent.TimeUnit._

import org.apache.spark.Logging


private[streaming]
class RateLimitedOutputStream(out: OutputStream, desiredBytesPerSec: Int)
  extends OutputStream
  with Logging {

  require(desiredBytesPerSec > 0)

  private val SYNC_INTERVAL = NANOSECONDS.convert(10, SECONDS)
  private val CHUNK_SIZE = 8192
  private var lastSyncTime = System.nanoTime
  private var bytesWrittenSinceSync = 0L

  override def write(b: Int) {
    waitToWrite(1)
    out.write(b)
  }

  override def write(bytes: Array[Byte]) {
    write(bytes, 0, bytes.length)
  }

  @tailrec
  override final def write(bytes: Array[Byte], offset: Int, length: Int) {
    val writeSize = math.min(length - offset, CHUNK_SIZE)
    if (writeSize > 0) {
      waitToWrite(writeSize)
      out.write(bytes, offset, writeSize)
      write(bytes, offset + writeSize, length)
    }
  }

  override def flush() {
    out.flush()
  }

  override def close() {
    out.close()
  }

  @tailrec
  private def waitToWrite(numBytes: Int) {
    val now = System.nanoTime
    val elapsedNanosecs = math.max(now - lastSyncTime, 1)
    val rate = bytesWrittenSinceSync.toDouble * 1000000000 / elapsedNanosecs
    if (rate < desiredBytesPerSec) {
      // It's okay to write; just update some variables and return
      bytesWrittenSinceSync += numBytes
      if (now > lastSyncTime + SYNC_INTERVAL) {
        // Sync interval has passed; let's resync
        lastSyncTime = now
        bytesWrittenSinceSync = numBytes
      }
    } else {
      // Calculate how much time we should sleep to bring ourselves to the desired rate.
      val targetTimeInMillis = bytesWrittenSinceSync * 1000 / desiredBytesPerSec
      val elapsedTimeInMillis = elapsedNanosecs / 1000000
      val sleepTimeInMillis = targetTimeInMillis - elapsedTimeInMillis
      if (sleepTimeInMillis > 0) {
        logTrace("Natural rate is " + rate + " per second but desired rate is " +
          desiredBytesPerSec + ", sleeping for " + sleepTimeInMillis + " ms to compensate.")
        Thread.sleep(sleepTimeInMillis)
      }
      waitToWrite(numBytes)
    }
  }
} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.lang.annotation.Annotation
import java.lang.reflect.Type
import java.text.SimpleDateFormat
import java.util.{Calendar, SimpleTimeZone}
import javax.ws.rs.Produces
import javax.ws.rs.core.{MediaType, MultivaluedMap}
import javax.ws.rs.ext.{MessageBodyWriter, Provider}

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}


@Provider
@Produces(Array(MediaType.APPLICATION_JSON))
private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{

  val mapper = new ObjectMapper() {
    override def writeValueAsString(t: Any): String = {
      super.writeValueAsString(t)
    }
  }
  mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
  mapper.enable(SerializationFeature.INDENT_OUTPUT)
  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)

  override def isWriteable(
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Boolean = {
      true
  }

  override def writeTo(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType,
      multivaluedMap: MultivaluedMap[String, AnyRef],
      outputStream: OutputStream): Unit = {
    t match {
      case ErrorWrapper(err) => outputStream.write(err.getBytes("utf-8"))
      case _ => mapper.writeValue(outputStream, t)
    }
  }

  override def getSize(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Long = {
    -1L
  }
}

private[spark] object JacksonMessageWriter {
  def makeISODateFormat: SimpleDateFormat = {
    val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'")
    val cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"))
    iso8601.setCalendar(cal)
    iso8601
  }
} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.util.zip.ZipOutputStream
import javax.ws.rs.{GET, Produces}
import javax.ws.rs.core.{MediaType, Response, StreamingOutput}

import scala.util.control.NonFatal

import org.apache.spark.{Logging, SparkConf}
import org.apache.spark.deploy.SparkHadoopUtil

@Produces(Array(MediaType.APPLICATION_OCTET_STREAM))
private[v1] class EventLogDownloadResource(
    val uIRoot: UIRoot,
    val appId: String,
    val attemptId: Option[String]) extends Logging {
  val conf = SparkHadoopUtil.get.newConfiguration(new SparkConf)

  @GET
  def getEventLogs(): Response = {
    try {
      val fileName = {
        attemptId match {
          case Some(id) => s"eventLogs-$appId-$id.zip"
          case None => s"eventLogs-$appId.zip"
        }
      }
      //实现StreamingOutput接口
      val stream = new StreamingOutput {
        override def write(output: OutputStream): Unit = {
          //ZipOutputStream实现打包
          val zipStream = new ZipOutputStream(output)
          try {
            uIRoot.writeEventLogs(appId, attemptId, zipStream)
          } finally {
            zipStream.close()
          }

        }
      }

      Response.ok(stream)
        .header("Content-Disposition", s"attachment; filename=$fileName")
        .header("Content-Type", MediaType.APPLICATION_OCTET_STREAM)
        .build()
    } catch {
      case NonFatal(e) =>
        Response.serverError()
          .entity(s"Event logs are not available for app: $appId.")
          .status(Response.Status.SERVICE_UNAVAILABLE)
          .build()
    }
  }
} 

package org.apache.spark.util.io

import java.io.OutputStream

import scala.collection.mutable.ArrayBuffer



  private var position = chunkSize

  override def write(b: Int): Unit = {
    allocateNewChunkIfNeeded()
    //注意前套数组取值方式
    chunks(lastChunkIndex)(position) = b.toByte
    position += 1
  }

  override def write(bytes: Array[Byte], off: Int, len: Int): Unit = {
    var written = 0
    while (written < len) {
      allocateNewChunkIfNeeded()
      val thisBatch = math.min(chunkSize - position, len - written)
      System.arraycopy(bytes, written + off, chunks(lastChunkIndex), position, thisBatch)
      written += thisBatch
      position += thisBatch
    }
  }

  @inline
  private def allocateNewChunkIfNeeded(): Unit = {
    if (position == chunkSize) {
      chunks += new Array[Byte](chunkSize)
      lastChunkIndex += 1
      position = 0
    }
  }

  def toArrays: Array[Array[Byte]] = {
    if (lastChunkIndex == -1) {
      new Array[Array[Byte]](0)
    } else {
      // Copy the first n-1 chunks to the output, and then create an array that fits the last chunk.
      // An alternative would have been returning an array of ByteBuffers, with the last buffer
      // bounded to only the last chunk's position. However, given our use case in Spark (to put
      // the chunks in block manager), only limiting the view bound of the buffer would still
      // require the block manager to store the whole chunk.
      //将第一个n-1块复制到输出，然后创建一个适合最后一个块的数组。一个替代方法是返回一个ByteBuffers数组，最后一个缓冲区
      //仅限于最后一个块的位置。 但是，考虑到我们在Spark中的用例（put块块中的块管理器），只会限制缓冲区的视图边界
      //要求块管理器存储整个块。
      val ret = new Array[Array[Byte]](chunks.size)
      for (i <- 0 until chunks.size - 1) {
        ret(i) = chunks(i)
      }
      if (position == chunkSize) {
        ret(lastChunkIndex) = chunks(lastChunkIndex)
      } else {
        ret(lastChunkIndex) = new Array[Byte](position)
        System.arraycopy(chunks(lastChunkIndex), 0, ret(lastChunkIndex), 0, position)
      }
      ret
    }
  }
} 

package seed.generation

import java.io.{File, FileInputStream, OutputStream}
import java.util.jar.{Attributes, JarEntry, JarOutputStream, Manifest}

import org.apache.commons.io.IOUtils
import java.nio.file.Path

import seed.Log
import seed.cli.util.Ansi

import scala.collection.mutable

// Adapted from https://stackoverflow.com/a/1281295
object Package {
  def create(
    source: List[(Path, String)],
    target: OutputStream,
    mainClass: Option[String],
    classPath: List[String],
    log: Log
  ): Unit = {
    val manifest       = new Manifest()
    val mainAttributes = manifest.getMainAttributes
    mainAttributes.put(Attributes.Name.MANIFEST_VERSION, "1.0")
    // TODO Set additional package fields: https://docs.oracle.com/javase/tutorial/deployment/jar/packageman.html
    mainClass.foreach(
      cls => mainAttributes.put(Attributes.Name.MAIN_CLASS, cls)
    )
    if (classPath.nonEmpty)
      mainAttributes.put(Attributes.Name.CLASS_PATH, classPath.mkString(" "))

    val targetFile = new JarOutputStream(target, manifest)
    val entryCache = mutable.Set[String]()
    source.foreach {
      case (path, jarPath) =>
        log.debug(s"Packaging ${Ansi.italic(path.toString)}...")
        add(path.toFile, jarPath, targetFile, entryCache, log)
    }
    targetFile.close()
  }

  def add(
    source: File,
    jarPath: String,
    target: JarOutputStream,
    entryCache: mutable.Set[String],
    log: Log
  ): Unit = {
    val path =
      if (source.isFile) jarPath
      else {
        require(!jarPath.endsWith("/"))
        jarPath + "/"
      }

    val addedEntry =
      if (entryCache.contains(path)) {
        if (source.isFile)
          log.warn(
            s"Skipping file ${Ansi.italic(source.toString)} as another module already added it"
          )

        false
      } else {
        val entry = new JarEntry(path)
        entry.setTime(source.lastModified)
        target.putNextEntry(entry)
        entryCache += path
        if (source.isFile) IOUtils.copy(new FileInputStream(source), target)

        true
      }

    if (!source.isFile)
      for (nestedFile <- source.listFiles)
        add(nestedFile, path + nestedFile.getName, target, entryCache, log)

    if (addedEntry) target.closeEntry()
  }
} 

package com.hazelcast.Scala.serialization

import java.io.OutputStream
import java.util.Arrays

private[serialization] object ByteArrayOutputStream {
  private final val MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8;
  private def hugeCapacity(minCapacity: Int): Int = {
    if (minCapacity < 0) // overflow
      throw new OutOfMemoryError();
    if (minCapacity > MAX_ARRAY_SIZE) Integer.MAX_VALUE
    else MAX_ARRAY_SIZE
  }
  private[this] val tlOut = new SoftThreadLocal(new ByteArrayOutputStream)
  def borrow[R](thunk: ByteArrayOutputStream => R): R = {
    tlOut.use { out =>
      out.reset()
      out -> thunk(out)
    }
  }
}
private[serialization] class ByteArrayOutputStream
    extends OutputStream {

  import ByteArrayOutputStream._

  private[this] var buf = new Array[Byte](256)
  private[this] var count: Int = 0

  def withArray[T](thunk: (Array[Byte], Int) => T): T = thunk(buf, count)
  def copyArray: Array[Byte] = Arrays.copyOf(buf, count)
  private def ensureCapacity(minCapacity: Int): Unit = {
    if (minCapacity - buf.length > 0) grow(minCapacity)
  }

  private def grow(minCapacity: Int): Unit = {
    val oldCapacity = buf.length
    var newCapacity = oldCapacity << 1
    if (newCapacity - minCapacity < 0)
      newCapacity = minCapacity
    if (newCapacity - MAX_ARRAY_SIZE > 0)
      newCapacity = hugeCapacity(minCapacity)
    buf = Arrays.copyOf(buf, newCapacity)
  }
  def write(b: Int): Unit = {
    ensureCapacity(count + 1)
    buf(count) = b.asInstanceOf[Byte]
    count += 1
  }

  override def write(b: Array[Byte], off: Int, len: Int): Unit = {
    ensureCapacity(count + len)
    System.arraycopy(b, off, buf, count, len)
    count += len
  }

  def reset(): Unit = {
    count = 0
  }

} 

package com.expedia.www.haystack.trace.commons.packer

import java.io.{ByteArrayInputStream, ByteArrayOutputStream, OutputStream}
import java.util.zip.GZIPOutputStream

import com.expedia.www.haystack.trace.commons.packer.PackerType.PackerType
import com.github.luben.zstd.ZstdOutputStream
import com.google.protobuf.GeneratedMessageV3
import org.apache.commons.io.IOUtils
import org.xerial.snappy.SnappyOutputStream

object PackerType extends Enumeration {
  type PackerType = Value
  val GZIP, SNAPPY, NONE, ZSTD = Value
}

case class PackedMetadata(t: PackerType)

abstract class Packer[T <: GeneratedMessageV3] {
  val packerType: PackerType

  protected def compressStream(stream: OutputStream): OutputStream

  private def pack(protoObj: T): Array[Byte] = {
    val outStream = new ByteArrayOutputStream
    val compressedStream = compressStream(outStream)
    if (compressedStream != null) {
      IOUtils.copy(new ByteArrayInputStream(protoObj.toByteArray), compressedStream)
      compressedStream.close() // this flushes the data to final outStream
      outStream.toByteArray
    } else {
      protoObj.toByteArray
    }
  }

  def apply(protoObj: T): PackedMessage[T] = {
    PackedMessage(protoObj, pack, PackedMetadata(packerType))
  }
}

class NoopPacker[T <: GeneratedMessageV3] extends Packer[T] {
  override val packerType = PackerType.NONE
  override protected def compressStream(stream: OutputStream): OutputStream = null
}

class SnappyPacker[T <: GeneratedMessageV3] extends Packer[T] {
  override val packerType = PackerType.SNAPPY
  override protected def compressStream(stream: OutputStream): OutputStream = new SnappyOutputStream(stream)
}


class ZstdPacker[T <: GeneratedMessageV3] extends Packer[T] {
  override val packerType = PackerType.ZSTD
  override protected def compressStream(stream: OutputStream): OutputStream = new ZstdOutputStream(stream)
}

class GzipPacker[T <: GeneratedMessageV3] extends Packer[T] {
  override val packerType = PackerType.GZIP
  override protected def compressStream(stream: OutputStream): OutputStream = new GZIPOutputStream(stream)
} 

package io.projectglow.sql.util

import java.io.OutputStream

import org.apache.hadoop.io.compress.{CompressionOutputStream, Compressor}
import org.seqdoop.hadoop_bam.util.{DatabricksBGZFOutputStream, BGZFCodec => HBBGZFCodec}


class BGZFCodec extends HBBGZFCodec {
  override def createOutputStream(out: OutputStream): CompressionOutputStream = {
    new DatabricksBGZFOutputStream(out)
  }

  override def createOutputStream(
      out: OutputStream,
      compressor: Compressor): CompressionOutputStream = {
    createOutputStream(out)
  }
} 

package org.apache.toree.kernel.api

import java.io.{InputStream, OutputStream}

import org.apache.toree.kernel.protocol.v5
import org.apache.toree.kernel.protocol.v5.{KMBuilder, KernelMessage}
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5.stream.{KernelOutputStream, KernelInputStream}
import com.typesafe.config.Config


  override def newKernelOutputStream(
    streamType: String = KernelOutputStream.DefaultStreamType,
    sendEmptyOutput: Boolean = config.getBoolean("send_empty_output")
  ): OutputStream = {
    new v5.stream.KernelOutputStream(
      actorLoader,
      kmBuilder,
      org.apache.toree.global.ScheduledTaskManager.instance,
      streamType = streamType,
      sendEmptyOutput = sendEmptyOutput
    )
  }
} 

package org.apache.toree.magic.builtin

import java.io.OutputStream
import java.net.URL

import org.apache.toree.interpreter.Interpreter
import org.apache.toree.magic.dependencies.{IncludeOutputStream, IncludeInterpreter}
import org.apache.toree.magic.{CellMagic, LineMagic}
import org.apache.spark.SparkContext
import org.scalatest.{Matchers, FunSpec}
import org.scalatest.mock.MockitoSugar

import org.mockito.Mockito._
import org.mockito.Matchers._

class TestLSMagic(sc: SparkContext, intp: Interpreter, os: OutputStream)
  extends LSMagic
  with IncludeInterpreter
  with IncludeOutputStream
  {
    override val interpreter: Interpreter = intp
    override val outputStream: OutputStream = os
  }

class LSMagicSpec extends FunSpec with Matchers with MockitoSugar {
  describe("LSMagic") {

    describe("#execute") {
      it("should call println with a magics message") {
        val lsm = spy(new TestLSMagic(
          mock[SparkContext], mock[Interpreter], mock[OutputStream])
        )
        val classList = new BuiltinLoader().loadClasses()
        lsm.execute("")
        verify(lsm).magicNames("%", classOf[LineMagic], classList)
        verify(lsm).magicNames("%%", classOf[CellMagic], classList)
      }
    }

    describe("#magicNames") {
      it("should filter classnames by interface") {
        val prefix = "%"
        val interface = classOf[LineMagic]
        val classes : List[Class[_]] = List(classOf[LSMagic], classOf[Integer])
        val lsm = new TestLSMagic(
          mock[SparkContext], mock[Interpreter], mock[OutputStream])
        lsm.magicNames(prefix, interface, classes).length should be(1)
      }
      it("should prepend prefix to each name"){
        val prefix = "%"
        val className = classOf[LSMagic].getSimpleName
        val interface = classOf[LineMagic]
        val expected = s"${prefix}${className}"
        val classes : List[Class[_]] = List(classOf[LSMagic], classOf[Integer])
        val lsm = new TestLSMagic(
          mock[SparkContext], mock[Interpreter], mock[OutputStream])
        lsm.magicNames(prefix, interface, classes) should be(List(expected))
      }
    }

  }

} 

package org.apache.toree.global

import java.io.{InputStream, OutputStream, PrintStream}


  def withStreams[T](thunk: => T): T = {
    init(_inputStream, _outputStream, _errorStream)

    val returnValue = Console.withIn(_inputStream) {
      Console.withOut(_outputStream) {
        Console.withErr(_errorStream) {
          thunk
        }
      }
    }

    reset()

    returnValue
  }
} 

package org.apache.toree.utils

import joptsimple.{OptionSpec, OptionParser}
import scala.collection.JavaConverters._
import scala.language.implicitConversions
import java.io.{PrintStream, OutputStream}

trait ArgumentParsingSupport {
  protected lazy val parser = new OptionParser()
  private var options: joptsimple.OptionSet = _
  parser.allowsUnrecognizedOptions()

  
  def parseArgs(args: String, delimiter: String = " ") = {
    options = parser.parse(args.split(delimiter): _*)

    options.nonOptionArguments().asScala.map(_.toString)
  }

  def printHelp(outputStream: OutputStream, usage: String) = {
    val printStream = new PrintStream(outputStream)

    printStream.println(s"Usage: $usage\n")
    parser.printHelpOn(outputStream)
  }

  implicit def has[T](spec: OptionSpec[T]): Boolean = {
    require(options != null, "Arguments not parsed yet!")
    options.has(spec)
  }

  implicit def get[T](spec: OptionSpec[T]): Option[T] = {
    require(options != null, "Arguments not parsed yet!")
    Some(options.valueOf(spec)).filter(_ != null)
  }

  // NOTE: Cannot be implicit as conflicts with get
  def getAll[T](spec: OptionSpec[T]): Option[List[T]] = {
    require(options != null, "Arguments not parsed yet!")
    Some(options.valuesOf(spec).asScala.toList).filter(_ != null)
  }
} 

package org.apache.toree.utils

import java.io.OutputStream

import org.scalatest.mock.MockitoSugar
import org.scalatest.{BeforeAndAfter, Matchers, FunSpec}
import org.mockito.Matchers._
import org.mockito.Mockito._

class MultiOutputStreamSpec
  extends FunSpec with Matchers with MockitoSugar with BeforeAndAfter {

  describe("MultiOutputStream") {
    val listOfMockOutputStreams = List(mock[OutputStream], mock[OutputStream])
    val multiOutputStream = MultiOutputStream(listOfMockOutputStreams)

    describe("#close") {
      it("should call #close on all internal output streams") {
        multiOutputStream.close()

        listOfMockOutputStreams.foreach(mockOutputStream => verify(mockOutputStream).close())
      }
    }

    describe("#flush") {
      it("should call #flush on all internal output streams") {
        multiOutputStream.flush()

        listOfMockOutputStreams.foreach(mockOutputStream => verify(mockOutputStream).flush())
      }
    }

    describe("#write(int)") {
      it("should call #write(int) on all internal output streams") {
        multiOutputStream.write(anyInt())

        listOfMockOutputStreams.foreach(
          mockOutputStream => verify(mockOutputStream).write(anyInt()))
      }
    }
    describe("#write(byte[])") {
      it("should call #write(byte[]) on all internal output streams") {
        multiOutputStream.write(any[Array[Byte]])

        listOfMockOutputStreams.foreach(
          mockOutputStream => verify(mockOutputStream).write(any[Array[Byte]]))
      }
    }

    describe("#write(byte[], int, int)") {
      it("should call #write(byte[], int, int) on all internal output streams") {
        multiOutputStream.write(any[Array[Byte]], anyInt(), anyInt())

        listOfMockOutputStreams.foreach(
          mockOutputStream =>
            verify(mockOutputStream).write(any[Array[Byte]], anyInt(), anyInt()))
      }
    }
  }
} 

package org.apache.toree.utils

import java.io.OutputStream

import org.scalatest.mock.MockitoSugar
import org.mockito.Mockito._
import org.mockito.Matchers._
import org.scalatest.{Matchers, FunSpec}

class ConditionalOutputStreamSpec extends FunSpec with Matchers with MockitoSugar {
  describe("ConditionalOutputStream") {
    describe("#()") {
      it("should throw an exception if the output stream is null") {
        intercept[IllegalArgumentException] {
          new ConditionalOutputStream(null, true)
        }
      }
    }

    describe("#write") {
      it("should call the underlying write if the condition is true") {
        val mockOutputStream = mock[OutputStream]
        val conditionalOutputStream =
          new ConditionalOutputStream(mockOutputStream, true)

        val expected = 101
        conditionalOutputStream.write(expected)

        verify(mockOutputStream).write(expected)
      }

      it("should call the underlying write if the condition becomes true") {
        val mockOutputStream = mock[OutputStream]
        var condition = false

        val conditionalOutputStream =
          new ConditionalOutputStream(mockOutputStream, condition)

        condition = true

        val expected = 101
        conditionalOutputStream.write(expected)

        verify(mockOutputStream).write(expected)
      }

      it("should not call the underlying write if the condition is false") {
        val mockOutputStream = mock[OutputStream]
        val conditionalOutputStream =
          new ConditionalOutputStream(mockOutputStream, false)

        val expected = 101
        conditionalOutputStream.write(expected)

        verify(mockOutputStream, never()).write(any[Byte])
      }

      it("should not call the underlying write if the condition becomes false") {
        val mockOutputStream = mock[OutputStream]
        var condition = true

        val conditionalOutputStream =
          new ConditionalOutputStream(mockOutputStream, condition)

        condition = false

        val expected = 101
        conditionalOutputStream.write(expected)

        verify(mockOutputStream, never()).write(any[Byte])
      }
    }
  }
} 

package almond.internals

import java.io.{OutputStream, PrintStream}
import java.nio.{ByteBuffer, CharBuffer}
import java.nio.charset.{Charset, CoderResult}

class FunctionOutputStream(
  inputBufferSize: Int,
  outputBufferSize: Int,
  internalCharset: Charset,
  f: String => Unit
) extends OutputStream {

  // not thread-safe

  private val decoder = internalCharset.newDecoder()

  private val inArray = Array.ofDim[Byte](inputBufferSize)
  private val outArray = Array.ofDim[Char](outputBufferSize)

  private val writeBuf = ByteBuffer.wrap(inArray)
  private val out = CharBuffer.wrap(outArray)

  private def flushIfNeeded(): Unit =
    if (!writeBuf.hasRemaining)
      flush()

  def write(b: Int): Unit = {
    writeBuf.put(b.toByte) // hope toByte doesn't box b
    flushIfNeeded()
  }

  override def write(b: Array[Byte], off: Int, len: Int) = {
    var off0 = off
    var len0 = len
    while (len0 > 0) {
      val take = math.min(len0, writeBuf.remaining())
      assert(take > 0)
      writeBuf.put(b, off0, take)
      off0 = off0 + take
      len0 = len0 - take
      flushIfNeeded()
    }
    assert(len0 == 0)
    assert(off0 == off + len)
  }

  override def flush(): Unit = {
    super.flush()

    val readBuf = ByteBuffer.wrap(inArray, 0, writeBuf.position())

    var r: CoderResult = null

    while (r == null || r.isOverflow) {
      if (r != null) {
        readBuf.position(0)
        readBuf.limit(writeBuf.position())
      }

      r = decoder.decode(readBuf, out, false)

      val outLen = out.position()

      if (r.isError || (r.isOverflow && outLen == 0))
        r.throwException()
      else {
        if (outLen > 0) {
          val s = new String(outArray, 0, outLen)
          out.clear()
          f(s)
        }
        val read = readBuf.position()
        val avail = writeBuf.position()
        val remaining = avail - read
        writeBuf.position(remaining)
        if (remaining > 0)
          System.arraycopy(inArray, read, inArray, 0, remaining)
      }
    }
  }

  def printStream(): PrintStream =
    new PrintStream(this, true, internalCharset.name())

} 

package org.seqdoop.hadoop_bam.util

import java.io.OutputStream

import htsjdk.samtools.util.{BlockCompressedOutputStream, BlockCompressedStreamConstants}
import org.apache.hadoop.io.compress.CompressionOutputStream


class DatabricksBGZFOutputStream(outputStream: OutputStream)
    extends CompressionOutputStream(outputStream) {

  var writeEmptyBlockOnClose: Boolean = false
  private var haveWrittenEmptyBlock = false
  private var output = new BlockCompressedOutputStream(outputStream, null: java.io.File)

  override def write(b: Int): Unit = {
    output.write(b)
  }

  override def write(b: Array[Byte], off: Int, len: Int): Unit = {
    output.write(b, off, len)
  }

  override def finish(): Unit = {
    output.flush()
  }

  override def resetState(): Unit = {
    output.flush()
    output = new BlockCompressedOutputStream(out, null: java.io.File)
  }

  override def close(): Unit = {
    output.flush() // don't close as we don't want to write terminator (empty gzip block)
    if (writeEmptyBlockOnClose && !haveWrittenEmptyBlock) {
      out.write(BlockCompressedStreamConstants.EMPTY_GZIP_BLOCK)
      haveWrittenEmptyBlock = true
    }
    out.close()
  }
}

object DatabricksBGZFOutputStream {
  def setWriteEmptyBlockOnClose(os: OutputStream, value: Boolean): Unit = os match {
    case s: DatabricksBGZFOutputStream => s.writeEmptyBlockOnClose = value
    case _ => // No op
  }
} 

package io.projectglow.vcf

import java.io.OutputStream

import htsjdk.samtools.ValidationStringency
import htsjdk.variant.vcf._
import org.apache.hadoop.conf.Configuration
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.execution.datasources.OutputWriter
import org.apache.spark.sql.types.StructType

import io.projectglow.common.GlowLogging

class VCFFileWriter(
    headerLineSet: Set[VCFHeaderLine],
    sampleIdInfo: SampleIdInfo,
    stringency: ValidationStringency,
    schema: StructType,
    conf: Configuration,
    stream: OutputStream,
    writeHeader: Boolean)
    extends OutputWriter
    with GlowLogging {

  private val converter =
    new InternalRowToVariantContextConverter(schema, headerLineSet, stringency)
  converter.validate()
  private val writer: VCFStreamWriter =
    new VCFStreamWriter(stream, headerLineSet, sampleIdInfo, writeHeader)

  override def write(row: InternalRow): Unit = {
    converter.convert(row).foreach(writer.write)
  }

  override def close(): Unit = {
    writer.close()
  }
} 

package io.projectglow.vcf

import java.io.OutputStream

import scala.collection.JavaConverters._

import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.catalyst.InternalRow

import io.projectglow.common.GlowLogging
import io.projectglow.transformers.pipe.{InputFormatter, InputFormatterFactory}


class VCFInputFormatter(converter: InternalRowToVariantContextConverter, sampleIdInfo: SampleIdInfo)
    extends InputFormatter
    with GlowLogging {

  private var writer: VCFStreamWriter = _
  private var stream: OutputStream = _

  override def init(stream: OutputStream): Unit = {
    this.stream = stream
    this.writer = new VCFStreamWriter(
      stream,
      converter.vcfHeader.getMetaDataInInputOrder.asScala.toSet,
      sampleIdInfo,
      writeHeader = true)
  }

  override def write(record: InternalRow): Unit = {
    converter.convert(record).foreach(writer.write)
  }

  override def close(): Unit = {
    logger.info("Closing VCF input formatter")
    writer.close()
  }
}

class VCFInputFormatterFactory extends InputFormatterFactory {
  override def name: String = "vcf"

  override def makeInputFormatter(df: DataFrame, options: Map[String, String]): InputFormatter = {
    val (headerLineSet, sampleIdInfo) =
      VCFHeaderUtils.parseHeaderLinesAndSamples(
        options,
        None,
        df.schema,
        df.sparkSession.sparkContext.hadoopConfiguration)
    val rowConverter = new InternalRowToVariantContextConverter(
      df.schema,
      headerLineSet,
      VCFOptionParser.getValidationStringency(options)
    )
    rowConverter.validate()

    new VCFInputFormatter(rowConverter, sampleIdInfo)
  }
} 

package io.projectglow.transformers.pipe

import java.io.{OutputStream, PrintWriter}

import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.execution.datasources.csv.SGUnivocityGenerator
import org.apache.spark.sql.types.StructType

import io.projectglow.SparkShim.CSVOptions

class CSVInputFormatter(schema: StructType, parsedOptions: CSVOptions) extends InputFormatter {

  private var writer: PrintWriter = _
  private var univocityGenerator: SGUnivocityGenerator = _

  override def init(stream: OutputStream): Unit = {
    writer = new PrintWriter(stream)
    univocityGenerator = new SGUnivocityGenerator(schema, writer, parsedOptions)
    if (parsedOptions.headerFlag) {
      univocityGenerator.writeHeaders()
    }
  }

  override def write(record: InternalRow): Unit = {
    univocityGenerator.write(record)
  }

  override def close(): Unit = {
    writer.close()
    univocityGenerator.close()
  }
}

class CSVInputFormatterFactory extends InputFormatterFactory {
  override def name: String = "csv"

  override def makeInputFormatter(
      df: DataFrame,
      options: Map[String, String]
  ): InputFormatter = {
    val sqlConf = df.sparkSession.sessionState.conf
    val parsedOptions =
      new CSVOptions(
        options,
        sqlConf.csvColumnPruning,
        sqlConf.sessionLocalTimeZone
      )
    new CSVInputFormatter(df.schema, parsedOptions)
  }
} 

package io.projectglow.transformers.pipe

import java.io.{OutputStream, PrintWriter}

import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.SQLUtils.dataTypesEqualExceptNullability
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.types.StringType


class UTF8TextInputFormatter() extends InputFormatter {

  private var writer: PrintWriter = _

  override def init(stream: OutputStream): Unit = {
    writer = new PrintWriter(stream)
  }

  override def write(record: InternalRow): Unit = {
    if (!record.isNullAt(0)) {
      writer.println(record.getUTF8String(0)) // scalastyle:ignore
    }
  }

  override def close(): Unit = {
    writer.close()
  }
}

class UTF8TextInputFormatterFactory extends InputFormatterFactory {
  override def name: String = "text"

  override def makeInputFormatter(df: DataFrame, options: Map[String, String]): InputFormatter = {
    require(df.schema.length == 1, "Input dataframe must have one column,")
    require(
      dataTypesEqualExceptNullability(df.schema.head.dataType, StringType),
      "Input dataframe must have one string column.")
    new UTF8TextInputFormatter
  }
} 

package org.apache.toree.kernel.protocol.v5.relay

import java.io.OutputStream
import akka.actor.Actor
import akka.pattern._
import akka.util.Timeout
import org.apache.toree.interpreter.{ExecuteAborted, ExecuteError, ExecuteFailure, ExecuteOutput}
import org.apache.toree.kernel.protocol.v5._
import org.apache.toree.kernel.protocol.v5.content._
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.kernel.protocol.v5.magic.MagicParser
import org.apache.toree.plugins.PluginManager
import org.apache.toree.utils.LogLike
import scala.concurrent.Future
import scala.concurrent.duration._
import org.apache.toree.plugins.NewOutputStream

case class ExecuteRequestRelay(
  actorLoader: ActorLoader,
  pluginManager: PluginManager,
  magicParser: MagicParser
)
  extends Actor with LogLike
{
  import context._
  implicit val timeout = Timeout(21474835.seconds)

  
  private def packageFutureResponse(
    future: Future[Either[ExecuteOutput, ExecuteFailure]]
  ): Future[(ExecuteReply, ExecuteResult)] = future.map { value =>
    if (value.isLeft) {
      val data = value.left.get
      (
        ExecuteReplyOk(1, Some(Payloads()), Some(UserExpressions())),
        ExecuteResult(1, data, Metadata())
      )
    } else {
      failureMatch(value.right.get)
    }
  }

  override def receive: Receive = {
    case (executeRequest: ExecuteRequest, parentMessage: KernelMessage,
      outputStream: OutputStream) =>
      val interpreterActor = actorLoader.load(SystemActorType.Interpreter)

      // Store our old sender so we don't lose it in the callback
      // NOTE: Should point back to our KernelMessageRelay
      val oldSender = sender()

      // Sets the outputStream for this particular ExecuteRequest
      import org.apache.toree.plugins.Implicits._
      pluginManager.fireEventFirstResult(
        NewOutputStream,
        "outputStream" -> outputStream
      )

      // Parse the code for magics before sending it to the interpreter and
      // pipe the response to sender
      (magicParser.parse(executeRequest.code) match {
        case Left(code) =>
          val parsedRequest =
            (executeRequest.copy(code = code), parentMessage, outputStream)
          val interpreterFuture = (interpreterActor ? parsedRequest)
            .mapTo[Either[ExecuteOutput, ExecuteFailure]]
          packageFutureResponse(interpreterFuture)

        case Right(error) =>
          val failure = ExecuteError("Error parsing magics!", error, Nil)
          Future { failureMatch(failure) }
      }) pipeTo oldSender
  }
} 

package com.spotify.scio.elasticsearch

import java.io.{InputStream, OutputStream}

import com.spotify.scio.coders._
import org.apache.beam.sdk.coders.{AtomicCoder, CoderException}
import org.elasticsearch.action.DocWriteRequest
import org.elasticsearch.action.delete.DeleteRequest
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.common.io.stream.{
  InputStreamStreamInput,
  OutputStreamStreamOutput,
  StreamInput,
  Writeable
}

trait CoderInstances {
  private val INDEX_REQ_INDEX = 0
  private val UPDATE_REQ_INDEX = 1
  private val DELETE_REQ_INDEX = 2
  private val KRYO_INDEX = 3

  private lazy val kryoCoder = Coder.kryo[DocWriteRequest[_]]
  private lazy val kryoBCoder = CoderMaterializer.beamWithDefault(kryoCoder)

  implicit val docWriteRequestCoder: Coder[DocWriteRequest[_]] =
    Coder.beam[DocWriteRequest[_]](new AtomicCoder[DocWriteRequest[_]] {
      override def encode(value: DocWriteRequest[_], outStream: OutputStream): Unit =
        value match {
          case i: IndexRequest =>
            outStream.write(INDEX_REQ_INDEX)
            indexRequestBCoder.encode(i, outStream)
          case u: UpdateRequest =>
            outStream.write(UPDATE_REQ_INDEX)
            updateRequestBCoder.encode(u, outStream)
          case d: DeleteRequest =>
            outStream.write(DELETE_REQ_INDEX)
            deleteRequestBCoder.encode(d, outStream)
          case _ =>
            outStream.write(KRYO_INDEX)
            kryoBCoder.encode(value, outStream)
        }

      override def decode(inStream: InputStream): DocWriteRequest[_] = {
        val request = inStream.read() match {
          case INDEX_REQ_INDEX  => indexRequestBCoder.decode(inStream)
          case UPDATE_REQ_INDEX => updateRequestBCoder.decode(inStream)
          case DELETE_REQ_INDEX => deleteRequestBCoder.decode(inStream)
          case KRYO_INDEX       => kryoBCoder.decode(inStream)
          case n                => throw new CoderException(s"Unknown index $n")
        }

        request.asInstanceOf[DocWriteRequest[_]]
      }
    })

  private def writableBCoder[T <: Writeable](
    constructor: StreamInput => T
  ): org.apache.beam.sdk.coders.Coder[T] = new AtomicCoder[T] {
    override def encode(value: T, outStream: OutputStream): Unit =
      value.writeTo(new OutputStreamStreamOutput(outStream))
    override def decode(inStream: InputStream): T =
      constructor(new InputStreamStreamInput(inStream))
  }

  private val indexRequestBCoder = writableBCoder[IndexRequest](new IndexRequest(_))
  implicit val indexRequestCoder: Coder[IndexRequest] =
    Coder.beam[IndexRequest](indexRequestBCoder)

  private val deleteRequestBCoder = writableBCoder[DeleteRequest](new DeleteRequest(_))
  implicit val deleteRequestCoder: Coder[DeleteRequest] =
    Coder.beam[DeleteRequest](deleteRequestBCoder)

  private val updateRequestBCoder = writableBCoder[UpdateRequest](new UpdateRequest(_))
  implicit val updateRequestCoder: Coder[UpdateRequest] =
    Coder.beam[UpdateRequest](updateRequestBCoder)
} 

package com.spotify.scio.elasticsearch

import java.io.{InputStream, OutputStream}

import com.spotify.scio.coders._
import org.apache.beam.sdk.coders.{AtomicCoder, CoderException}
import org.elasticsearch.action.DocWriteRequest
import org.elasticsearch.action.delete.DeleteRequest
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.common.io.stream.{
  InputStreamStreamInput,
  OutputStreamStreamOutput,
  Streamable
}

trait CoderInstances {
  private val INDEX_REQ_INDEX = 0
  private val UPDATE_REQ_INDEX = 1
  private val DELETE_REQ_INDEX = 2
  private val KRYO_INDEX = 3

  private lazy val kryoCoder = Coder.kryo[DocWriteRequest[_]]
  private lazy val kryoBCoder = CoderMaterializer.beamWithDefault(kryoCoder)

  implicit val docWriteRequestCoder: Coder[DocWriteRequest[_]] =
    Coder.beam[DocWriteRequest[_]](new AtomicCoder[DocWriteRequest[_]] {
      override def encode(value: DocWriteRequest[_], outStream: OutputStream): Unit =
        value match {
          case i: IndexRequest =>
            outStream.write(INDEX_REQ_INDEX)
            indexRequestBCoder.encode(i, outStream)
          case u: UpdateRequest =>
            outStream.write(UPDATE_REQ_INDEX)
            updateRequestBCoder.encode(u, outStream)
          case d: DeleteRequest =>
            outStream.write(DELETE_REQ_INDEX)
            deleteRequestBCoder.encode(d, outStream)
          case _ =>
            outStream.write(KRYO_INDEX)
            kryoBCoder.encode(value, outStream)
        }

      override def decode(inStream: InputStream): DocWriteRequest[_] = {
        val request = inStream.read() match {
          case INDEX_REQ_INDEX  => indexRequestBCoder.decode(inStream)
          case UPDATE_REQ_INDEX => updateRequestBCoder.decode(inStream)
          case DELETE_REQ_INDEX => deleteRequestBCoder.decode(inStream)
          case KRYO_INDEX       => kryoBCoder.decode(inStream)
          case n                => throw new CoderException(s"Unknown index $n")
        }

        request.asInstanceOf[DocWriteRequest[_]]
      }
    })

  private def writableBCoder[T <: Streamable](
    constructor: () => T
  ): org.apache.beam.sdk.coders.Coder[T] = new AtomicCoder[T] {
    override def encode(value: T, outStream: OutputStream): Unit =
      value.writeTo(new OutputStreamStreamOutput(outStream))
    override def decode(inStream: InputStream): T = {
      val value = constructor()
      value.readFrom(new InputStreamStreamInput(inStream))
      value
    }
  }

  private val indexRequestBCoder = writableBCoder[IndexRequest](() => new IndexRequest())
  implicit val indexRequestCoder: Coder[IndexRequest] =
    Coder.beam[IndexRequest](indexRequestBCoder)

  private val deleteRequestBCoder = writableBCoder[DeleteRequest](() => new DeleteRequest())
  implicit val deleteRequestCoder: Coder[DeleteRequest] =
    Coder.beam[DeleteRequest](deleteRequestBCoder)

  private val updateRequestBCoder = writableBCoder[UpdateRequest](() => new UpdateRequest())
  implicit val updateRequestCoder: Coder[UpdateRequest] =
    Coder.beam[UpdateRequest](updateRequestBCoder)
} 

package com.spotify.scio.elasticsearch

import java.io.{InputStream, OutputStream}

import com.spotify.scio.coders._
import org.apache.beam.sdk.coders.{AtomicCoder, CoderException}
import org.elasticsearch.action.DocWriteRequest
import org.elasticsearch.action.delete.DeleteRequest
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.common.io.stream.{
  InputStreamStreamInput,
  OutputStreamStreamOutput,
  Streamable,
  Writeable
}

trait CoderInstances {
  private val INDEX_REQ_INDEX = 0
  private val UPDATE_REQ_INDEX = 1
  private val DELETE_REQ_INDEX = 2
  private val KRYO_INDEX = 3

  private lazy val kryoCoder = Coder.kryo[DocWriteRequest[_]]
  private lazy val kryoBCoder = CoderMaterializer.beamWithDefault(kryoCoder)

  implicit val docWriteRequestCoder: Coder[DocWriteRequest[_]] =
    Coder.beam[DocWriteRequest[_]](new AtomicCoder[DocWriteRequest[_]] {
      override def encode(value: DocWriteRequest[_], outStream: OutputStream): Unit =
        value match {
          case i: IndexRequest =>
            outStream.write(INDEX_REQ_INDEX)
            indexRequestBCoder.encode(i, outStream)
          case u: UpdateRequest =>
            outStream.write(UPDATE_REQ_INDEX)
            updateRequestBCoder.encode(u, outStream)
          case d: DeleteRequest =>
            outStream.write(DELETE_REQ_INDEX)
            deleteRequestBCoder.encode(d, outStream)
          case _ =>
            outStream.write(KRYO_INDEX)
            kryoBCoder.encode(value, outStream)
        }

      override def decode(inStream: InputStream): DocWriteRequest[_] = {
        val request = inStream.read() match {
          case INDEX_REQ_INDEX  => indexRequestBCoder.decode(inStream)
          case UPDATE_REQ_INDEX => updateRequestBCoder.decode(inStream)
          case DELETE_REQ_INDEX => deleteRequestBCoder.decode(inStream)
          case KRYO_INDEX       => kryoBCoder.decode(inStream)
          case n                => throw new CoderException(s"Unknown index $n")
        }

        request.asInstanceOf[DocWriteRequest[_]]
      }
    })

  private def writableBCoder[T <: Writeable with Streamable](
    constructor: () => T
  ): org.apache.beam.sdk.coders.Coder[T] = new AtomicCoder[T] {
    override def encode(value: T, outStream: OutputStream): Unit =
      value.writeTo(new OutputStreamStreamOutput(outStream))
    override def decode(inStream: InputStream): T = {
      val value = constructor()
      value.readFrom(new InputStreamStreamInput(inStream))
      value
    }
  }

  private val indexRequestBCoder = writableBCoder[IndexRequest](() => new IndexRequest())
  implicit val indexRequestCoder: Coder[IndexRequest] =
    Coder.beam[IndexRequest](indexRequestBCoder)

  private val deleteRequestBCoder = writableBCoder[DeleteRequest](() => new DeleteRequest())
  implicit val deleteRequestCoder: Coder[DeleteRequest] =
    Coder.beam[DeleteRequest](deleteRequestBCoder)

  private val updateRequestBCoder = writableBCoder[UpdateRequest](() => new UpdateRequest())
  implicit val updateRequestCoder: Coder[UpdateRequest] =
    Coder.beam[UpdateRequest](updateRequestBCoder)
} 

package com.spotify.scio.coders.instances

import java.io.{InputStream, OutputStream}

import com.spotify.scio.coders.{AvroCoderMacros, Coder}
import org.apache.avro.Schema
import org.apache.avro.generic.GenericRecord
import org.apache.avro.specific.{SpecificData, SpecificFixed}
import org.apache.beam.sdk.coders.Coder.NonDeterministicException
import org.apache.beam.sdk.coders.{AtomicCoder, AvroCoder, StringUtf8Coder}
import org.apache.beam.sdk.util.common.ElementByteSizeObserver

import scala.reflect.{classTag, ClassTag}

final private class SlowGenericRecordCoder extends AtomicCoder[GenericRecord] {
  // TODO: can we find something more efficient than String ?
  private[this] val sc = StringUtf8Coder.of()

  override def encode(value: GenericRecord, os: OutputStream): Unit = {
    val schema = value.getSchema
    val coder = AvroCoder.of(schema)
    sc.encode(schema.toString, os)
    coder.encode(value, os)
  }

  override def decode(is: InputStream): GenericRecord = {
    val schemaStr = sc.decode(is)
    val schema = new Schema.Parser().parse(schemaStr)
    val coder = AvroCoder.of(schema)
    coder.decode(is)
  }

  // delegate methods for determinism and equality checks
  override def verifyDeterministic(): Unit =
    throw new NonDeterministicException(
      this,
      "Coder[GenericRecord] without schema is non-deterministic"
    )
  override def consistentWithEquals(): Boolean = false
  override def structuralValue(value: GenericRecord): AnyRef =
    AvroCoder.of(value.getSchema).structuralValue(value)

  // delegate methods for byte size estimation
  override def isRegisterByteSizeObserverCheap(value: GenericRecord): Boolean =
    AvroCoder.of(value.getSchema).isRegisterByteSizeObserverCheap(value)
  override def registerByteSizeObserver(
    value: GenericRecord,
    observer: ElementByteSizeObserver
  ): Unit =
    AvroCoder.of(value.getSchema).registerByteSizeObserver(value, observer)
}


  // TODO: Use a coder that does not serialize the schema
  def avroGenericRecordCoder(schema: Schema): Coder[GenericRecord] =
    Coder.beam(AvroCoder.of(schema))

  // XXX: similar to GenericAvroSerializer
  def avroGenericRecordCoder: Coder[GenericRecord] =
    Coder.beam(new SlowGenericRecordCoder)

  import org.apache.avro.specific.SpecificRecordBase
  implicit def genAvro[T <: SpecificRecordBase]: Coder[T] =
    macro AvroCoderMacros.staticInvokeCoder[T]

  implicit def avroSpecificFixedCoder[T <: SpecificFixed: ClassTag]: Coder[T] =
    SpecificFixedCoder[T]
} 

package com.spotify.scio.testing

import java.io.{InputStream, OutputStream}

import com.spotify.scio.coders.Coder
import com.spotify.scio.testing.CoderAssertions._
import org.apache.beam.sdk.coders.{AtomicCoder, StringUtf8Coder}
import org.scalatest.exceptions.TestFailedException
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

case class Foo(id: String)

class CoderAssertionsTest extends AnyFlatSpec with Matchers {
  // A coder which roundtrips incorrectly
  private def incorrectCoder: Coder[Foo] =
    Coder.beam(new AtomicCoder[Foo] {
      override def encode(value: Foo, outStream: OutputStream): Unit =
        StringUtf8Coder.of().encode(value.id, outStream)
      override def decode(inStream: InputStream): Foo =
        Foo(StringUtf8Coder.of().decode(inStream) + "wrongBytes")
    })

  "CoderAssertions" should "support roundtrip" in {
    Foo("bar") coderShould roundtrip()

    an[TestFailedException] should be thrownBy {
      implicit def coder: Coder[Foo] = incorrectCoder

      Foo("baz") coderShould roundtrip()
    }
  }

  it should "support fallback" in {
    val str = "boom"
    val cs: java.lang.CharSequence = str
    cs coderShould fallback()

    an[TestFailedException] should be thrownBy {
      str coderShould fallback()
    }
  }

  it should "support notFallback" in {
    val str = "boom"
    str coderShould notFallback()

    an[TestFailedException] should be thrownBy {
      val cs: java.lang.CharSequence = str
      cs coderShould notFallback()
    }
  }

  it should "support coderIsSerializable" in {
    coderIsSerializable[Foo]
    coderIsSerializable(Coder[Foo])

    // Inner class's Coder is not serializable
    case class InnerCaseClass(id: String)

    an[TestFailedException] should be thrownBy {
      coderIsSerializable[InnerCaseClass]
    }

    an[TestFailedException] should be thrownBy {
      coderIsSerializable(Coder[InnerCaseClass])
    }
  }
} 

package isabelle


import java.io.{InputStream, OutputStream}
import java.net.{ServerSocket, InetAddress}


object System_Channel
{
  def apply(): System_Channel = new System_Channel
}

class System_Channel private
{
  private val server = new ServerSocket(0, 50, InetAddress.getByName("127.0.0.1"))

  val server_name: String = "127.0.0.1:" + server.getLocalPort
  override def toString: String = server_name

  def rendezvous(): (OutputStream, InputStream) =
  {
    val socket = server.accept
    socket.setTcpNoDelay(true)
    (socket.getOutputStream, socket.getInputStream)
  }

  def accepted() { server.close }
} 

package isabelle

import java.io.{ByteArrayOutputStream, OutputStream, InputStream, IOException}


object Byte_Message
{
  

  private def is_length(msg: Bytes): Boolean =
    !msg.is_empty && msg.iterator.forall(b => Symbol.is_ascii_digit(b.toChar))

  private def is_terminated(msg: Bytes): Boolean =
  {
    val len = msg.length
    len > 0 && Symbol.is_ascii_line_terminator(msg.charAt(len - 1))
  }

  def write_line_message(stream: OutputStream, msg: Bytes)
  {
    if (is_length(msg) || is_terminated(msg))
      error ("Bad content for line message:\n" ++ msg.text.take(100))

    val n = msg.length
    write(stream,
      (if (n > 100 || msg.iterator.contains(10)) make_header(List(n + 1)) else Nil) :::
        List(msg, Bytes.newline))
    flush(stream)
  }

  def read_line_message(stream: InputStream): Option[Bytes] =
    read_line(stream) match {
      case None => None
      case Some(line) =>
        Value.Nat.unapply(line.text) match {
          case None => Some(line)
          case Some(n) => read_block(stream, n)._1.map(_.trim_line)
        }
    }
} 

package isabelle


import java.io.{InputStream, OutputStream}
import java.net.{ServerSocket, InetAddress}


object System_Channel
{
  def apply(): System_Channel = new System_Channel
}

class System_Channel private
{
  private val server = new ServerSocket(0, 50, Server.localhost)

  val address: String = Server.print_address(server.getLocalPort)
  val password: String = UUID.random().toString

  override def toString: String = address

  def shutdown() { server.close }

  def rendezvous(): (OutputStream, InputStream) =
  {
    val socket = server.accept
    try {
      val out_stream = socket.getOutputStream
      val in_stream = socket.getInputStream

      if (Byte_Message.read_line(in_stream).map(_.text) == Some(password)) (out_stream, in_stream)
      else {
        out_stream.close
        in_stream.close
        error("Failed to connect system channel: bad password")
      }
    }
    finally { shutdown() }
  }
} 

package isabelle


import java.io.{InputStream, OutputStream}
import java.net.{ServerSocket, InetAddress}


object System_Channel
{
  def apply(): System_Channel = new System_Channel
}

class System_Channel private
{
  private val server = new ServerSocket(0, 50, InetAddress.getByName("127.0.0.1"))

  val server_name: String = "127.0.0.1:" + server.getLocalPort
  override def toString: String = server_name

  def rendezvous(): (OutputStream, InputStream) =
  {
    val socket = server.accept
    socket.setTcpNoDelay(true)
    (socket.getOutputStream, socket.getInputStream)
  }

  def accepted() { server.close }
} 

package io.github.mkotsur.aws.handler

import java.io.{InputStream, OutputStream}

import com.amazonaws.services.lambda.runtime.{Context, RequestStreamHandler}
import io.circe.generic.auto._
import io.github.mkotsur.aws.codecs._
import io.github.mkotsur.aws.proxy.{ProxyRequest, ProxyResponse}
import org.slf4j.LoggerFactory
import cats.syntax.either.catsSyntaxEither
import io.github.mkotsur.aws.handler.Lambda.HandleResult

import scala.language.{higherKinds, postfixOps}
import scala.util.{Failure, Success, Try}

object Lambda extends AllCodec with ProxyRequestCodec {

  type Handle[I, O]    = (I, Context) => HandleResult[O]
  type HandleResult[O] = Either[Throwable, O]
  type Proxy[I, O]     = Lambda[ProxyRequest[I], ProxyResponse[O]]

  object Proxy {
    type Handle[I, O]    = (ProxyRequest[I], Context) => HandleResult[O]
    type HandleResult[O] = Either[Throwable, ProxyResponse[O]]

    private type CanDecodeProxyRequest[A] = CanDecode[ProxyRequest[A]]
    private type CanEncodeProxyRequest[A] = CanEncode[ProxyResponse[A]]

    def instance[I: CanDecodeProxyRequest, O: CanEncodeProxyRequest](
        doHandle: Proxy.Handle[I, O]): Lambda[ProxyRequest[I], ProxyResponse[O]] =
      new Lambda.Proxy[I, O] {
        override protected def handle(i: ProxyRequest[I], c: Context) = doHandle(i, c)
      }
  }

  def instance[I: CanDecode, O: CanEncode](doHandle: Handle[I, O]) =
    new Lambda[I, O] {
      override protected def handle(i: I, c: Context): Either[Throwable, O] = {
        super.handle(i, c)
        doHandle(i, c)
      }
    }

  type ReadStream[I]       = InputStream => Either[Throwable, I]
  type ObjectHandler[I, O] = I => Either[Throwable, O]
  type WriteStream[O]      = (OutputStream, Either[Throwable, O], Context) => Either[Throwable, Unit]

  private val logger = LoggerFactory.getLogger(getClass)

}

abstract class Lambda[I: CanDecode, O: CanEncode] extends RequestStreamHandler {

  
  final def handle(input: InputStream, output: OutputStream, context: Context): Unit =
    handleRequest(input, output, context)

  // This function will ultimately be used as the external handler
  final def handleRequest(input: InputStream, output: OutputStream, context: Context): Unit = {
    val read = implicitly[CanDecode[I]].readStream(input)
    val handled = read.flatMap { input =>
      Try(handle(input, context)) match {
        case Success(v) => v
        case Failure(e) =>
          Lambda.logger.error(s"Error while executing lambda handler: ${e.getMessage}", e)
          Left(e)
      }
    }
    val written = implicitly[CanEncode[O]].writeStream(output, handled, context)
    output.close()
    written.left.foreach(e => throw e)
  }

} 

package fr.hmil.roshttp.tools.io

import java.io.{ByteArrayOutputStream, OutputStream, Writer, _}

import scala.annotation.tailrec
import scala.reflect.ClassTag


  def pipe(in: Reader, out: Writer): Unit = {
    val buffer = newBuffer[Char]

    @tailrec
    def loop(): Unit = {
      val size = in.read(buffer)
      if (size > 0) {
        out.write(buffer, 0, size)
        loop()
      }
    }
    loop()
  }

  @inline
  private def newBuffer[T: ClassTag] = new Array[T](4096)
} 

package com.datamountaineer.streamreactor.connect.serialization

import java.io.{ByteArrayOutputStream, InputStream, OutputStream}

import com.sksamuel.avro4s.{RecordFormat, SchemaFor}
import org.apache.avro.Schema
import org.apache.avro.generic.{GenericDatumReader, GenericDatumWriter, GenericRecord}
import org.apache.avro.io.{DecoderFactory, EncoderFactory}

object AvroSerializer {
  def write[T <: Product](t: T)(implicit os: OutputStream, formatter: RecordFormat[T], schemaFor: SchemaFor[T]): Unit = write(apply(t), schemaFor())

  def write(record: GenericRecord, schema: Schema)(implicit os: OutputStream) = {
    val writer = new GenericDatumWriter[GenericRecord](schema)
    val encoder = EncoderFactory.get().binaryEncoder(os, null)

    writer.write(record, encoder)
    encoder.flush()
    os.flush()
  }

  def getBytes[T <: Product](t: T)(implicit recordFormat: RecordFormat[T], schemaFor: SchemaFor[T]): Array[Byte] = getBytes(recordFormat.to(t), schemaFor())

  def getBytes(record: GenericRecord, schema: Schema): Array[Byte] = {
    implicit val output = new ByteArrayOutputStream()
    write(record, schema)
    output.toByteArray
  }

  def read(is: InputStream, schema: Schema): GenericRecord = {
    val reader = new GenericDatumReader[GenericRecord](schema)
    val decoder = DecoderFactory.get().binaryDecoder(is, null)
    reader.read(null, decoder)
  }

  def read[T <: Product](is: InputStream)(implicit schemaFor: SchemaFor[T], recordFormat: RecordFormat[T]): T = recordFormat.from(read(is, schemaFor()))

  def apply[T <: Product](t: T)(implicit formatter: RecordFormat[T]): GenericRecord = formatter.to(t)
} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.lang.annotation.Annotation
import java.lang.reflect.Type
import java.text.SimpleDateFormat
import java.util.{Calendar, SimpleTimeZone}
import javax.ws.rs.Produces
import javax.ws.rs.core.{MediaType, MultivaluedMap}
import javax.ws.rs.ext.{MessageBodyWriter, Provider}

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}


@Provider
@Produces(Array(MediaType.APPLICATION_JSON))
private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{

  val mapper = new ObjectMapper() {
    override def writeValueAsString(t: Any): String = {
      super.writeValueAsString(t)
    }
  }
  mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
  mapper.enable(SerializationFeature.INDENT_OUTPUT)
  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)

  override def isWriteable(
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Boolean = {
      true
  }

  override def writeTo(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType,
      multivaluedMap: MultivaluedMap[String, AnyRef],
      outputStream: OutputStream): Unit = {
    t match {
      case ErrorWrapper(err) => outputStream.write(err.getBytes("utf-8"))
      case _ => mapper.writeValue(outputStream, t)
    }
  }

  override def getSize(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Long = {
    -1L
  }
}

private[spark] object JacksonMessageWriter {
  def makeISODateFormat: SimpleDateFormat = {
    val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'")
    val cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"))
    iso8601.setCalendar(cal)
    iso8601
  }
} 

package receiver

import org.apache.spark.streaming.receiver.Receiver
import org.apache.spark.storage.StorageLevel
import org.apache.spark.Logging
import com.ning.http.client.AsyncHttpClientConfig
import com.ning.http.client._
import scala.collection.mutable.ArrayBuffer
import java.io.OutputStream
import java.io.ByteArrayInputStream
import java.io.InputStreamReader
import java.io.BufferedReader
import java.io.InputStream
import java.io.PipedInputStream
import java.io.PipedOutputStream

class MeetupReceiver(url: String) extends Receiver[String](StorageLevel.MEMORY_AND_DISK_2) with Logging {
  
  @transient var client: AsyncHttpClient = _
  
  @transient var inputPipe: PipedInputStream = _
  @transient var outputPipe: PipedOutputStream = _  
       
  def onStart() {    
    val cf = new AsyncHttpClientConfig.Builder()
    cf.setRequestTimeout(Integer.MAX_VALUE)
    cf.setReadTimeout(Integer.MAX_VALUE)
    cf.setPooledConnectionIdleTimeout(Integer.MAX_VALUE)      
    client= new AsyncHttpClient(cf.build())
    
    inputPipe = new PipedInputStream(1024 * 1024)
    outputPipe = new PipedOutputStream(inputPipe)
    val producerThread = new Thread(new DataConsumer(inputPipe))
    producerThread.start()
    
    client.prepareGet(url).execute(new AsyncHandler[Unit]{
        
      def onBodyPartReceived(bodyPart: HttpResponseBodyPart) = {
        bodyPart.writeTo(outputPipe)
        AsyncHandler.STATE.CONTINUE        
      }
      
      def onStatusReceived(status: HttpResponseStatus) = {
        AsyncHandler.STATE.CONTINUE
      }
      
      def onHeadersReceived(headers: HttpResponseHeaders) = {
        AsyncHandler.STATE.CONTINUE
      }
            
      def onCompleted = {
        println("completed")
      }
      
      
      def onThrowable(t: Throwable)={
        t.printStackTrace()
      }
        
    })    
    
    
  }

  def onStop() {
    if (Option(client).isDefined) client.close()
    if (Option(outputPipe).isDefined) {
     outputPipe.flush()
     outputPipe.close() 
    }
    if (Option(inputPipe).isDefined) {
     inputPipe.close() 
    }    
  }
  
  class DataConsumer(inputStream: InputStream) extends Runnable 
  {
       
      override
      def run()
      {        
        val bufferedReader = new BufferedReader( new InputStreamReader( inputStream ))
        var input=bufferedReader.readLine()
        while(input!=null){          
          store(input)
          input=bufferedReader.readLine()
        }            
      }  
      
  }

} 

package com.sksamuel.avro4s

import java.io.OutputStream

import org.apache.avro.Schema
import org.apache.avro.file.{CodecFactory, DataFileWriter}
import org.apache.avro.generic.{GenericDatumWriter, GenericRecord}


case class AvroDataOutputStream[T](os: OutputStream,
                                   codec: CodecFactory)
                                  (implicit encoder: Encoder[T]) extends AvroOutputStream[T] {

  val resolved = encoder.resolveEncoder()

  val (writer, writeFn) = resolved.schema.getType match {
    case Schema.Type.DOUBLE | Schema.Type.LONG | Schema.Type.BOOLEAN | Schema.Type.STRING | Schema.Type.INT | Schema.Type.FLOAT =>
      val datumWriter = new GenericDatumWriter[T](resolved.schema)
      val dataFileWriter = new DataFileWriter[T](datumWriter)
      dataFileWriter.setCodec(codec)
      dataFileWriter.create(resolved.schema, os)
      (dataFileWriter, (t: T) => dataFileWriter.append(t))
    case _ =>
      val datumWriter = new GenericDatumWriter[GenericRecord](resolved.schema)
      val dataFileWriter = new DataFileWriter[GenericRecord](datumWriter)
      dataFileWriter.setCodec(codec)
      dataFileWriter.create(resolved.schema, os)
      (dataFileWriter, (t: T) => {
        val record = resolved.encode(t).asInstanceOf[GenericRecord]
        dataFileWriter.append(record)
      })
  }

  override def close(): Unit = {
    flush()
    writer.close()
  }

  override def write(t: T): Unit = {
    writeFn(t)
  }

  override def flush(): Unit = writer.flush()
  override def fSync(): Unit = writer.fSync()
} 

package org.apache.spark.sql.execution.datasources

import java.io.{InputStream, OutputStream, OutputStreamWriter}
import java.nio.charset.{Charset, StandardCharsets}

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.hadoop.io.compress._
import org.apache.hadoop.mapreduce.JobContext
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
import org.apache.hadoop.util.ReflectionUtils

import org.apache.spark.TaskContext

object CodecStreams {
  private def getDecompressionCodec(config: Configuration, file: Path): Option[CompressionCodec] = {
    val compressionCodecs = new CompressionCodecFactory(config)
    Option(compressionCodecs.getCodec(file))
  }

  def createInputStream(config: Configuration, file: Path): InputStream = {
    val fs = file.getFileSystem(config)
    val inputStream: InputStream = fs.open(file)

    getDecompressionCodec(config, file)
      .map(codec => codec.createInputStream(inputStream))
      .getOrElse(inputStream)
  }

  
  def getCompressionExtension(context: JobContext): String = {
    getCompressionCodec(context)
      .map(_.getDefaultExtension)
      .getOrElse("")
  }
} 

package org.apache.spark.sql.execution.streaming


import java.io.{InputStream, OutputStream}
import java.nio.charset.StandardCharsets._

import scala.io.{Source => IOSource}

import org.apache.spark.sql.SparkSession


class OffsetSeqLog(sparkSession: SparkSession, path: String)
  extends HDFSMetadataLog[OffsetSeq](sparkSession, path) {

  override protected def deserialize(in: InputStream): OffsetSeq = {
    // called inside a try-finally where the underlying stream is closed in the caller
    def parseOffset(value: String): Offset = value match {
      case OffsetSeqLog.SERIALIZED_VOID_OFFSET => null
      case json => SerializedOffset(json)
    }
    val lines = IOSource.fromInputStream(in, UTF_8.name()).getLines()
    if (!lines.hasNext) {
      throw new IllegalStateException("Incomplete log file")
    }

    val version = parseVersion(lines.next(), OffsetSeqLog.VERSION)

    // read metadata
    val metadata = lines.next().trim match {
      case "" => None
      case md => Some(md)
    }
    OffsetSeq.fill(metadata, lines.map(parseOffset).toArray: _*)
  }

  override protected def serialize(offsetSeq: OffsetSeq, out: OutputStream): Unit = {
    // called inside a try-finally where the underlying stream is closed in the caller
    out.write(("v" + OffsetSeqLog.VERSION).getBytes(UTF_8))

    // write metadata
    out.write('\n')
    out.write(offsetSeq.metadata.map(_.json).getOrElse("").getBytes(UTF_8))

    // write offsets, one per line
    offsetSeq.offsets.map(_.map(_.json)).foreach { offset =>
      out.write('\n')
      offset match {
        case Some(json: String) => out.write(json.getBytes(UTF_8))
        case None => out.write(OffsetSeqLog.SERIALIZED_VOID_OFFSET.getBytes(UTF_8))
      }
    }
  }
}

object OffsetSeqLog {
  private[streaming] val VERSION = 1
  private val SERIALIZED_VOID_OFFSET = "-"
} 

package org.apache.spark.sql.execution.streaming

import java.io.{InputStream, OutputStream}
import java.nio.charset.StandardCharsets._

import scala.io.{Source => IOSource}

import org.apache.spark.sql.SparkSession


class CommitLog(sparkSession: SparkSession, path: String)
  extends HDFSMetadataLog[String](sparkSession, path) {

  import CommitLog._

  def add(batchId: Long): Unit = {
    super.add(batchId, EMPTY_JSON)
  }

  override def add(batchId: Long, metadata: String): Boolean = {
    throw new UnsupportedOperationException(
      "CommitLog does not take any metadata, use 'add(batchId)' instead")
  }

  override protected def deserialize(in: InputStream): String = {
    // called inside a try-finally where the underlying stream is closed in the caller
    val lines = IOSource.fromInputStream(in, UTF_8.name()).getLines()
    if (!lines.hasNext) {
      throw new IllegalStateException("Incomplete log file in the offset commit log")
    }
    parseVersion(lines.next.trim, VERSION)
    EMPTY_JSON
  }

  override protected def serialize(metadata: String, out: OutputStream): Unit = {
    // called inside a try-finally where the underlying stream is closed in the caller
    out.write(s"v${VERSION}".getBytes(UTF_8))
    out.write('\n')

    // write metadata
    out.write(EMPTY_JSON.getBytes(UTF_8))
  }
}

object CommitLog {
  private val VERSION = 1
  private val EMPTY_JSON = "{}"
} 

package org.apache.spark.streaming.util

import java.io.OutputStream
import java.util.concurrent.TimeUnit._

import scala.annotation.tailrec

import org.apache.spark.internal.Logging

private[streaming]
class RateLimitedOutputStream(out: OutputStream, desiredBytesPerSec: Int)
  extends OutputStream
  with Logging {

  require(desiredBytesPerSec > 0)

  private val SYNC_INTERVAL = NANOSECONDS.convert(10, SECONDS)
  private val CHUNK_SIZE = 8192
  private var lastSyncTime = System.nanoTime
  private var bytesWrittenSinceSync = 0L

  override def write(b: Int) {
    waitToWrite(1)
    out.write(b)
  }

  override def write(bytes: Array[Byte]) {
    write(bytes, 0, bytes.length)
  }

  @tailrec
  override final def write(bytes: Array[Byte], offset: Int, length: Int) {
    val writeSize = math.min(length - offset, CHUNK_SIZE)
    if (writeSize > 0) {
      waitToWrite(writeSize)
      out.write(bytes, offset, writeSize)
      write(bytes, offset + writeSize, length)
    }
  }

  override def flush() {
    out.flush()
  }

  override def close() {
    out.close()
  }

  @tailrec
  private def waitToWrite(numBytes: Int) {
    val now = System.nanoTime
    val elapsedNanosecs = math.max(now - lastSyncTime, 1)
    val rate = bytesWrittenSinceSync.toDouble * 1000000000 / elapsedNanosecs
    if (rate < desiredBytesPerSec) {
      // It's okay to write; just update some variables and return
      bytesWrittenSinceSync += numBytes
      if (now > lastSyncTime + SYNC_INTERVAL) {
        // Sync interval has passed; let's resync
        lastSyncTime = now
        bytesWrittenSinceSync = numBytes
      }
    } else {
      // Calculate how much time we should sleep to bring ourselves to the desired rate.
      val targetTimeInMillis = bytesWrittenSinceSync * 1000 / desiredBytesPerSec
      val elapsedTimeInMillis = elapsedNanosecs / 1000000
      val sleepTimeInMillis = targetTimeInMillis - elapsedTimeInMillis
      if (sleepTimeInMillis > 0) {
        logTrace("Natural rate is " + rate + " per second but desired rate is " +
          desiredBytesPerSec + ", sleeping for " + sleepTimeInMillis + " ms to compensate.")
        Thread.sleep(sleepTimeInMillis)
      }
      waitToWrite(numBytes)
    }
  }
} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.lang.annotation.Annotation
import java.lang.reflect.Type
import java.nio.charset.StandardCharsets
import java.text.SimpleDateFormat
import java.util.{Calendar, Locale, SimpleTimeZone}
import javax.ws.rs.Produces
import javax.ws.rs.core.{MediaType, MultivaluedMap}
import javax.ws.rs.ext.{MessageBodyWriter, Provider}

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}


@Provider
@Produces(Array(MediaType.APPLICATION_JSON))
private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{

  val mapper = new ObjectMapper() {
    override def writeValueAsString(t: Any): String = {
      super.writeValueAsString(t)
    }
  }
  mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
  mapper.enable(SerializationFeature.INDENT_OUTPUT)
  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)

  override def isWriteable(
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Boolean = {
      true
  }

  override def writeTo(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType,
      multivaluedMap: MultivaluedMap[String, AnyRef],
      outputStream: OutputStream): Unit = {
    t match {
      case ErrorWrapper(err) => outputStream.write(err.getBytes(StandardCharsets.UTF_8))
      case _ => mapper.writeValue(outputStream, t)
    }
  }

  override def getSize(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Long = {
    -1L
  }
}

private[spark] object JacksonMessageWriter {
  def makeISODateFormat: SimpleDateFormat = {
    val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'", Locale.US)
    val cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"))
    iso8601.setCalendar(cal)
    iso8601
  }
} 

package org.apache.spark.util.io

import java.io.OutputStream
import java.nio.ByteBuffer

import scala.collection.mutable.ArrayBuffer

import org.apache.spark.storage.StorageUtils


  private[this] var position = chunkSize
  private[this] var _size = 0
  private[this] var closed: Boolean = false

  def size: Long = _size

  override def close(): Unit = {
    if (!closed) {
      super.close()
      closed = true
    }
  }

  override def write(b: Int): Unit = {
    require(!closed, "cannot write to a closed ChunkedByteBufferOutputStream")
    allocateNewChunkIfNeeded()
    chunks(lastChunkIndex).put(b.toByte)
    position += 1
    _size += 1
  }

  override def write(bytes: Array[Byte], off: Int, len: Int): Unit = {
    require(!closed, "cannot write to a closed ChunkedByteBufferOutputStream")
    var written = 0
    while (written < len) {
      allocateNewChunkIfNeeded()
      val thisBatch = math.min(chunkSize - position, len - written)
      chunks(lastChunkIndex).put(bytes, written + off, thisBatch)
      written += thisBatch
      position += thisBatch
    }
    _size += len
  }

  @inline
  private def allocateNewChunkIfNeeded(): Unit = {
    if (position == chunkSize) {
      chunks += allocator(chunkSize)
      lastChunkIndex += 1
      position = 0
    }
  }

  def toChunkedByteBuffer: ChunkedByteBuffer = {
    require(closed, "cannot call toChunkedByteBuffer() unless close() has been called")
    require(!toChunkedByteBufferWasCalled, "toChunkedByteBuffer() can only be called once")
    toChunkedByteBufferWasCalled = true
    if (lastChunkIndex == -1) {
      new ChunkedByteBuffer(Array.empty[ByteBuffer])
    } else {
      // Copy the first n-1 chunks to the output, and then create an array that fits the last chunk.
      // An alternative would have been returning an array of ByteBuffers, with the last buffer
      // bounded to only the last chunk's position. However, given our use case in Spark (to put
      // the chunks in block manager), only limiting the view bound of the buffer would still
      // require the block manager to store the whole chunk.
      val ret = new Array[ByteBuffer](chunks.size)
      for (i <- 0 until chunks.size - 1) {
        ret(i) = chunks(i)
        ret(i).flip()
      }
      if (position == chunkSize) {
        ret(lastChunkIndex) = chunks(lastChunkIndex)
        ret(lastChunkIndex).flip()
      } else {
        ret(lastChunkIndex) = allocator(position)
        chunks(lastChunkIndex).flip()
        ret(lastChunkIndex).put(chunks(lastChunkIndex))
        ret(lastChunkIndex).flip()
        StorageUtils.dispose(chunks(lastChunkIndex))
      }
      new ChunkedByteBuffer(ret)
    }
  }
} 

package mimir.data.staging

import java.net.URL
import java.io.{ File, InputStream, OutputStream, FileOutputStream }
import java.sql.SQLException
import scala.util.Random
import com.typesafe.scalalogging.LazyLogging
import org.apache.spark.sql.DataFrame
import mimir.algebra.ID


  private def transferBytes(input: InputStream, output: OutputStream): Unit =
  {
    val buffer = Array.ofDim[Byte](1024*1024) // 1MB buffer
    var bytesRead = input.read(buffer)
    while(bytesRead >= 0) { 
      output.write(buffer, 0, bytesRead)
      bytesRead = input.read(buffer)
    }
  }

  def stage(input: InputStream, fileExtension: String, nameHint: Option[String]): String = 
  {
    val file = makeName(fileExtension, nameHint)
    transferBytes(input, new FileOutputStream(file))
    return file.toString
  }
  def stage(url: URL, nameHint: Option[String]): String =
  {
    val pathComponents = url.getPath.split("/")
    val nameComponents = pathComponents.reverse.head.split(".")
    val extension = 
      if(nameComponents.size > 1) { nameComponents.reverse.head }
      else { "data" } // default to generic 'data' if there's no extension
    stage(url.openStream(), extension, nameHint)
  }
  def stage(input: DataFrame, format: ID, nameHint:Option[String]): String =
  {
    val targetFile = makeName(format.id, nameHint).toString
    input.write
         .format(format.id)
         .save(targetFile)
    return targetFile
  }
  def drop(local: String): Unit = 
  {
    new File(local).delete()
  }
} 

package mimir.data.staging

import java.net.URL
import java.io.{ File, InputStream, OutputStream, FileOutputStream }
import java.sql.SQLException
import scala.util.Random
import com.typesafe.scalalogging.LazyLogging
import org.apache.spark.sql.DataFrame
import mimir.algebra.ID
import mimir.util.HadoopUtils
import mimir.exec.spark.MimirSpark


  private def makeName(extension: String, nameHint: Option[String]): File =
  {
    val rand = new Random().alphanumeric
    // Try 1000 times to create a randomly named file
    for(i <- 0 until 1000){
      val candidate = new File(basePath,
        nameHint match { 
          case Some(hint) => s"${hint.replaceAll("[^a-zA-Z0-9]", "")}-${rand.take(10).mkString}.${extension}"
          case None => s"${rand.take(20).mkString}.${extension}"
        }
      )
      // If the randomly named file doesn't exist, we're done.
      if(!candidate.exists()){ return candidate }
    }
    // Fail after 1000 attempts.
    throw new SQLException(s"Can't allocate name for $nameHint")
  }


  def stage(input: InputStream, fileExtension: String, nameHint: Option[String]): String = 
  {
    val file = makeName(fileExtension, nameHint)
    logger.debug("Stage File To HDFS: " +hdfsHome+File.separator+file.toString)
    //if(!HadoopUtils.fileExistsHDFS(sparkSql.sparkSession.sparkContext, fileName))
    HadoopUtils.writeToHDFS(MimirSpark.get.sparkSession.sparkContext, file.getName, input, true)
    logger.debug("... done\n")
    return s"$hdfsHome/${file.getName}"
  }
  def stage(url: URL, nameHint: Option[String]): String =
  {
    val pathComponents = url.getPath.split("/")
    val nameComponents = pathComponents.reverse.head.split(".")
    val extension = 
      if(nameComponents.size > 1) { nameComponents.reverse.head }
      else { "data" } // default to generic 'data' if there's no extension
    stage(url.openStream(), extension, nameHint)
  }
  def stage(input: DataFrame, format: ID, nameHint:Option[String]): String =
  {
    val targetFile = makeName(format.id, nameHint).toString
    input.write
         .format(format.id)
         .save(targetFile)
    return targetFile
  }
  def drop(local: String): Unit = 
  {
    new File(local).delete()
  }
} 

package bloomfilter.mutable._128bit

import java.io.{DataInputStream, DataOutputStream, InputStream, OutputStream}

import bloomfilter.CanGenerate128HashFrom
import bloomfilter.mutable.UnsafeBitArray

import scala.math._

@SerialVersionUID(2L)
class BloomFilter[T] private (val numberOfBits: Long, val numberOfHashes: Int, private val bits: UnsafeBitArray)
    (implicit canGenerateHash: CanGenerate128HashFrom[T]) extends Serializable {

  def this(numberOfBits: Long, numberOfHashes: Int)(implicit canGenerateHash: CanGenerate128HashFrom[T]) {
    this(numberOfBits, numberOfHashes, new UnsafeBitArray(numberOfBits))
  }

  def add(x: T): Unit = {
    val hash = canGenerateHash.generateHash(x)

    var i = 0
    while (i < numberOfHashes) {
      val computedHash = hash._1 + i * hash._2
      bits.set((computedHash & Long.MaxValue) % numberOfBits)
      i += 1
    }
  }

  def mightContain(x: T): Boolean = {
    val hash = canGenerateHash.generateHash(x)

    var i = 0
    while (i < numberOfHashes) {
      val computedHash = hash._1 + i * hash._2
      if (!bits.get((computedHash & Long.MaxValue) % numberOfBits))
        return false
      i += 1
    }
    true
  }

  def expectedFalsePositiveRate(): Double = {
    math.pow(bits.getBitCount.toDouble / numberOfBits, numberOfHashes.toDouble)
  }

  def writeTo(out: OutputStream): Unit = {
    val dout = new DataOutputStream(out)
    dout.writeLong(numberOfBits)
    dout.writeInt(numberOfHashes)
    bits.writeTo(out)
  }

  def approximateElementCount(): Long = {
    val fractionOfBitsSet = bits.getBitCount.toDouble / numberOfBits
    val x = -log1p(-fractionOfBitsSet) * numberOfBits / numberOfHashes
    val z = rint(x)
    if (abs(x - z) == 0.5) {
      (x + Math.copySign(0.5, x)).toLong
    } else {
      z.toLong
    }
  }

  def dispose(): Unit = bits.dispose()

}

object BloomFilter {

  def apply[T](numberOfItems: Long, falsePositiveRate: Double)
      (implicit canGenerateHash: CanGenerate128HashFrom[T]): BloomFilter[T] = {

    val nb = optimalNumberOfBits(numberOfItems, falsePositiveRate)
    val nh = optimalNumberOfHashes(numberOfItems, nb)
    new BloomFilter[T](nb, nh)
  }

  def optimalNumberOfBits(numberOfItems: Long, falsePositiveRate: Double): Long = {
    math.ceil(-1 * numberOfItems * math.log(falsePositiveRate) / math.log(2) / math.log(2)).toLong
  }

  def optimalNumberOfHashes(numberOfItems: Long, numberOfBits: Long): Int = {
    math.ceil(numberOfBits / numberOfItems * math.log(2)).toInt
  }

  def readFrom[T](in: InputStream)(implicit canGenerateHash: CanGenerate128HashFrom[T]): BloomFilter[T] = {
    val din = new DataInputStream(in)
    val numberOfBits = din.readLong()
    val numberOfHashes = din.readInt()
    val bits = new UnsafeBitArray(numberOfBits)
    bits.readFrom(in)
    new BloomFilter[T](numberOfBits, numberOfHashes, bits)
  }

} 

package org.apache.spark.streaming.util

import scala.annotation.tailrec

import java.io.OutputStream
import java.util.concurrent.TimeUnit._

import org.apache.spark.Logging


private[streaming]
class RateLimitedOutputStream(out: OutputStream, desiredBytesPerSec: Int)
  extends OutputStream
  with Logging {

  require(desiredBytesPerSec > 0)

  private val SYNC_INTERVAL = NANOSECONDS.convert(10, SECONDS)
  private val CHUNK_SIZE = 8192
  private var lastSyncTime = System.nanoTime
  private var bytesWrittenSinceSync = 0L

  override def write(b: Int) {
    waitToWrite(1)
    out.write(b)
  }

  override def write(bytes: Array[Byte]) {
    write(bytes, 0, bytes.length)
  }

  @tailrec
  override final def write(bytes: Array[Byte], offset: Int, length: Int) {
    val writeSize = math.min(length - offset, CHUNK_SIZE)
    if (writeSize > 0) {
      waitToWrite(writeSize)
      out.write(bytes, offset, writeSize)
      write(bytes, offset + writeSize, length)
    }
  }

  override def flush() {
    out.flush()
  }

  override def close() {
    out.close()
  }

  @tailrec
  private def waitToWrite(numBytes: Int) {
    val now = System.nanoTime
    val elapsedNanosecs = math.max(now - lastSyncTime, 1)
    val rate = bytesWrittenSinceSync.toDouble * 1000000000 / elapsedNanosecs
    if (rate < desiredBytesPerSec) {
      // It's okay to write; just update some variables and return
      bytesWrittenSinceSync += numBytes
      if (now > lastSyncTime + SYNC_INTERVAL) {
        // Sync interval has passed; let's resync
        lastSyncTime = now
        bytesWrittenSinceSync = numBytes
      }
    } else {
      // Calculate how much time we should sleep to bring ourselves to the desired rate.
      val targetTimeInMillis = bytesWrittenSinceSync * 1000 / desiredBytesPerSec
      val elapsedTimeInMillis = elapsedNanosecs / 1000000
      val sleepTimeInMillis = targetTimeInMillis - elapsedTimeInMillis
      if (sleepTimeInMillis > 0) {
        logTrace("Natural rate is " + rate + " per second but desired rate is " +
          desiredBytesPerSec + ", sleeping for " + sleepTimeInMillis + " ms to compensate.")
        Thread.sleep(sleepTimeInMillis)
      }
      waitToWrite(numBytes)
    }
  }
} 

package com.spotify.ratatool.io

import java.io.{File, InputStream, OutputStream}
import java.nio.ByteBuffer
import java.nio.channels.SeekableByteChannel

import com.google.common.io.ByteStreams
import org.apache.avro.Schema
import org.apache.avro.file.{DataFileReader, DataFileWriter, SeekableByteArrayInput, SeekableInput}
import org.apache.avro.generic.{GenericDatumReader, GenericDatumWriter, GenericRecord}
import org.apache.avro.io.{DatumReader, DatumWriter}
import org.apache.avro.reflect.{ReflectDatumReader, ReflectDatumWriter}
import org.apache.avro.specific.{SpecificDatumReader, SpecificDatumWriter, SpecificRecord}
import org.apache.beam.sdk.io.FileSystems
import org.apache.beam.sdk.io.fs.MatchResult.Metadata

import scala.jdk.CollectionConverters._
import scala.reflect.ClassTag


  def writeToOutputStream[T: ClassTag](data: Iterable[T],
                                       schema: Schema,
                                       os: OutputStream): Unit = {
    val fileWriter = new DataFileWriter(createDatumWriter[T]).create(schema, os)
    data.foreach(fileWriter.append)
    fileWriter.close()
  }

  def getAvroSchemaFromFile(path: String): Schema = {
    require(FileStorage(path).exists, s"File `$path` does not exist!")
    val files = FileStorage(path).listFiles.filter(_.resourceId.getFilename.endsWith(".avro"))
    require(files.nonEmpty, s"File `$path` does not contain avro files")
    val reader = new GenericDatumReader[GenericRecord]()
    val dfr = new DataFileReader[GenericRecord](AvroIO.getAvroSeekableInput(files.head), reader)
    dfr.getSchema
  }

  private def getAvroSeekableInput(meta: Metadata): SeekableInput = new SeekableInput {
    require(meta.isReadSeekEfficient)
    private val in = FileSystems.open(meta.resourceId()).asInstanceOf[SeekableByteChannel]
    override def read(b: Array[Byte], off: Int, len: Int): Int =
      in.read(ByteBuffer.wrap(b, off, len))
    override def tell(): Long = in.position()
    override def length(): Long = in.size()
    override def seek(p: Long): Unit = in.position(p)
    override def close(): Unit = in.close()
  }

} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.util.zip.ZipOutputStream
import javax.ws.rs.{GET, Produces}
import javax.ws.rs.core.{MediaType, Response, StreamingOutput}

import scala.util.control.NonFatal

import org.apache.spark.{Logging, SparkConf}
import org.apache.spark.deploy.SparkHadoopUtil

@Produces(Array(MediaType.APPLICATION_OCTET_STREAM))
private[v1] class EventLogDownloadResource(
    val uIRoot: UIRoot,
    val appId: String,
    val attemptId: Option[String]) extends Logging {
  val conf = SparkHadoopUtil.get.newConfiguration(new SparkConf)

  @GET
  def getEventLogs(): Response = {
    try {
      val fileName = {
        attemptId match {
          case Some(id) => s"eventLogs-$appId-$id.zip"
          case None => s"eventLogs-$appId.zip"
        }
      }

      val stream = new StreamingOutput {
        override def write(output: OutputStream): Unit = {
          val zipStream = new ZipOutputStream(output)
          try {
            uIRoot.writeEventLogs(appId, attemptId, zipStream)
          } finally {
            zipStream.close()
          }

        }
      }

      Response.ok(stream)
        .header("Content-Disposition", s"attachment; filename=$fileName")
        .header("Content-Type", MediaType.APPLICATION_OCTET_STREAM)
        .build()
    } catch {
      case NonFatal(e) =>
        Response.serverError()
          .entity(s"Event logs are not available for app: $appId.")
          .status(Response.Status.SERVICE_UNAVAILABLE)
          .build()
    }
  }
} 

package org.apache.spark.util.io

import java.io.OutputStream

import scala.collection.mutable.ArrayBuffer



  private var position = chunkSize

  override def write(b: Int): Unit = {
    allocateNewChunkIfNeeded()
    chunks(lastChunkIndex)(position) = b.toByte
    position += 1
  }

  override def write(bytes: Array[Byte], off: Int, len: Int): Unit = {
    var written = 0
    while (written < len) {
      allocateNewChunkIfNeeded()
      val thisBatch = math.min(chunkSize - position, len - written)
      System.arraycopy(bytes, written + off, chunks(lastChunkIndex), position, thisBatch)
      written += thisBatch
      position += thisBatch
    }
  }

  @inline
  private def allocateNewChunkIfNeeded(): Unit = {
    if (position == chunkSize) {
      chunks += new Array[Byte](chunkSize)
      lastChunkIndex += 1
      position = 0
    }
  }

  def toArrays: Array[Array[Byte]] = {
    if (lastChunkIndex == -1) {
      new Array[Array[Byte]](0)
    } else {
      // Copy the first n-1 chunks to the output, and then create an array that fits the last chunk.
      // An alternative would have been returning an array of ByteBuffers, with the last buffer
      // bounded to only the last chunk's position. However, given our use case in Spark (to put
      // the chunks in block manager), only limiting the view bound of the buffer would still
      // require the block manager to store the whole chunk.
      val ret = new Array[Array[Byte]](chunks.size)
      for (i <- 0 until chunks.size - 1) {
        ret(i) = chunks(i)
      }
      if (position == chunkSize) {
        ret(lastChunkIndex) = chunks(lastChunkIndex)
      } else {
        ret(lastChunkIndex) = new Array[Byte](position)
        System.arraycopy(chunks(lastChunkIndex), 0, ret(lastChunkIndex), 0, position)
      }
      ret
    }
  }
} 

package sttp.client

import java.io.{ByteArrayOutputStream, InputStream, OutputStream}
import java.nio.{Buffer, ByteBuffer}

import scala.annotation.{implicitNotFound, tailrec}

package object internal {
  private[client] def contentTypeWithCharset(ct: String, charset: String): String =
    s"$ct; charset=$charset"

  private[client] def charsetFromContentType(ct: String): Option[String] =
    ct.split(";").map(_.trim.toLowerCase).collectFirst {
      case s if s.startsWith("charset=") && s.substring(8).trim != "" => s.substring(8).trim
    }

  private[client] def transfer(is: InputStream, os: OutputStream): Unit = {
    var read = 0
    val buf = new Array[Byte](1024)

    @tailrec
    def transfer(): Unit = {
      read = is.read(buf, 0, buf.length)
      if (read != -1) {
        os.write(buf, 0, read)
        transfer()
      }
    }

    transfer()
  }

  private[client] def toByteArray(is: InputStream): Array[Byte] = {
    val os = new ByteArrayOutputStream
    transfer(is, os)
    os.toByteArray
  }

  private[client] def concatByteBuffers(bb1: ByteBuffer, bb2: ByteBuffer): ByteBuffer = {
    val buf = ByteBuffer
      .allocate(bb1.array().length + bb2.array().length)
      .put(bb1)
      .put(bb2)
    // rewind() returns Buffer in Java8, and ByteBuffer in Java11
    // calling the method from the base class to avoid NoSuchMethodError
    (buf: Buffer).rewind()
    buf
  }

  
  private[client] def sanitizeCharset(charset: String): String = {
    val c2 = charset.trim()
    val c3 = if (c2.startsWith("\"")) c2.substring(1) else c2
    if (c3.endsWith("\"")) c3.substring(0, c3.length - 1) else c3
  }

  @implicitNotFound(
    "This is a partial request, the method & url are not specified. Use " +
      ".get(...), .post(...) etc. to obtain a non-partial request."
  )
  private[client] type IsIdInRequest[U[_]] = U[Unit] =:= Identity[Unit]

  private[client] val Utf8 = "utf-8"
  private[client] val Iso88591 = "iso-8859-1"
  private[client] val CrLf = "\r\n"
} 

package com.spotify.featran.numpy

import java.io.{ByteArrayOutputStream, OutputStream}

import org.scalatest._
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class NumPyTest extends AnyFlatSpec with Matchers {
  private def test(f: OutputStream => Unit)(expectedFile: String): Unit = {
    val actual = {
      val baos = new ByteArrayOutputStream()
      f(baos)
      baos.toByteArray
    }

    val expected = {
      val in = this.getClass.getResourceAsStream(expectedFile)
      val out = new ByteArrayOutputStream(math.max(32, in.available()))
      val buf = new Array[Byte](8192)
      var r = in.read(buf)
      while (r != -1) {
        out.write(buf, 0, r)
        r = in.read(buf)
      }
      out.toByteArray
    }

    actual shouldBe expected
  }

  "NumPy" should "work with 1-dimensional arrays" in {
    val a1d = (0 until 10).toArray
    test(NumPy.write(_, a1d))("/a1d-int.npy")
    test(NumPy.write(_, a1d.map(_.toLong)))("/a1d-long.npy")
    test(NumPy.write(_, a1d.map(_.toFloat)))("/a1d-float.npy")
    test(NumPy.write(_, a1d.map(_.toDouble)))("/a1d-double.npy")

    // scalastyle:off no.whitespace.before.left.bracket
    the[IllegalArgumentException] thrownBy {
      test(NumPy.write(_, a1d, Seq(20)))("/a1d-int.npy")
    } should have message "requirement failed: Invalid shape, 20 != 10"
    // scalastyle:on no.whitespace.before.left.bracket
  }

  it should "work with 2-dimensional arrays" in {
    val a2d = (for {
      i <- 0 until 10
      j <- 0 until 5
    } yield i * 10 + j).toArray
    test(NumPy.write(_, a2d, Seq(10, 5)))("/a2d-int.npy")
    test(NumPy.write(_, a2d.map(_.toLong), Seq(10, 5)))("/a2d-long.npy")
    test(NumPy.write(_, a2d.map(_.toFloat), Seq(10, 5)))("/a2d-float.npy")
    test(NumPy.write(_, a2d.map(_.toDouble), Seq(10, 5)))("/a2d-double.npy")

    // scalastyle:off no.whitespace.before.left.bracket
    the[IllegalArgumentException] thrownBy {
      test(NumPy.write(_, a2d, Seq(20, 5)))("/a1d-int.npy")
    } should have message "requirement failed: Invalid shape, 20 * 5 != 50"
    // scalastyle:on no.whitespace.before.left.bracket
  }

  it should "work with iterators" in {
    val a2d = (0 until 10).map(i => (0 until 5).map(j => i * 10 + j).toArray)
    test(NumPy.write(_, a2d.iterator, 10, 5))("/a2d-int.npy")
    test(NumPy.write(_, a2d.iterator.map(_.map(_.toLong)), 10, 5))("/a2d-long.npy")
    test(NumPy.write(_, a2d.iterator.map(_.map(_.toFloat)), 10, 5))("/a2d-float.npy")
    test(NumPy.write(_, a2d.iterator.map(_.map(_.toDouble)), 10, 5))("/a2d-double.npy")

    // scalastyle:off no.whitespace.before.left.bracket
    the[IllegalArgumentException] thrownBy {
      test(NumPy.write(_, a2d.iterator, 10, 10))("/a2d-int.npy")
    } should have message "requirement failed: Invalid row size, expected: 10, actual: 5"

    the[IllegalArgumentException] thrownBy {
      test(NumPy.write(_, a2d.iterator, 20, 5))("/a2d-int.npy")
    } should have message "requirement failed: Invalid number of rows, expected: 20, actual: 10"

    // hit the header.length % 16 == 0 condition
    the[IllegalArgumentException] thrownBy {
      test(NumPy.write(_, a2d.iterator, 1000000000, 50))("/a2d-int.npy")
    } should have message "requirement failed: Invalid row size, expected: 50, actual: 5"
    // scalastyle:on no.whitespace.before.left.bracket
  }
} 

package swave.compat.akka.impl

import java.io.OutputStream
import java.nio.charset.{CharacterCodingException, Charset}
import java.nio.{ByteBuffer, CharBuffer}
import java.util
import scala.annotation.tailrec
import scala.collection.GenTraversableOnce
import akka.util.ByteString
import swave.core.io.Bytes
import swave.core.macros._

class ByteStringBytes extends Bytes[ByteString] {

  ///////////////// CONSTRUCTION ///////////////////

  def empty = ByteString.empty
  def fill[A: Integral](size: Long)(byte: A) = {
    requireArg(0 <= size && size <= Int.MaxValue, "`size` must be >= 0 and <= Int.MaxValue")
    val b = implicitly[Integral[A]].toInt(byte).toByte
    apply(Array.fill(size.toInt)(b))
  }
  def apply(array: Array[Byte]) = ByteString(array)
  def apply(bytes: Array[Byte], offset: Int, length: Int) =
    ByteString(util.Arrays.copyOfRange(bytes, offset, offset + length))
  def apply[A: Integral](bytes: A*) = {
    val integral = implicitly[Integral[A]]
    val buf      = new Array[Byte](bytes.size)
    @tailrec def rec(ix: Int): ByteString =
      if (ix < buf.length) {
        buf(ix) = integral.toInt(bytes(ix)).toByte
        rec(ix + 1)
      } else view(buf)
    rec(0)
  }
  def apply(bytes: Vector[Byte])                  = ByteString(bytes: _*)
  def apply(buffer: ByteBuffer)                   = ByteString(buffer)
  def apply(bs: GenTraversableOnce[Byte])         = ByteString(bs.toArray)
  def view(bytes: Array[Byte])                    = ByteString(bytes) // no view-like constructor available on ByteStrings
  def view(bytes: ByteBuffer)                     = ByteString(bytes) // no view-like constructor available on ByteStrings
  def encodeString(str: String, charset: Charset) = ByteString(str, charset.name)
  def encodeStringStrict(str: String, charset: Charset) =
    try Right(ByteString(charset.newEncoder.encode(CharBuffer.wrap(str))))
    catch { case e: CharacterCodingException ⇒ Left(e) }

  ///////////////// QUERY ///////////////////

  def size(value: ByteString): Long = value.size.toLong
  def byteAt(value: ByteString, ix: Long): Byte = {
    requireArg(0 <= ix && ix <= Int.MaxValue, "`ix` must be >= 0 and <= Int.MaxValue")
    value(ix.toInt)
  }
  def indexOfSlice(value: ByteString, slice: ByteString, startIx: Long): Long = {
    requireArg(0 <= startIx && startIx <= Int.MaxValue, "`startIx` must be >= 0 and <= Int.MaxValue")
    value.indexOfSlice(slice, startIx.toInt).toLong
  }

  ///////////////// TRANSFORMATION TO ByteString ///////////////////

  def update(value: ByteString, ix: Long, byte: Byte) = concat(concat(take(value, ix), byte), drop(value, ix + 1))
  def concat(value: ByteString, other: ByteString)    = value ++ other
  def concat(value: ByteString, byte: Byte)           = value ++ ByteString(byte)
  def concat(byte: Byte, value: ByteString)           = ByteString(byte) ++ value
  def drop(value: ByteString, n: Long) = {
    requireArg(0 <= n && n <= Int.MaxValue, "`n` must be >= 0 and <= Int.MaxValue")
    value.drop(n.toInt)
  }
  def take(value: ByteString, n: Long) = {
    requireArg(0 <= n && n <= Int.MaxValue, "`n` must be >= 0 and <= Int.MaxValue")
    value.take(n.toInt)
  }
  def map(value: ByteString, f: Byte ⇒ Byte) = value.map(f)
  def reverse(value: ByteString)             = value.reverse
  def compact(value: ByteString)             = value.compact

  ///////////////// TRANSFORMATION TO OTHER TYPES ///////////////////

  def toArray(value: ByteString)                                   = value.toArray
  def copyToArray(value: ByteString, xs: Array[Byte], offset: Int) = value.copyToArray(xs, offset)
  def copyToArray(value: ByteString, sourceOffset: Long, xs: Array[Byte], destOffset: Int, len: Int) =
    drop(value, sourceOffset).copyToArray(xs, destOffset, len)
  def copyToBuffer(value: ByteString, buffer: ByteBuffer): Int = value.copyToBuffer(buffer)
  def copyToOutputStream(value: ByteString, s: OutputStream) = {
    @tailrec def rec(ix: Int, size: Int): Unit = if (ix < size) { s.write(value(ix).toInt); rec(ix + 1, size) }
    rec(0, value.size)
  }
  def toByteBuffer(value: ByteString)                   = value.toByteBuffer
  def toIndexedSeq(value: ByteString): IndexedSeq[Byte] = value
  def toSeq(value: ByteString): Seq[Byte]               = value
  def decodeString(value: ByteString, charset: Charset): Either[CharacterCodingException, String] =
    try Right(charset.newDecoder.decode(toByteBuffer(value)).toString)
    catch { case e: CharacterCodingException ⇒ Left(e) }

  ///////////////// ITERATION ///////////////////

  def foldLeft[A](value: ByteString, z: A, f: (A, Byte) ⇒ A)  = value.foldLeft(z)(f)
  def foldRight[A](value: ByteString, z: A, f: (Byte, A) ⇒ A) = value.foldRight(z)(f)
  def foreach(value: ByteString, f: Byte ⇒ Unit)              = value.foreach(f)

} 

package swave.compat.scodec.impl

import java.io.OutputStream
import java.nio.ByteBuffer
import java.nio.charset.{CharacterCodingException, Charset}
import scala.collection.GenTraversableOnce
import scodec.bits.ByteVector
import swave.core.io.Bytes

class ByteVectorBytes extends Bytes[ByteVector] {

  ///////////////// CONSTRUCTION ///////////////////

  def empty                                               = ByteVector.empty
  def fill[A: Integral](size: Long)(byte: A)              = ByteVector.fill(size)(byte)
  def apply(array: Array[Byte])                           = ByteVector(array)
  def apply(bytes: Array[Byte], offset: Int, length: Int) = ByteVector(bytes, offset, length)
  def apply[A: Integral](bytes: A*)                       = ByteVector(bytes: _*)
  def apply(bytes: Vector[Byte])                          = ByteVector(bytes)
  def apply(buffer: ByteBuffer)                           = ByteVector(buffer)
  def apply(bs: GenTraversableOnce[Byte])                 = ByteVector(bs)
  def view(bytes: Array[Byte])                            = ByteVector(bytes)
  def view(bytes: ByteBuffer)                             = ByteVector(bytes)
  def encodeString(str: String, charset: Charset)         = if (str.isEmpty) empty else ByteVector(str getBytes charset)
  def encodeStringStrict(str: String, charset: Charset)   = ByteVector.encodeString(str)(charset)

  ///////////////// QUERY ///////////////////

  def size(value: ByteVector)                                           = value.size
  def byteAt(value: ByteVector, ix: Long)                               = value(ix)
  def indexOfSlice(value: ByteVector, slice: ByteVector, startIx: Long) = value.indexOfSlice(slice, startIx)

  ///////////////// TRANSFORMATION TO ByteVector ///////////////////

  def update(value: ByteVector, ix: Long, byte: Byte) = value.update(ix, byte)
  def concat(value: ByteVector, other: ByteVector)    = value ++ other
  def concat(value: ByteVector, byte: Byte)           = value :+ byte
  def concat(byte: Byte, value: ByteVector)           = byte +: value
  def drop(value: ByteVector, n: Long)                = value.drop(n)
  def take(value: ByteVector, n: Long)                = value.take(n)
  def map(value: ByteVector, f: Byte ⇒ Byte)          = value.map(f)
  def reverse(value: ByteVector)                      = value.reverse
  def compact(value: ByteVector)                      = value.compact

  ///////////////// TRANSFORMATION TO OTHER TYPES ///////////////////

  def toArray(value: ByteVector)                                   = value.toArray
  def copyToArray(value: ByteVector, xs: Array[Byte], offset: Int) = value.copyToArray(xs, offset)
  def copyToArray(value: ByteVector, sourceOffset: Long, xs: Array[Byte], destOffset: Int, len: Int) =
    value.copyToArray(xs, destOffset, sourceOffset, len)
  def copyToBuffer(value: ByteVector, buffer: ByteBuffer): Int = value.copyToBuffer(buffer)
  def copyToOutputStream(value: ByteVector, s: OutputStream)   = value.copyToStream(s)
  def toByteBuffer(value: ByteVector)                          = value.toByteBuffer
  def toIndexedSeq(value: ByteVector): IndexedSeq[Byte]        = value.toIndexedSeq
  def toSeq(value: ByteVector): Seq[Byte]                      = value.toSeq
  def decodeString(value: ByteVector, charset: Charset): Either[CharacterCodingException, String] =
    value.decodeString(charset)

  ///////////////// ITERATION ///////////////////

  def foldLeft[A](value: ByteVector, z: A, f: (A, Byte) ⇒ A)  = value.foldLeft(z)(f)
  def foldRight[A](value: ByteVector, z: A, f: (Byte, A) ⇒ A) = value.foldRight(z)(f)
  def foreach(value: ByteVector, f: Byte ⇒ Unit)              = value.foreach(f)

} 

package polynote.kernel.util

import java.io.OutputStream
import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets
import java.util.concurrent.atomic.AtomicBoolean

import polynote.kernel.{Output, Result}

class ResultOutputStream(publishSync: Result => Unit, bufSize: Int = 1024) extends OutputStream {
  private val buf: ByteBuffer = ByteBuffer.allocate(bufSize)
  private val closed = new AtomicBoolean(false)

  def write(b: Int): Unit = buf.synchronized {
    if (!buf.hasRemaining) {
      flush()
    }

    buf.put(b.toByte)
  }

  override def flush(): Unit = {
    super.flush()
    buf.synchronized {
      val len = buf.position()
      if (len > 0) {
        val b = ByteBuffer.allocate(buf.position())
        val arr = new Array[Byte](buf.position())
        buf.rewind()
        buf.get(arr)
        publishSync(Output("text/plain; rel=stdout", new String(arr, StandardCharsets.UTF_8)))
        buf.rewind()
      }
    }
  }

  override def close(): Unit = buf.synchronized {
    if (!closed.getAndSet(true)) {
      flush()
      super.close()
    }
  }

} 

package polynote.kernel.interpreter

import java.io.{OutputStream, PrintStream}
import java.lang.reflect.InvocationHandler

import polynote.kernel.environment.{CurrentRuntime, CurrentTask, PublishResult, PublishStatus}
import polynote.kernel.util.ResultOutputStream
import polynote.kernel.{BaseEnv, ExecutionStatus, InterpreterEnv, Output, Result, ScalaCompiler, withContextClassLoader}
import polynote.messages.CellID
import polynote.runtime.KernelRuntime
import zio.{RIO, Runtime, Task, UIO, ZIO, ZLayer}
import zio.blocking.Blocking
import zio.internal.{ExecutionMetrics, Executor}


class CellExecutor(publishSync: Result => Unit, classLoader: ClassLoader, blockingExecutor: Executor) extends Executor {
  // we have to make sure that the Java console does the same thing as the Scala console, which is thread-local
  CellExecutor.initJavaConsole

  def yieldOpCount: Int = blockingExecutor.yieldOpCount
  def metrics: Option[ExecutionMetrics] = blockingExecutor.metrics
  def submit(runnable: Runnable): Boolean = {
    blockingExecutor.submit {
      new Runnable {
        def run(): Unit = {
          val console = new PrintStream(new ResultOutputStream(publishSync), true)
          withContextClassLoader(classLoader) {
            try {
              Console.withOut(console) {
                runnable.run()
              }
            } finally {
              console.close()
            }
          }
        }
      }
    }
  }
  def here: Boolean = blockingExecutor.here
}

object CellExecutor {

  // Make sure Java's console uses the thread-local mechanism of the Scala console
  // This way it can reset properly but still obey the Console.withOut mechanism
  lazy val initJavaConsole: Unit = {
    // make sure to initialize Console
    val _ = Console.out
    val dynamicOut = new OutputStream {
      override def write(b: Int): Unit = Console.out.write(b)
    }

    System.setOut(new PrintStream(dynamicOut))
  }

  def layer(classLoader: ClassLoader): ZLayer[BaseEnv with InterpreterEnv, Throwable, Blocking] =
    ZLayer.fromEffect {
      ZIO.mapN(PublishResult.access, ZIO.runtime[Any]) {
        (publish, runtime) => ZIO.access[Blocking] {
          hasBlocking =>
            new Blocking.Service {
              override def blockingExecutor: Executor =
                new CellExecutor(
                  result => runtime.unsafeRun(publish.publish1(result)),
                  classLoader,
                  hasBlocking.get.blockingExecutor)
            }
        }
      }.flatten
    }

} 

package com.github.shadowsocks

import java.io.{IOException, InputStream, OutputStream}
import java.lang.System.currentTimeMillis
import java.util.concurrent.Semaphore

import android.util.Log

import scala.collection.JavaConversions._


class GuardedProcess(cmd: Seq[String]) extends Process {
  private val TAG = classOf[GuardedProcess].getSimpleName

  @volatile private var guardThread: Thread = _
  @volatile private var isDestroyed: Boolean = _
  @volatile private var process: Process = _
  @volatile private var isRestart = false

  def start(onRestartCallback: () => Unit = null): GuardedProcess = {
    val semaphore = new Semaphore(1)
    semaphore.acquire
    @volatile var ioException: IOException = null

    guardThread = new Thread(() => {
      try {
        var callback: () => Unit = null
        while (!isDestroyed) {
          Log.i(TAG, "start process: " + cmd)
          val startTime = currentTimeMillis

          process = new ProcessBuilder(cmd).redirectErrorStream(true).start

          if (callback == null) callback = onRestartCallback else callback()

          semaphore.release
          process.waitFor

          if (isRestart) {
            isRestart = false
          } else {
            if (currentTimeMillis - startTime < 1000) {
              Log.w(TAG, "process exit too fast, stop guard: " + cmd)
              isDestroyed = true
            }
          }
        }
      } catch {
        case ignored: InterruptedException =>
          Log.i(TAG, "thread interrupt, destroy process: " + cmd)
          process.destroy()
        case e: IOException => ioException = e
      } finally semaphore.release
    }, "GuardThread-" + cmd)

    guardThread.start()
    semaphore.acquire

    if (ioException != null) {
      throw ioException
    }

    this
  }

  def destroy() {
    isDestroyed = true
    guardThread.interrupt()
    process.destroy()
    try guardThread.join() catch {
      case ignored: InterruptedException =>
    }
  }

  def restart() {
    isRestart = true
    process.destroy()
  }

  def exitValue: Int = throw new UnsupportedOperationException
  def getErrorStream: InputStream = throw new UnsupportedOperationException
  def getInputStream: InputStream = throw new UnsupportedOperationException
  def getOutputStream: OutputStream = throw new UnsupportedOperationException

  @throws(classOf[InterruptedException])
  def waitFor = {
    guardThread.join()
    0
  }
} 

package cats.effect.internals

import java.io.{ByteArrayOutputStream, OutputStream, PrintStream}
import java.nio.charset.StandardCharsets

import scala.util.control.NonFatal


  def catchSystemErrInto[T](outStream: OutputStream)(thunk: => T): T = synchronized {
    val oldErr = System.err
    val fakeErr = new PrintStream(outStream)
    System.setErr(fakeErr)
    try {
      thunk
    } finally {
      System.setErr(oldErr)
      fakeErr.close()
    }
  }
} 

package org.apache.toree.kernel.protocol.v5.stream

import java.io.OutputStream
import java.nio.charset.Charset

import org.apache.toree.kernel.protocol.v5.content.StreamContent
import org.apache.toree.kernel.protocol.v5.{SystemActorType, MessageType, KMBuilder}
import org.apache.toree.kernel.protocol.v5.kernel.ActorLoader
import org.apache.toree.utils.{LogLike, ScheduledTaskManager}
import scala.collection.mutable.ListBuffer
import KernelOutputStream._

object KernelOutputStream {
  val DefaultStreamType = "stdout"
  val DefaultSendEmptyOutput = false
}


  override def write(b: Int): Unit = internalBytes.synchronized {
    // Begin periodic flushing if this is a new set of bytes
    enableAutoFlush()

    internalBytes += b.toByte
  }
} 

package blobstore.gcs

import java.io.OutputStream

import cats.effect.{ConcurrentEffect, Effect}
import fs2.{Chunk, Stream}
import fs2.concurrent.Queue
import cats.syntax.functor._

import scala.annotation.tailrec

private[gcs] class Fs2OutputStream[F[_]](
  queue: Queue[F, Option[Chunk[Byte]]],
  chunkSize: Int
)(implicit eff: Effect[F])
  extends OutputStream {
  @SuppressWarnings(Array("scalafix:DisableSyntax.var"))
  private var bufferedChunk: Chunk[Byte] = Chunk.empty

  @tailrec
  private def addChunk(newChunk: Chunk[Byte]): Unit = {
    val newChunkSize         = newChunk.size
    val bufferedChunkSize    = bufferedChunk.size
    val spaceLeftInTheBuffer = chunkSize - bufferedChunkSize

    if (newChunkSize > spaceLeftInTheBuffer) {
      bufferedChunk = Chunk.concatBytes(Seq(bufferedChunk, newChunk.take(spaceLeftInTheBuffer)), chunkSize)
      flushBuffer()
      addChunk(newChunk.drop(spaceLeftInTheBuffer))
    } else {
      bufferedChunk =
        Chunk.concatBytes(Seq(bufferedChunk, newChunk), bufferedChunkSize + newChunkSize)
    }
  }

  private def flushBuffer(): Unit = {
    enqueueChunkSync(Some(bufferedChunk))
    bufferedChunk = Chunk.empty
  }

  private def enqueueChunkSync(c: Option[Chunk[Byte]]): Unit =
    eff.toIO(queue.enqueue1(c)).unsafeRunSync()

  val stream: Stream[F, Byte] = queue.dequeue.unNoneTerminate.flatMap(Stream.chunk)

  override def write(bytes: Array[Byte]): Unit =
    addChunk(Chunk.bytes(bytes))
  override def write(bytes: Array[Byte], off: Int, len: Int): Unit =
    addChunk(Chunk.bytes(bytes, off, len))
  override def write(b: Int): Unit =
    addChunk(Chunk.singleton(b.toByte))

  override def flush(): Unit = flushBuffer()

  override def close(): Unit = {
    flush()
    enqueueChunkSync(None)
  }
}

private[gcs] object Fs2OutputStream {
  def apply[F[_]: ConcurrentEffect](chunkSize: Int, queueSize: Option[Int]): F[Fs2OutputStream[F]] = {
    val fQueue = queueSize match {
      case None       => Queue.unbounded[F, Option[Chunk[Byte]]]
      case Some(size) => Queue.bounded[F, Option[Chunk[Byte]]](size)
    }
    fQueue.map { queue => new Fs2OutputStream[F](queue, chunkSize) }
  }
} 

package wvlet.airframe.msgpack.spi
import java.io.{InputStream, OutputStream}

import wvlet.airframe.msgpack.impl.{PureScalaBufferPacker, PureScalaBufferUnpacker}
import wvlet.airframe.msgpack.io.ByteArrayBuffer


object Compat {
  def isScalaJS = true

  // Javascript has no NaN in float/double values
  // See https://github.com/scala-js/scala-js/issues/2327
  def floatToIntBits(v: Float): Int     = java.lang.Float.floatToIntBits(v)
  def doubleToLongBits(v: Double): Long = java.lang.Double.doubleToLongBits(v)

  def newBufferPacker: BufferPacker = {
    new PureScalaBufferPacker
  }
  def newPacker(out: OutputStream): Packer   = ???
  def newUnpacker(in: InputStream): Unpacker = ???
  def newUnpacker(msgpack: Array[Byte]): Unpacker = {
    newUnpacker(msgpack, 0, msgpack.length)
  }
  def newUnpacker(msgpack: Array[Byte], offset: Int, len: Int): Unpacker = {
    new PureScalaBufferUnpacker(ByteArrayBuffer.fromArray(msgpack, offset, len))
  }
}