java.nio.CharBuffer Scala Examples

package org.apache.daffodil.io

import java.nio.ByteBuffer
import java.nio.CharBuffer

import org.junit.Assert._
import org.junit.Test

import org.apache.daffodil.processors.charset.CharsetUtils
import org.apache.daffodil.util.Misc

class TestNonByteSizedCharsetEncoders8Bit {

  @Test def test8BitMSBF_01(): Unit = {
    val cs = CharsetUtils.getCharset("X-DFDL-ISO-88591-8-BIT-PACKED-MSB-FIRST")
    val encoder = cs.newEncoder()
    val cb = CharBuffer.wrap("01234567")
    val expectedBytes = Misc.bits2Bytes("00110000 00110001 00110010 00110011 00110100 00110101 00110110 00110111").toList
    val bb = ByteBuffer.allocate(8)
    val res = encoder.encode(cb, bb, false)
    assertTrue(res.isUnderflow())
    bb.flip()
    val actualBytes = bb.array().toList
    assertEquals(expectedBytes, actualBytes)
  }

  @Test def test8BitMSBF_02(): Unit = {
    val cs = CharsetUtils.getCharset("X-DFDL-ISO-88591-8-BIT-PACKED-MSB-FIRST")
    val encoder = cs.newEncoder()
    val cb = CharBuffer.wrap("012345677")
    val expectedBytes = Misc.bits2Bytes("00110000 00110001 00110010 00110011 00110100 00110101 00110110 00110111").toList
    val bb = ByteBuffer.allocate(8) // not enough space for last digit
    val res = encoder.encode(cb, bb, false)
    assertTrue(res.isOverflow())
    bb.flip()
    val actualBytes = bb.array().toList
    assertEquals(expectedBytes, actualBytes)
  }

  @Test def test8BitLSBF_01(): Unit = {
    val cs = CharsetUtils.getCharset("X-DFDL-ISO-88591-8-BIT-PACKED-LSB-FIRST")
    val encoder = cs.newEncoder()
    val cb = CharBuffer.wrap("01234567")
    val expectedBytes = Misc.bits2Bytes("00110000 00110001 00110010 00110011 00110100 00110101 00110110 00110111").toList
    val bb = ByteBuffer.allocate(8)
    val res = encoder.encode(cb, bb, false)
    assertTrue(res.isUnderflow())
    bb.flip()
    val actualBytes = bb.array().toList
    assertEquals(expectedBytes, actualBytes)
  }

  @Test def test8BitLSBF_02(): Unit = {
    val cs = CharsetUtils.getCharset("X-DFDL-ISO-88591-8-BIT-PACKED-LSB-FIRST")
    val encoder = cs.newEncoder()
    assertNotNull(encoder)
    val cb = CharBuffer.wrap("012345677")
    val expectedBytes = Misc.bits2Bytes("00110000 00110001 00110010 00110011 00110100 00110101 00110110 00110111").toList
    val bb = ByteBuffer.allocate(8) // not enough space for last digit
    val res = encoder.encode(cb, bb, false)
    assertTrue(res.isOverflow())
    bb.flip()
    val actualBytes = bb.array().toList
    assertEquals(expectedBytes, actualBytes)
  }
} 

package isabelle


import java.nio.Buffer
import java.nio.{ByteBuffer, CharBuffer}
import java.nio.charset.{Charset, CharsetDecoder, CharsetEncoder, CoderResult}
import java.nio.charset.spi.CharsetProvider


object Isabelle_Charset
{
  val name: String = "UTF-8-Isabelle-test"  // FIXME
  lazy val charset: Charset = new Isabelle_Charset
}


class Isabelle_Charset extends Charset(Isabelle_Charset.name, null)
{
  override def contains(cs: Charset): Boolean =
    cs.name.equalsIgnoreCase(UTF8.charset_name) || UTF8.charset.contains(cs)

  override def newDecoder(): CharsetDecoder = UTF8.charset.newDecoder

  override def newEncoder(): CharsetEncoder = UTF8.charset.newEncoder
}


class Isabelle_Charset_Provider extends CharsetProvider
{
  override def charsetForName(name: String): Charset =
  {
    // FIXME inactive
    // if (name.equalsIgnoreCase(Isabelle_Charset.name)) Isabelle_Charset.charset
    // else null
    null
  }

  override def charsets(): java.util.Iterator[Charset] =
  {
    import scala.collection.JavaConversions._
    // FIXME inactive
    // Iterator(Isabelle_Charset.charset)
    Iterator()
  }
} 

package isabelle


import java.nio.Buffer
import java.nio.{ByteBuffer, CharBuffer}
import java.nio.charset.{Charset, CharsetDecoder, CharsetEncoder, CoderResult}
import java.nio.charset.spi.CharsetProvider


object Isabelle_Charset
{
  val name: String = "UTF-8-Isabelle-test"  // FIXME
  lazy val charset: Charset = new Isabelle_Charset
}


class Isabelle_Charset extends Charset(Isabelle_Charset.name, null)
{
  override def contains(cs: Charset): Boolean =
    cs.name.equalsIgnoreCase(UTF8.charset_name) || UTF8.charset.contains(cs)

  override def newDecoder(): CharsetDecoder = UTF8.charset.newDecoder

  override def newEncoder(): CharsetEncoder = UTF8.charset.newEncoder
}


class Isabelle_Charset_Provider extends CharsetProvider
{
  override def charsetForName(name: String): Charset =
  {
    // FIXME inactive
    // if (name.equalsIgnoreCase(Isabelle_Charset.name)) Isabelle_Charset.charset
    // else null
    null
  }

  override def charsets(): java.util.Iterator[Charset] =
  {
    import scala.collection.JavaConversions._
    // FIXME inactive
    // Iterator(Isabelle_Charset.charset)
    Iterator()
  }
} 

package isabelle


import java.nio.Buffer
import java.nio.{ByteBuffer, CharBuffer}
import java.nio.charset.{Charset, CharsetDecoder, CharsetEncoder, CoderResult}
import java.nio.charset.spi.CharsetProvider


object Isabelle_Charset
{
  val name: String = "UTF-8-Isabelle-test"  // FIXME
  lazy val charset: Charset = new Isabelle_Charset
}


class Isabelle_Charset extends Charset(Isabelle_Charset.name, null)
{
  override def contains(cs: Charset): Boolean =
    cs.name.equalsIgnoreCase(UTF8.charset_name) || UTF8.charset.contains(cs)

  override def newDecoder(): CharsetDecoder = UTF8.charset.newDecoder

  override def newEncoder(): CharsetEncoder = UTF8.charset.newEncoder
}


class Isabelle_Charset_Provider extends CharsetProvider
{
  override def charsetForName(name: String): Charset =
  {
    // FIXME inactive
    // if (name.equalsIgnoreCase(Isabelle_Charset.name)) Isabelle_Charset.charset
    // else null
    null
  }

  override def charsets(): java.util.Iterator[Charset] =
  {
    import scala.collection.JavaConversions._
    // FIXME inactive
    // Iterator(Isabelle_Charset.charset)
    Iterator()
  }
} 

package almond.internals

import java.io.{OutputStream, PrintStream}
import java.nio.{ByteBuffer, CharBuffer}
import java.nio.charset.{Charset, CoderResult}

class FunctionOutputStream(
  inputBufferSize: Int,
  outputBufferSize: Int,
  internalCharset: Charset,
  f: String => Unit
) extends OutputStream {

  // not thread-safe

  private val decoder = internalCharset.newDecoder()

  private val inArray = Array.ofDim[Byte](inputBufferSize)
  private val outArray = Array.ofDim[Char](outputBufferSize)

  private val writeBuf = ByteBuffer.wrap(inArray)
  private val out = CharBuffer.wrap(outArray)

  private def flushIfNeeded(): Unit =
    if (!writeBuf.hasRemaining)
      flush()

  def write(b: Int): Unit = {
    writeBuf.put(b.toByte) // hope toByte doesn't box b
    flushIfNeeded()
  }

  override def write(b: Array[Byte], off: Int, len: Int) = {
    var off0 = off
    var len0 = len
    while (len0 > 0) {
      val take = math.min(len0, writeBuf.remaining())
      assert(take > 0)
      writeBuf.put(b, off0, take)
      off0 = off0 + take
      len0 = len0 - take
      flushIfNeeded()
    }
    assert(len0 == 0)
    assert(off0 == off + len)
  }

  override def flush(): Unit = {
    super.flush()

    val readBuf = ByteBuffer.wrap(inArray, 0, writeBuf.position())

    var r: CoderResult = null

    while (r == null || r.isOverflow) {
      if (r != null) {
        readBuf.position(0)
        readBuf.limit(writeBuf.position())
      }

      r = decoder.decode(readBuf, out, false)

      val outLen = out.position()

      if (r.isError || (r.isOverflow && outLen == 0))
        r.throwException()
      else {
        if (outLen > 0) {
          val s = new String(outArray, 0, outLen)
          out.clear()
          f(s)
        }
        val read = readBuf.position()
        val avail = writeBuf.position()
        val remaining = avail - read
        writeBuf.position(remaining)
        if (remaining > 0)
          System.arraycopy(inArray, read, inArray, 0, remaining)
      }
    }
  }

  def printStream(): PrintStream =
    new PrintStream(this, true, internalCharset.name())

} 

package loci

import java.nio.charset.{CharacterCodingException, CodingErrorAction, StandardCharsets}
import java.nio.{ByteBuffer, CharBuffer}

import scala.util.{Failure, Success, Try}

private object MessageBufferEncoding {
  locally(MessageBufferEncoding)

  def byteBufferToString(byteBuffer: ByteBuffer, offset: Int, count: Int, fatal: Boolean): Try[String] = {
    val decoder = StandardCharsets.UTF_8.newDecoder()

    if (!fatal)
      decoder
        .onMalformedInput(CodingErrorAction.REPLACE)
        .onUnmappableCharacter(CodingErrorAction.REPLACE)

    val size = (count * decoder.maxCharsPerByte.toDouble).toInt
    val array = new Array[Char](size)
    val charBuffer = CharBuffer.wrap(array)

    byteBuffer.position(offset)
    byteBuffer.limit(offset + count)

    try {
      var result = decoder.decode(byteBuffer, charBuffer, true)

      if (!result.isUnderflow)
        result.throwException()
      result = decoder.flush(charBuffer)
      if (!result.isUnderflow)
        result.throwException()

      Success(new String(array, 0, charBuffer.position()))
    }
    catch {
      case exception: CharacterCodingException =>
        Failure(exception)
    }
  }

  def stringToByteBuffer(string: String)(allocateByteBuffer: Int => ByteBuffer): ByteBuffer = {
    val encoder = StandardCharsets.UTF_8.newEncoder()
      .onMalformedInput(CodingErrorAction.REPLACE)
      .onUnmappableCharacter(CodingErrorAction.REPLACE)

    val size = (string.length * encoder.maxBytesPerChar.toDouble).toInt
    val byteBuffer = allocateByteBuffer(size)
    val charBuffer = CharBuffer.wrap(string)

    byteBuffer.position(0)
    byteBuffer.limit(size)

    var result = encoder.encode(charBuffer, byteBuffer, true)

    if (!result.isUnderflow)
      result.throwException()
    result = encoder.flush(byteBuffer)
    if (!result.isUnderflow)
      result.throwException()

    byteBuffer
  }

  private val hex = Array(
    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F')

  def messageBufferToHexString(buffer: MessageBuffer): String = {
    val result = new Array[Char](3 * buffer.length)
    var i = 0
    var j = 0
    while (i < buffer.length) {
      result(j) = hex((buffer(i) & 0xF0) >> 4)
      j += 1
      result(j) = hex(buffer(i) & 0x0F)
      j += 1
      result(j) = ' '
      j += 1
      i += 1
    }

    if (result.isEmpty)
      ""
    else
      new String(result, 0, result.length - 1)
  }
} 

package swave.compat.akka.impl

import java.io.OutputStream
import java.nio.charset.{CharacterCodingException, Charset}
import java.nio.{ByteBuffer, CharBuffer}
import java.util
import scala.annotation.tailrec
import scala.collection.GenTraversableOnce
import akka.util.ByteString
import swave.core.io.Bytes
import swave.core.macros._

class ByteStringBytes extends Bytes[ByteString] {

  ///////////////// CONSTRUCTION ///////////////////

  def empty = ByteString.empty
  def fill[A: Integral](size: Long)(byte: A) = {
    requireArg(0 <= size && size <= Int.MaxValue, "`size` must be >= 0 and <= Int.MaxValue")
    val b = implicitly[Integral[A]].toInt(byte).toByte
    apply(Array.fill(size.toInt)(b))
  }
  def apply(array: Array[Byte]) = ByteString(array)
  def apply(bytes: Array[Byte], offset: Int, length: Int) =
    ByteString(util.Arrays.copyOfRange(bytes, offset, offset + length))
  def apply[A: Integral](bytes: A*) = {
    val integral = implicitly[Integral[A]]
    val buf      = new Array[Byte](bytes.size)
    @tailrec def rec(ix: Int): ByteString =
      if (ix < buf.length) {
        buf(ix) = integral.toInt(bytes(ix)).toByte
        rec(ix + 1)
      } else view(buf)
    rec(0)
  }
  def apply(bytes: Vector[Byte])                  = ByteString(bytes: _*)
  def apply(buffer: ByteBuffer)                   = ByteString(buffer)
  def apply(bs: GenTraversableOnce[Byte])         = ByteString(bs.toArray)
  def view(bytes: Array[Byte])                    = ByteString(bytes) // no view-like constructor available on ByteStrings
  def view(bytes: ByteBuffer)                     = ByteString(bytes) // no view-like constructor available on ByteStrings
  def encodeString(str: String, charset: Charset) = ByteString(str, charset.name)
  def encodeStringStrict(str: String, charset: Charset) =
    try Right(ByteString(charset.newEncoder.encode(CharBuffer.wrap(str))))
    catch { case e: CharacterCodingException ⇒ Left(e) }

  ///////////////// QUERY ///////////////////

  def size(value: ByteString): Long = value.size.toLong
  def byteAt(value: ByteString, ix: Long): Byte = {
    requireArg(0 <= ix && ix <= Int.MaxValue, "`ix` must be >= 0 and <= Int.MaxValue")
    value(ix.toInt)
  }
  def indexOfSlice(value: ByteString, slice: ByteString, startIx: Long): Long = {
    requireArg(0 <= startIx && startIx <= Int.MaxValue, "`startIx` must be >= 0 and <= Int.MaxValue")
    value.indexOfSlice(slice, startIx.toInt).toLong
  }

  ///////////////// TRANSFORMATION TO ByteString ///////////////////

  def update(value: ByteString, ix: Long, byte: Byte) = concat(concat(take(value, ix), byte), drop(value, ix + 1))
  def concat(value: ByteString, other: ByteString)    = value ++ other
  def concat(value: ByteString, byte: Byte)           = value ++ ByteString(byte)
  def concat(byte: Byte, value: ByteString)           = ByteString(byte) ++ value
  def drop(value: ByteString, n: Long) = {
    requireArg(0 <= n && n <= Int.MaxValue, "`n` must be >= 0 and <= Int.MaxValue")
    value.drop(n.toInt)
  }
  def take(value: ByteString, n: Long) = {
    requireArg(0 <= n && n <= Int.MaxValue, "`n` must be >= 0 and <= Int.MaxValue")
    value.take(n.toInt)
  }
  def map(value: ByteString, f: Byte ⇒ Byte) = value.map(f)
  def reverse(value: ByteString)             = value.reverse
  def compact(value: ByteString)             = value.compact

  ///////////////// TRANSFORMATION TO OTHER TYPES ///////////////////

  def toArray(value: ByteString)                                   = value.toArray
  def copyToArray(value: ByteString, xs: Array[Byte], offset: Int) = value.copyToArray(xs, offset)
  def copyToArray(value: ByteString, sourceOffset: Long, xs: Array[Byte], destOffset: Int, len: Int) =
    drop(value, sourceOffset).copyToArray(xs, destOffset, len)
  def copyToBuffer(value: ByteString, buffer: ByteBuffer): Int = value.copyToBuffer(buffer)
  def copyToOutputStream(value: ByteString, s: OutputStream) = {
    @tailrec def rec(ix: Int, size: Int): Unit = if (ix < size) { s.write(value(ix).toInt); rec(ix + 1, size) }
    rec(0, value.size)
  }
  def toByteBuffer(value: ByteString)                   = value.toByteBuffer
  def toIndexedSeq(value: ByteString): IndexedSeq[Byte] = value
  def toSeq(value: ByteString): Seq[Byte]               = value
  def decodeString(value: ByteString, charset: Charset): Either[CharacterCodingException, String] =
    try Right(charset.newDecoder.decode(toByteBuffer(value)).toString)
    catch { case e: CharacterCodingException ⇒ Left(e) }

  ///////////////// ITERATION ///////////////////

  def foldLeft[A](value: ByteString, z: A, f: (A, Byte) ⇒ A)  = value.foldLeft(z)(f)
  def foldRight[A](value: ByteString, z: A, f: (Byte, A) ⇒ A) = value.foldRight(z)(f)
  def foreach(value: ByteString, f: Byte ⇒ Unit)              = value.foreach(f)

} 

package better.files

import java.io.{OutputStream, Writer}
import java.nio.charset.{Charset, CharsetDecoder, CodingErrorAction}
import java.nio.{ByteBuffer, CharBuffer}

import scala.annotation.tailrec


  private[this] val decoderIn = ByteBuffer.allocate(bufferSize >> 4)

  def this(
      writer: Writer,
      bufferSize: Int = DefaultBufferSize,
      flushImmediately: Boolean = false
  )(implicit
      charset: Charset = DefaultCharset
  ) =
    this(
      writer = writer,
      decoder = charset.newDecoder
        .onMalformedInput(CodingErrorAction.REPLACE)
        .onUnmappableCharacter(CodingErrorAction.REPLACE)
        .replaceWith("?"),
      bufferSize = bufferSize,
      flushImmediately = flushImmediately
    )

  override def write(b: Array[Byte], off: Int, len: Int) = {
    @tailrec def loop(off: Int, len: Int): Unit =
      if (len > 0) {
        val c = decoderIn.remaining min len
        decoderIn.put(b, off, c)
        processInput(endOfInput = false)
        loop(off + c, len - c)
      }
    loop(off, len)
    if (flushImmediately) flushOutput()
  }

  override def write(b: Int) = write(Array(b.toByte))

  override def flush() = {
    flushOutput()
    writer.flush()
  }

  override def close() = {
    processInput(endOfInput = true)
    flushOutput()
    writer.close()
  }

  private[this] def processInput(endOfInput: Boolean) = {
    decoderIn.flip()
    @tailrec def loop(): Unit = {
      val coderResult = decoder.decode(decoderIn, decoderOut, endOfInput)
      if (coderResult.isOverflow) {
        flushOutput()
        loop()
      } else {
        assert(coderResult.isUnderflow, "decoder is configured to replace malformed input and unmappable characters")
      }
    }
    loop()
    decoderIn.compact()
  }

  private[this] def flushOutput(): Unit = {
    val p = decoderOut.position()
    if (p > 0) {
      writer.write(decoderOut.array, 0, p)
      val _ = decoderOut.rewind()
    }
  }
} 

package better.files

import java.io.{InputStream, Reader}
import java.nio.{ByteBuffer, CharBuffer}
import java.nio.charset.{Charset, CharsetEncoder, CoderResult, CodingErrorAction}

import scala.annotation.tailrec


  private[this] val encoderOut = ByteBuffer.allocate(bufferSize >> 4).flip().asInstanceOf[ByteBuffer]

  private[this] var lastCoderResult = CoderResult.UNDERFLOW
  private[this] var endOfInput      = false

  private[this] def fillBuffer() = {
    assert(!endOfInput)
    if (lastCoderResult.isUnderflow) {
      val position = encoderIn.compact().position()
      //  We don't use Reader#read(CharBuffer) here because it is more efficient to write directly to the underlying char array
      // since the default implementation copies data to a temporary char array anyway
      reader.read(encoderIn.array, position, encoderIn.remaining) match {
        case EOF => endOfInput = true
        case c   => encoderIn.position(position + c)
      }
      encoderIn.flip()
    }
    lastCoderResult = encoder.encode(encoderIn, encoderOut.compact(), endOfInput)
    encoderOut.flip()
  }

  override def read(b: Array[Byte], off: Int, len: Int) = {
    if (len < 0 || off < 0 || (off + len) > b.length)
      throw new IndexOutOfBoundsException("Array Size=" + b.length + ", offset=" + off + ", length=" + len)
    if (len == 0) {
      0 // Always return 0 if len == 0
    } else {
      var read = 0
      @tailrec def loop(off: Int, len: Int): Unit =
        if (len > 0) {
          if (encoderOut.hasRemaining) {
            val c = encoderOut.remaining min len
            encoderOut.get(b, off, c)
            read += c
            loop(off + c, len - c)
          } else if (!endOfInput) {
            fillBuffer()
            loop(off, len)
          }
        }
      loop(off, len)
      if (read == 0 && endOfInput) EOF else read
    }
  }

  @tailrec final override def read() = {
    if (encoderOut.hasRemaining) {
      encoderOut.get & 0xff
    } else if (endOfInput) {
      EOF
    } else {
      fillBuffer()
      read()
    }
  }

  override def close() = reader.close()
} 

package org.apache.daffodil.io

import java.nio.ByteBuffer
import java.nio.CharBuffer

import org.junit.Assert._
import org.junit.Test

import org.apache.daffodil.processors.charset.CharsetUtils
import org.apache.daffodil.util.Misc

class TestNonByteSizedCharsetEncoders1Bit {

  @Test def test1BitMSBF_01(): Unit = {
    val cs1Bit = CharsetUtils.getCharset("X-DFDL-BITS-MSBF")
    val encoder = cs1Bit.newEncoder()
    val cb = CharBuffer.wrap(Misc.hex2Bits("DEADBEEF"))
    val bb = ByteBuffer.allocate(4)
    val expectedBytes = Misc.hex2Bytes("DEADBEEF").toList
    val res = encoder.encode(cb, bb, false)
    assertTrue(res.isUnderflow())
    bb.flip()
    val actualBytes = bb.array().toList
    assertEquals(expectedBytes, actualBytes)
  }

  @Test def test1BitMSBF_02(): Unit = {
    val cs1Bit = CharsetUtils.getCharset("X-DFDL-BITS-MSBF")
    val encoder = cs1Bit.newEncoder()
    val cb = CharBuffer.wrap(Misc.hex2Bits("DEADBEEF57"))
    val bb = ByteBuffer.allocate(4)
    val expectedBytes = Misc.hex2Bytes("DEADBEEF").toList
    val res = encoder.encode(cb, bb, false)
    assertTrue(res.isOverflow())
    bb.flip()
    val actualBytes = bb.array().toList
    assertEquals(expectedBytes, actualBytes)
  }

  @Test def test1BitLSBF_01(): Unit = {
    val cs1Bit = CharsetUtils.getCharset("X-DFDL-BITS-LSBF")
    val encoder = cs1Bit.newEncoder()
    val cb = CharBuffer.wrap(Misc.hex2Bits("7BB57DF7"))
    val bb = ByteBuffer.allocate(4)
    val expectedBytes = Misc.hex2Bytes("DEADBEEF").toList
    val res = encoder.encode(cb, bb, false)
    assertTrue(res.isUnderflow())
    bb.flip()
    val actualBytes = bb.array().toList
    assertEquals(expectedBytes, actualBytes)
  }

  @Test def test1BitLSBF_02(): Unit = {
    val cs1Bit = CharsetUtils.getCharset("X-DFDL-BITS-LSBF")
    val encoder = cs1Bit.newEncoder()
    val cb = CharBuffer.wrap(Misc.hex2Bits("7BB57DF757"))
    val bb = ByteBuffer.allocate(4)
    val expectedBytes = Misc.hex2Bytes("DEADBEEF").toList
    val res = encoder.encode(cb, bb, false)
    assertTrue(res.isOverflow())
    bb.flip()
    val actualBytes = bb.array().toList
    assertEquals(expectedBytes, actualBytes)
  }

} 

package coinyser

import java.io.{BufferedOutputStream, StringReader}
import java.nio.CharBuffer
import java.sql.Timestamp

import cats.effect.IO
import org.apache.spark.sql._
import org.apache.spark.sql.test.SharedSparkSession
import org.scalatest.{Matchers, WordSpec}


class BatchProducerSpec extends WordSpec with Matchers with SharedSparkSession {

  val httpTransaction1 = HttpTransaction("1532365695", "70683282", "7740.00", "0", "0.10041719")
  val httpTransaction2 = HttpTransaction("1532365693", "70683281", "7739.99", "0", "0.00148564")

  "BatchProducer.jsonToHttpTransaction" should {
    "create a Dataset[HttpTransaction] from a Json string" in {
      val json =
        """[{"date": "1532365695", "tid": "70683282", "price": "7740.00", "type": "0", "amount": "0.10041719"},
          |{"date": "1532365693", "tid": "70683281", "price": "7739.99", "type": "0", "amount": "0.00148564"}]""".stripMargin

      val ds: Dataset[HttpTransaction] = BatchProducer.jsonToHttpTransactions(json)
      ds.collect() should contain theSameElementsAs Seq(httpTransaction1, httpTransaction2)
    }
  }

  "BatchProducer.httpToDomainTransactions" should {
    "transform a Dataset[HttpTransaction] into a Dataset[Transaction]" in {
      import testImplicits._
      val source: Dataset[HttpTransaction] = Seq(httpTransaction1, httpTransaction2).toDS()
      val target: Dataset[Transaction] = BatchProducer.httpToDomainTransactions(source)
      val transaction1 = Transaction(timestamp = new Timestamp(1532365695000L), tid = 70683282, price = 7740.00, sell = false, amount = 0.10041719)
      val transaction2 = Transaction(timestamp = new Timestamp(1532365693000L), tid = 70683281, price = 7739.99, sell = false, amount = 0.00148564)

      target.collect() should contain theSameElementsAs Seq(transaction1, transaction2)
    }
  }

} 

package org.apache.daffodil.io

import java.nio.ByteBuffer
import java.nio.CharBuffer

import org.junit.Assert._
import org.junit.Test

import org.apache.daffodil.processors.charset.CharsetUtils
import org.apache.daffodil.util.Misc

class TestNonByteSizedCharsetEncoders3Bit {

  @Test def test3BitMSBF_01(): Unit = {
    val cs = CharsetUtils.getCharset("X-DFDL-OCTAL-MSBF")
    val encoder = cs.newEncoder()
    val cb = CharBuffer.wrap("01234567")
    val expectedBytes = Misc.bits2Bytes("000 001 010 011 100 101 110 111").toList
    val bb = ByteBuffer.allocate(3)
    val res = encoder.encode(cb, bb, false)
    assertTrue(res.isUnderflow())
    bb.flip()
    val actualBytes = bb.array().toList
    assertEquals(expectedBytes, actualBytes)
  }

  @Test def test3BitMSBF_02(): Unit = {
    val cs = CharsetUtils.getCharset("X-DFDL-OCTAL-MSBF")
    val encoder = cs.newEncoder()
    val cb = CharBuffer.wrap("012345677")
    val expectedBytes = Misc.bits2Bytes("000 001 010 011 100 101 110 111").toList
    val bb = ByteBuffer.allocate(3) // not enough space for last digit
    val res = encoder.encode(cb, bb, false)
    assertTrue(res.isOverflow())
    bb.flip()
    val actualBytes = bb.array().toList
    assertEquals(expectedBytes, actualBytes)
  }

  @Test def test3BitLSBF_01(): Unit = {
    val cs = CharsetUtils.getCharset("X-DFDL-OCTAL-LSBF")
    val encoder = cs.newEncoder()
    val cb = CharBuffer.wrap("01234567")
    val expectedBytes = Misc.bits2Bytes("10 001 000  1 100 011 0  111 110 10").toList
    val bb = ByteBuffer.allocate(3)
    val res = encoder.encode(cb, bb, false)
    assertTrue(res.isUnderflow())
    bb.flip()
    val actualBytes = bb.array().toList
    assertEquals(expectedBytes, actualBytes)
  }

  @Test def test3BitLSBF_02(): Unit = {
    val cs = CharsetUtils.getCharset("X-DFDL-OCTAL-LSBF")
    val encoder = cs.newEncoder()
    assertNotNull(encoder)
    val cb = CharBuffer.wrap("012345677")
    val expectedBytes = Misc.bits2Bytes("10 001 000  1 100 011 0  111 110 10").toList
    val bb = ByteBuffer.allocate(3) // not enough space for last digit
    val res = encoder.encode(cb, bb, false)
    assertTrue(res.isOverflow())
    bb.flip()
    val actualBytes = bb.array().toList
    assertEquals(expectedBytes, actualBytes)
  }

} 

package org.apache.daffodil.io

import java.nio.charset.CodingErrorAction
import java.nio.CharBuffer
import java.nio.LongBuffer

import org.apache.daffodil.api.DaffodilTunables
import org.apache.daffodil.util.Maybe
import org.apache.daffodil.schema.annotation.props.gen.BitOrder
import org.apache.daffodil.schema.annotation.props.gen.BinaryFloatRep
import org.apache.daffodil.util.MaybeInt
import org.apache.daffodil.schema.annotation.props.gen.ByteOrder
import org.apache.daffodil.schema.annotation.props.gen.UTF16Width
import org.apache.daffodil.schema.annotation.props.gen.EncodingErrorPolicy
import org.apache.daffodil.processors.charset.BitsCharset
import org.apache.daffodil.processors.charset.BitsCharsetDecoder
import org.apache.daffodil.processors.charset.BitsCharsetEncoder
import org.apache.daffodil.processors.charset.BitsCharsetNonByteSize
import org.apache.daffodil.processors.charset.StandardBitsCharsets

object FormatInfoForUnitTest {
  def apply() = {
    val obj = new FormatInfoForUnitTest()
    obj.init()
    obj
  }
}

class FormatInfoForUnitTest private ()
  extends FormatInfo {
  var priorEncoding: BitsCharset = StandardBitsCharsets.UTF_8

  var encoder: BitsCharsetEncoder = priorEncoding.newEncoder()
  var decoder: BitsCharsetDecoder = priorEncoding.newDecoder()

  var byteOrder: ByteOrder = ByteOrder.BigEndian
  var bitOrder: BitOrder = BitOrder.MostSignificantBitFirst
  var fillByte: Byte = 0x00.toByte
  var binaryFloatRep: BinaryFloatRep = BinaryFloatRep.Ieee
  var maybeCharWidthInBits: MaybeInt = MaybeInt.Nope
  var maybeUTF16Width: Maybe[UTF16Width] = Maybe.Nope
  var encodingMandatoryAlignmentInBits: Int = 8
  var encodingErrorPolicy: EncodingErrorPolicy = EncodingErrorPolicy.Replace
  var tunable: DaffodilTunables = DaffodilTunables()
  var regexMatchBuffer = CharBuffer.allocate(1024)
  var regexMatchBitPositionBuffer = LongBuffer.allocate(1024)

  def reset(cs: BitsCharset): Unit = {
    priorEncoding = cs
    init()
  }

  def init(): Unit = {
    encoder = priorEncoding.newEncoder()
    encoder.onMalformedInput(CodingErrorAction.REPLACE)
    encoder.onUnmappableCharacter(CodingErrorAction.REPLACE)
    decoder = priorEncoding.newDecoder()
    priorEncoding match {
      case decoderWithBits: BitsCharsetNonByteSize => {
        encodingMandatoryAlignmentInBits = 1
        maybeCharWidthInBits = MaybeInt(decoderWithBits.bitWidthOfACodeUnit)
      }
      case _ => {
        encodingMandatoryAlignmentInBits = 8
        val maxBytes = encoder.maxBytesPerChar()
        if (maxBytes == encoder.averageBytesPerChar()) {
          maybeCharWidthInBits = MaybeInt((maxBytes * 8).toInt)
        } else {
          maybeCharWidthInBits = MaybeInt.Nope
        }
      }
    }
  }
} 

package org.apache.daffodil.io

import java.nio.ByteBuffer
import java.nio.CharBuffer

import org.junit.Assert._
import org.junit.Test

import org.apache.daffodil.processors.charset.CharsetUtils
import org.apache.daffodil.util.Misc
import org.apache.daffodil.schema.annotation.props.gen.ByteOrder
import org.apache.daffodil.schema.annotation.props.gen.BitOrder

class TestNonByteSizedCharsetDecoders8Bit {

  val lsbfFinfo = FormatInfoForUnitTest()
  lsbfFinfo.byteOrder = ByteOrder.BigEndian
  lsbfFinfo.bitOrder = BitOrder.LeastSignificantBitFirst

  val msbfFinfo = FormatInfoForUnitTest()
  msbfFinfo.byteOrder = ByteOrder.BigEndian
  msbfFinfo.bitOrder = BitOrder.MostSignificantBitFirst


  @Test def test8BitMSBF_01(): Unit = {
    val cs = CharsetUtils.getCharset("X-DFDL-ISO-88591-8-BIT-PACKED-MSB-FIRST")
    val decoder = cs.newDecoder()
    val cb = CharBuffer.allocate(8)
    val bb = ByteBuffer.wrap(Misc.bits2Bytes("00110000 00110001 00110010 00110011 00110100 00110101 00110110 00110111"))
    val dis = InputSourceDataInputStream(bb)
    val res = decoder.decode(dis, msbfFinfo, cb)
    assertEquals(8, res)
    cb.flip()
    val digits = cb.toString()
    assertEquals("01234567", digits)
  }

  @Test def test8BitMSBF_02(): Unit = {
    val cs = CharsetUtils.getCharset("X-DFDL-ISO-88591-8-BIT-PACKED-MSB-FIRST")
    val decoder = cs.newDecoder()
    val cb = CharBuffer.allocate(7) // not enough space for last digit
    val bb = ByteBuffer.wrap(Misc.bits2Bytes("00110000 00110001 00110010 00110011 00110100 00110101 00110110 00110111"))
    val dis = InputSourceDataInputStream(bb)
    //dis.skip(4, msbfFinfo)
    val res = decoder.decode(dis, msbfFinfo, cb)
    assertEquals(7, res)
    cb.flip()
    val digits = cb.toString()
    assertEquals("0123456", digits)
  }

  @Test def test8BitLSBF_01(): Unit = {
    val cs = CharsetUtils.getCharset("X-DFDL-ISO-88591-8-BIT-PACKED-LSB-FIRST")
    val decoder = cs.newDecoder()
    val cb = CharBuffer.allocate(64)
    val bb = ByteBuffer.wrap(Misc.bits2Bytes("00110000 00110001 00110010 00110011 00110100 00110101 00110110 00110111"))
    val dis = InputSourceDataInputStream(bb)
    val res = decoder.decode(dis, lsbfFinfo, cb)
    assertEquals(8, res)
    cb.flip()
    val digits = cb.toString()
    assertEquals("01234567", digits)
  }
} 

package org.apache.daffodil.io

import org.junit.Test
import org.junit.Assert._
import org.apache.daffodil.xml.XMLUtils
import java.nio.ByteBuffer
import java.nio.CharBuffer
import org.apache.daffodil.util.Misc

class TestDumpDisplay {
  val dfdl = XMLUtils.DFDL_NAMESPACE
  val tdml = XMLUtils.TDML_NAMESPACE
  val xsd = XMLUtils.XSD_NAMESPACE
  val tns = XMLUtils.EXAMPLE_NAMESPACE
  val xsi = XMLUtils.XSI_NAMESPACE
  
  @Test def testAllPrintableChars() = {

    // val bytes = 0 to 255 map { _.toByte }
    val bb = ByteBuffer.allocate(256)
    0 to 255 foreach { n => bb.put(n, n.toByte) }
    val cb = CharBuffer.allocate(256)
    Misc.remapBytesToVisibleGlyphs(bb, cb)
    val res = cb.toString
    val exp =
      // 
      // C0 Controls - use unicode control picture characters.
      "␀␁␂␃␄␅␆␇␈␉␊␋␌␍␎␏␐␑␒␓␔␕␖␗␘␙␚␛␜␝␞␟" +
        "␣" + // space 
        """!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~""" + // ascii glyph
        "\u2421" + // DEL
        //
        // C1 controls, which windows 1252 mostly has glyphs for
        // These are the equivalent unicode characters. The ones that windows-1252 has no glyph or are whitespace
        // are remapped to other characters noted 
        "€" +
        "Ɓ" + // 0x81 mapped to Ɓ
        "‚ƒ„…†‡ˆ‰Š‹Œ" +
        "ƍ" + // 0x8D mapped to ƍ
        "Ž" +
        "Ə" + // 0x8F mapped to Ə
        "Ɛ" + // 0x90 mapped to Ɛ
        """‘’“”•–—˜™š›œ""" +
        "Ɲ" + // 0x9D mapped to Ɲ
        "žŸ" +
        """␢""" + // 0xA0 non-break space mapped to small b with stroke
        "¡¢£¤¥¦§¨©ª«¬" +
        "-" + // 0xAD soft hyphen mapped to regular hyphen (because soft hyphen seems to be a zero-width in many fonts.
        "®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ"
    assertEquals(exp, res)
  }

} 

package org.apache.daffodil.io

import org.apache.daffodil.exceptions.Assert
import org.apache.daffodil.util.Maybe
import org.apache.daffodil.util.Maybe._
import java.nio.CharBuffer
import java.nio.ByteBuffer
import org.apache.daffodil.util.LocalStack

abstract class LocalBuffer[T <: java.nio.Buffer] {
  protected def allocate(length: Long): T

  private var tempBuf: Maybe[T] = Nope

  def getBuf(length: Long) = {
    Assert.usage(length <= Int.MaxValue)
    if (tempBuf.isEmpty || tempBuf.get.capacity < length) {
      tempBuf = Maybe(allocate(length.toInt))
    }
    val buf = tempBuf.get
    buf.clear
    buf.limit(length.toInt)
    buf
  }
}


  final class LocalCharBuffer extends LocalBuffer[CharBuffer] {
    protected def allocate(length: Long) = CharBuffer.allocate(length.toInt)
  }

  final class LocalByteBuffer extends LocalBuffer[ByteBuffer] {
    protected def allocate(length: Long) = ByteBuffer.allocate(length.toInt)
  }

  final lazy val withLocalCharBuffer = new LocalStack[LocalCharBuffer](new LocalCharBuffer)

  final lazy val withLocalByteBuffer = new LocalStack[LocalByteBuffer](new LocalByteBuffer)
} 

package s4j.scala.chapter12

import java.io.File
import java.nio.CharBuffer

class FileReader(file: File) extends Readable with AutoCloseable {
  def read(buffer: CharBuffer): Int = {
    val linesRead: Int = 0
    linesRead
  }

  def close(): Unit = ???

}

object FileReader extends App {
  val reader = new FileReader(new File("autoexec.bat"))
  reader.read(CharBuffer.allocate(100))
} 

package tsec.passwordhashers.jca

import java.nio.CharBuffer

import cats.effect.Sync
import tsec.common._
import tsec.passwordhashers._
import tsec.passwordhashers.jca.internal.JBCrypt

sealed trait BCrypt

object BCrypt extends JCAPasswordPlatform[BCrypt] {

  private[tsec] def unsafeHashpw(p: Array[Byte]): String =
    JBCrypt.hashpw(p, JBCryptUtil.genSalt(DefaultBcryptRounds))

  private[tsec] def unsafeCheckpw(p: Array[Byte], hash: PasswordHash[BCrypt]): Boolean =
    JBCrypt.checkpw(p, hash)

  def hashpwWithRounds[F[_]](p: String, rounds: Int)(implicit F: Sync[F]): F[PasswordHash[BCrypt]] =
    hashpwWithRounds[F](p.asciiBytes, rounds)

  def hashpwWithRounds[F[_]](p: Array[Byte], rounds: Int)(implicit F: Sync[F]): F[PasswordHash[BCrypt]] =
    if (rounds < 10 || rounds > 30)
      F.raiseError(PasswordError("Invalid number of rounds"))
    else
      F.delay {
        val out = PasswordHash[BCrypt](JBCrypt.hashpw(p, JBCryptUtil.genSalt(rounds)))
        ByteUtils.zeroByteArray(p)
        out
      }

  def hashpwWithRounds[F[_]](p: Array[Char], rounds: Int)(implicit F: Sync[F]): F[PasswordHash[BCrypt]] =
    if (rounds < 10 || rounds > 30)
      F.raiseError(PasswordError("Invalid number of rounds"))
    else
      F.delay {
        val charbuffer = CharBuffer.wrap(p)
        val bytes      = defaultCharset.encode(charbuffer).array()
        val out        = PasswordHash[BCrypt](JBCrypt.hashpw(bytes, JBCryptUtil.genSalt(rounds)))
        //Clear pass
        ByteUtils.zeroCharArray(p)
        ByteUtils.zeroByteArray(bytes)
        out
      }
} 

package featherbed

import java.nio.CharBuffer
import java.nio.charset.{Charset, CodingErrorAction}
import scala.util.Try

import cats.data.{Validated, ValidatedNel}
import com.twitter.finagle.http.Response
import com.twitter.io.Buf
import shapeless.Witness
import sun.nio.cs.ThreadLocalCoders

package object content {
  type ContentType = String

  trait Decoder[ContentType] {
    type Out
    val contentType: String //widened version of ContentType
    def apply(buf: Response): ValidatedNel[Throwable, Out]
  }

  object Decoder extends LowPriorityDecoders {
    type Aux[CT, A1] = Decoder[CT] { type Out = A1 }

    def of[T <: ContentType, A1](t: T)(fn: Response => ValidatedNel[Throwable, A1]): Decoder.Aux[t.type, A1] =
      new Decoder[t.type] {
        type Out = A1
        val contentType = t
        def apply(response: Response) = fn(response)
      }

    def decodeString(response: Response): ValidatedNel[Throwable, String] = {
      Validated.fromTry(Try {
        response.charset.map(Charset.forName).getOrElse(Charset.defaultCharset)
      }).andThen { charset: Charset =>
        val decoder = ThreadLocalCoders.decoderFor(charset)
        Validated.fromTry(
          Try(
            decoder
              .onMalformedInput(CodingErrorAction.REPORT)
              .onUnmappableCharacter(CodingErrorAction.REPORT)
              .decode(Buf.ByteBuffer.Owned.extract(response.content).asReadOnlyBuffer()))).map[String](_.toString)
      }.toValidatedNel
    }
  }

  private[featherbed] trait LowPriorityDecoders {
    implicit val plainTextDecoder: Decoder.Aux[Witness.`"text/plain"`.T, String] = Decoder.of("text/plain") {
      response => Decoder.decodeString(response)
    }

    implicit val anyResponseDecoder: Decoder.Aux[Witness.`"**") {
      response => Validated.Valid(response)
    }
  }

  trait Encoder[A, ForContentType] {
    def apply(value: A, charset: Charset): ValidatedNel[Throwable, Buf]
  }

  object Encoder extends LowPriorityEncoders {
    def of[A, T <: ContentType](t: T)(fn: (A, Charset) => ValidatedNel[Throwable, Buf]): Encoder[A, t.type] =
      new Encoder[A, t.type] {
        def apply(value: A, charset: Charset) = fn(value, charset)
      }

    def encodeString(value: String, charset: Charset): ValidatedNel[Throwable, Buf] = {
      val encoder = ThreadLocalCoders.encoderFor(charset)
      Validated.fromTry(Try(encoder
        .onMalformedInput(CodingErrorAction.REPORT)
        .onUnmappableCharacter(CodingErrorAction.REPORT)
        .encode(CharBuffer.wrap(value)))).toValidatedNel.map[Buf](Buf.ByteBuffer.Owned(_))
    }
  }

  private[featherbed] trait LowPriorityEncoders {
    implicit val plainTextEncoder: Encoder[String, Witness.`"text/plain"`.T] = Encoder.of("text/plain") {
      case (value, charset) => Encoder.encodeString(value, charset)
    }
  }
}