scala.concurrent.duration.SECONDS Scala Examples

// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package com.daml.ledger.api.testtool.infrastructure.participant

import com.daml.ledger.api.testtool.infrastructure.LedgerServices
import com.daml.ledger.api.v1.ledger_identity_service.GetLedgerIdentityRequest
import com.daml.ledger.api.v1.transaction_service.GetLedgerEndRequest
import com.daml.timer.RetryStrategy
import io.grpc.ManagedChannel
import io.netty.channel.nio.NioEventLoopGroup
import org.slf4j.LoggerFactory

import scala.concurrent.duration.{DurationInt, SECONDS}
import scala.concurrent.{ExecutionContext, Future}

private[participant] final class ParticipantSession(
    val config: ParticipantSessionConfiguration,
    channel: ManagedChannel,
    eventLoopGroup: NioEventLoopGroup,
)(implicit val executionContext: ExecutionContext) {

  private[this] val logger = LoggerFactory.getLogger(classOf[ParticipantSession])

  private[this] val services: LedgerServices = new LedgerServices(channel)

  // The ledger identifier is retrieved only once when the participant session is created
  // Changing the ledger identifier during the execution of a session can result in unexpected consequences
  // The test tool is designed to run tests in an isolated environment but changing the
  // global state of the ledger breaks this assumption, no matter what
  private[this] val ledgerIdF =
    RetryStrategy.exponentialBackoff(10, 10.millis) { (attempt, wait) =>
      logger.debug(s"Fetching ledgerId to create context (attempt #$attempt, next one in $wait)...")
      services.identity.getLedgerIdentity(new GetLedgerIdentityRequest).map(_.ledgerId)
    }

  private[testtool] def createTestContext(
      endpointId: String,
      applicationId: String,
      identifierSuffix: String,
  ): Future[ParticipantTestContext] =
    for {
      ledgerId <- ledgerIdF
      end <- services.transaction.getLedgerEnd(new GetLedgerEndRequest(ledgerId)).map(_.getOffset)
    } yield
      new ParticipantTestContext(
        ledgerId,
        endpointId,
        applicationId,
        identifierSuffix,
        end,
        services,
        config.partyAllocation,
      )

  private[testtool] def close(): Unit = {
    logger.info(s"Disconnecting from participant at ${config.host}:${config.port}...")
    channel.shutdownNow()
    if (!channel.awaitTermination(10L, SECONDS)) {
      sys.error("Channel shutdown stuck. Unable to recover. Terminating.")
    }
    logger.info(s"Connection to participant at ${config.host}:${config.port} shut down.")
    if (!eventLoopGroup
        .shutdownGracefully(0, 0, SECONDS)
        .await(10L, SECONDS)) {
      sys.error("Unable to shutdown event loop. Unable to recover. Terminating.")
    }
    logger.info(s"Connection to participant at ${config.host}:${config.port} closed.")
  }
} 

package com.webtrends.harness.libs.iteratee

import com.webtrends.harness.libs.iteratee.internal.executeFuture
import scala.concurrent.{ Await, ExecutionContext, Future, Promise }
import scala.concurrent.duration.{ Duration, SECONDS, MILLISECONDS }
import scala.util.Try


  def delayed(it: => Iteratee[String, String], delay: Duration = Duration(5, MILLISECONDS))(implicit ec: ExecutionContext): Iteratee[String, String] = {
    Iteratee.flatten(timeout(it, delay))
  }

  val timer = new java.util.Timer(true)
  def timeout[A](a: => A, d: Duration)(implicit e: ExecutionContext): Future[A] = {
    val p = Promise[A]()
    timer.schedule(new java.util.TimerTask {
      def run() {
        p.complete(Try(a))
      }
    }, d.toMillis)
    p.future
  }
} 

package com.webtrends.harness.libs.iteratee

import scala.language.reflectiveCalls

import org.specs2.mutable._
import scala.concurrent.{ ExecutionContext, Future, Await }
import scala.concurrent.duration.{ Duration, SECONDS }
import scala.util.Try

object ExecutionSpec extends Specification {
  import Execution.trampoline

  val waitTime = Duration(5, SECONDS)

  "trampoline" should {

    "execute code in the same thread" in {
      val f = Future(Thread.currentThread())(trampoline)
      Await.result(f, waitTime) must equalTo(Thread.currentThread())
    }

    "not overflow the stack" in {
      def executeRecursively(ec: ExecutionContext, times: Int) {
        if (times > 0) {
          ec.execute(new Runnable {
            def run() = executeRecursively(ec, times - 1)
          })
        }
      }

      // Work out how deep to go to cause an overflow
      val overflowingExecutionContext = new ExecutionContext {
        def execute(runnable: Runnable): Unit = {
          runnable.run()
        }
        def reportFailure(t: Throwable): Unit = t.printStackTrace()
      }

      var overflowTimes = 1 << 10
      try {
        while (overflowTimes > 0) {
          executeRecursively(overflowingExecutionContext, overflowTimes)
          overflowTimes = overflowTimes << 1
        }
        sys.error("Can't get the stack to overflow")
      } catch {
        case _: StackOverflowError => ()
      }

      // Now verify that we don't overflow
      Try(executeRecursively(trampoline, overflowTimes)) must beSuccessfulTry[Unit]
    }

    "execute code in the order it was submitted" in {
      val runRecord = scala.collection.mutable.Buffer.empty[Int]
      case class TestRunnable(id: Int, children: Runnable*) extends Runnable {
        def run() = {
          runRecord += id
          for (c <- children) trampoline.execute(c)
        }
      }

      trampoline.execute(
        TestRunnable(0,
          TestRunnable(1),
          TestRunnable(2,
            TestRunnable(4,
              TestRunnable(6),
              TestRunnable(7)),
            TestRunnable(5,
              TestRunnable(8))),
          TestRunnable(3))
      )

      runRecord must equalTo(0 to 8)
    }

  }

} 

package com.webtrends.harness.libs.concurrent

import scala.language.reflectiveCalls

import org.specs2.mutable._
import java.util.concurrent.atomic.AtomicInteger
import scala.concurrent.{ ExecutionContext, Promise, Future, Await }
import scala.concurrent.duration.{ Duration, SECONDS }

object NonBlockingMutexSpec extends Specification {

  val waitTime = Duration(2, SECONDS)

  trait Tester {
    def run(body: => Unit): Unit
  }

  class MutexTester extends Tester {
    val mutex = new NonBlockingMutex()
    def run(body: => Unit) = mutex.exclusive(body)
  }

  class NaiveTester extends Tester {
    def run(body: => Unit) = body
  }

  def countOrderingErrors(runs: Int, tester: Tester)(implicit ec: ExecutionContext): Future[Int] = {
    val result = Promise[Int]()
    val runCount = new AtomicInteger(0)
    val orderingErrors = new AtomicInteger(0)

    for (i <- 0 until runs) {
      tester.run {
        val observedRunCount = runCount.getAndIncrement()

        // We see observedRunCount != i then this task was run out of order
        if (observedRunCount != i) {
          orderingErrors.incrementAndGet() // Record the error
        }
        // If this is the last task, complete our result promise
        if ((observedRunCount + 1) >= runs) {
          result.success(orderingErrors.get)
        }
      }
    }
    result.future
  }

  "NonBlockingMutex" should {

    "run a single operation" in {
      val p = Promise[Int]()
      val mutex = new NonBlockingMutex()
      mutex.exclusive { p.success(1) }
      Await.result(p.future, waitTime) must_== (1)
    }

    "run two operations" in {
      val p1 = Promise[Unit]()
      val p2 = Promise[Unit]()
      val mutex = new NonBlockingMutex()
      mutex.exclusive { p1.success(()) }
      mutex.exclusive { p2.success(()) }
      Await.result(p1.future, waitTime) must_== (())
      Await.result(p2.future, waitTime) must_== (())
    }

    "run code in order" in {
      import ExecutionContext.Implicits.global

      def percentageOfRunsWithOrderingErrors(runSize: Int, tester: Tester): Int = {
        val results: Seq[Future[Int]] = for (i <- 0 until 9) yield {
          countOrderingErrors(runSize, tester)
        }
        Await.result(Future.sequence(results), waitTime).filter(_ > 0).size * 10
      }

      // Iteratively increase the run size until we get observable errors 90% of the time
      // We want a high error rate because we want to then use the MutexTester
      // on the same run size and know that it is fixing up some problems. If the run size
      // is too small then the MutexTester probably isn't doing anything. We use
      // dynamic run sizing because the actual size that produces errors will vary
      // depending on the environment in which this test is run.
      var runSize = 8 // This usually reaches 8192 on my dev machine with 10 simultaneous queues
      var errorPercentage = 0
      while (errorPercentage < 90 && runSize < 1000000) {
        runSize = runSize << 1
        errorPercentage = percentageOfRunsWithOrderingErrors(runSize, new NaiveTester())
      }
      //println(s"Got $errorPercentage% ordering errors on run size of $runSize")

      // Now show that this run length works fine with the MutexTester
      percentageOfRunsWithOrderingErrors(runSize, new MutexTester()) must_== 0
    }

  }

} 

package com.raphtory.spouts

import java.util
import java.util.Properties

import com.raphtory.core.components.Spout.SpoutTrait
import org.apache.kafka.clients.consumer.KafkaConsumer

import scala.collection.JavaConverters._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.Duration
import scala.concurrent.duration.MILLISECONDS
import scala.concurrent.duration.SECONDS
import java.util.concurrent.LinkedBlockingQueue

import akka.actor.Props
import com.raphtory.core.components.Router.RouterManager

import scala.concurrent.duration.Duration
import scala.concurrent.duration._
import com.raphtory.core.utils.SchedulerUtil

import scala.util.Random
class KafkaSpout extends SpoutTrait {
  println("Starting kafka")
  var kafkaServer     = System.getenv().getOrDefault("KAFKA_ADDRESS", "127.0.0.1").trim
  var kafkaIP     = System.getenv().getOrDefault("KAFKA_PORT", "9092").trim
  var offset     = System.getenv().getOrDefault("KAFKA_OFFSET", "earliest").trim
  val x     = new Random().nextLong()
  var groupID   = System.getenv().getOrDefault("KAFKA_GROUP", "group" + x).trim
  var topic   = System.getenv().getOrDefault("KAFKA_TOPIC", "sample_topic").trim
  var restart   = System.getenv().getOrDefault("RESTART_RATE", "10").trim

  val queue = new LinkedBlockingQueue[String]

  val props = new Properties()
  props.put("bootstrap.servers", s"$kafkaServer:$kafkaIP")
  props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer")
  props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer")
  props.put("auto.offset.reset", offset)
  props.put("group.id", groupID)
  val consumer: KafkaConsumer[String, String] = new KafkaConsumer[String, String](props)
  consumer.subscribe(util.Arrays.asList(topic))


  //val helper = context.system.actorOf(Props(new KafkaSpoutBackPressure(queue)), "Spout_Helper")

  protected def ProcessSpoutTask(message: Any): Unit = message match {
    case StartSpout => AllocateSpoutTask(Duration(1, MILLISECONDS), "newLine")
    case "newLine"  => consumeFromKafka()
    case _          => println("message not recognized!")
  }

  def consumeFromKafka() = {
    //println("Consuming")
    val record = consumer.poll(java.time.Duration.ofMillis(5000)).asScala
    for (data <- record.iterator) {
      sendTuple(data.value())
      //helper ! KafkaData(data.value())
    }
    AllocateSpoutTask(Duration(restart.toInt, MILLISECONDS), "newLine")
  }
}
case class KafkaData(data:String)
class KafkaSpoutBackPressure(queue:LinkedBlockingQueue[String]) extends SpoutTrait {
  var startingSpeed     = System.getenv().getOrDefault("STARTING_SPEED", "1000").trim.toInt
  var increaseBy        = System.getenv().getOrDefault("INCREASE_BY", "1000").trim.toInt
  override def preStart(): Unit = {
    super.preStart()
    SchedulerUtil.scheduleTask(initialDelay = 60 seconds, interval = 60 second, receiver = self, message = "increase")
  }
  override protected def ProcessSpoutTask(receivedMessage: Any): Unit = receivedMessage match {
    case StartSpout => AllocateSpoutTask(Duration(1, MILLISECONDS), "newLine")
    case KafkaData(data) => queue.put(data)
    case "newLine"  => consumeFromQueue
    case "increase"  => startingSpeed+=increaseBy
    case _          => println("message not recognized!")
  }
  def consumeFromQueue() = {
    for(i<-0 to startingSpeed/100){
      if(!queue.isEmpty) {
        sendTuple(queue.take())
      }
    }
    AllocateSpoutTask(Duration(10, MILLISECONDS), "newLine")
  }
} 

package com.raphtory.examples.gab.actors

import java.util
import java.util.Properties

import com.raphtory.core.components.Spout.SpoutTrait
import org.apache.kafka.clients.consumer.KafkaConsumer

import scala.collection.JavaConverters._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.Duration
import scala.concurrent.duration.MILLISECONDS
import scala.concurrent.duration.SECONDS
import scala.util.Random
class GabKafkaSpout extends SpoutTrait {
  val x     = new Random().nextLong()
  val props = new Properties()
  props.put("bootstrap.servers", "moe.eecs.qmul.ac.uk:9092")
  props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer")
  props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer")
  props.put("auto.offset.reset", "earliest")
  props.put("group.id", "group" + x)
  val consumer: KafkaConsumer[String, String] = new KafkaConsumer[String, String](props)
  consumer.subscribe(util.Arrays.asList("gabResortedGraph"))

  protected def ProcessSpoutTask(message: Any): Unit = message match {
    case StartSpout => AllocateSpoutTask(Duration(1, MILLISECONDS), "newLine")
    case "newLine"  => consumeFromKafka()
    case _          => println("message not recognized!")
  }

  def consumeFromKafka() = {
    val record = consumer.poll(1000).asScala
    for (data <- record.iterator)
      sendTuple(data.value())
    AllocateSpoutTask(Duration(1, MILLISECONDS), "newLine")
  }
} 

package com.raphtory.examples.twitterRumour

import akka.actor.ActorContext
import com.raphtory.core.components.Spout.SpoutTrait

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.Duration
import scala.concurrent.duration.MILLISECONDS
import scala.concurrent.duration.SECONDS
import java.io.File

class rumourSpout extends SpoutTrait {
  var directory = System.getenv().getOrDefault("PHEME_DIRECTORY", "/home/tsunade/qmul/datasets/pheme-rnr-dataset").trim
  var r_case    = System.getenv().getOrDefault("RUMOUR_CASE", "/sydneysiege").trim
  val rpath     = directory + r_case + "/rumours"
  val nrpath    = directory + r_case + "/non-rumours"
  val max_r     = 500 //Option(new File(rpath).list.size).getOrElse(0)
  val max_nr    = 500 //Option(new File(nrpath).list.size).getOrElse(0)

  var rtweetCounter  = 0
  var nrtweetCounter = 0
//println("at spout....")

  override protected def ProcessSpoutTask(message: Any): Unit =
    // println(message)
    if ((rtweetCounter < max_r) || (nrtweetCounter < max_nr))
      //println("inside if stm")
      message match {
        case StartSpout =>
          AllocateSpoutTask(Duration(1, SECONDS), "rumourTweets")
          AllocateSpoutTask(Duration(1, SECONDS), "nonRumourTweets")
        case "rumourTweets" =>
          if (rtweetCounter < max_r) {
            //     println("sending ingest cmd r...")
            running(rpath, position = rtweetCounter, "R__")
            rtweetCounter += 1
            context.system.scheduler.scheduleOnce(Duration(1, SECONDS), self, "rumourTweets")
          }
        case "nonRumourTweets" =>
          if (nrtweetCounter < max_nr) {
            //  println("sending ingest cmd nr...")
            running(nrpath, position = nrtweetCounter, "nR__")
            nrtweetCounter += 1
            context.system.scheduler.scheduleOnce(Duration(1, SECONDS), self, "nonRumourTweets")
          }
        case _ => println("message not recognized!")
      }

  def ingestTweet(path: String, rnr: String): Unit = {
    //  println("inside ingest....")
    val tweet_l = new File(path)
    if (tweet_l.exists && tweet_l.isDirectory)
      tweet_l.listFiles.filter(_.isFile).toList.map(_.toString).foreach { tw_path: String =>
        sendTuple(rnr + tw_path)
      // println("running sendTuple...")
      }
  }

  def running(rpath: String, position: Int, rnr: String): Unit = {
    val rumours = new File(rpath)
    // println("inside runnign...")

    if (rumours.exists && rumours.isDirectory) {

      //println("inside runnign if...")
      val r_tweet = rumours.listFiles.toList.map(_.toString)
      ingestTweet(r_tweet(position) + "/source-tweet", rnr)
      ingestTweet(r_tweet(position) + "/reactions", rnr)
    }
  }
} 

package com.raphtory.examples.blockchain.spouts

import cats.effect.Blocker
import cats.effect.IO
import com.raphtory.core.components.Spout.SpoutTrait
import doobie.implicits._
import doobie.util.ExecutionContexts
import doobie.util.transactor.Transactor

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.Duration
import scala.concurrent.duration.MILLISECONDS
import scala.concurrent.duration.SECONDS

class EthereumPostgresSpout extends SpoutTrait {
  var startBlock = System.getenv().getOrDefault("STARTING_BLOCK", "46147").trim.toInt //first block to have a transaction by default
  val batchSize  = System.getenv().getOrDefault("BLOCK_BATCH_SIZE", "100").trim.toInt //number of blocks to pull each query
  val maxblock   = System.getenv().getOrDefault("MAX_BLOCK", "8828337").trim.toInt    //Maximum block in database to stop querying once this is reached

  val dbURL      = System.getenv().getOrDefault("DB_URL", "jdbc:postgresql:ether").trim //db connection string, default is for local with db called ether
  val dbUSER     = System.getenv().getOrDefault("DB_USER", "postgres").trim             //db user defaults to postgres
  val dbPASSWORD = System.getenv().getOrDefault("DB_PASSWORD", "").trim                 //default no password

  // querying done with doobie wrapper for JDBC (https://tpolecat.github.io/doobie/)
  implicit val cs = IO.contextShift(ExecutionContexts.synchronous)
  val dbconnector = Transactor.fromDriverManager[IO](
          "org.postgresql.Driver",
          dbURL,
          dbUSER,
          dbPASSWORD,
          Blocker.liftExecutionContext(ExecutionContexts.synchronous)
  )

  override def ProcessSpoutTask(message: Any): Unit = message match {
    case StartSpout  => AllocateSpoutTask(Duration(1, MILLISECONDS), "nextBatch")
    case "nextBatch" => running()
    case _           => println("message not recognized!")
  }

  protected def running(): Unit = {
    sql"select from_address, to_address, value,block_timestamp from transactions where block_number >= $startBlock AND block_number < ${startBlock + batchSize} "
      .query[
              (String, String, String, String)
      ]                                      //get the to,from,value and time for transactions within the set block batch
      .to[List]                              // ConnectionIO[List[String]]
      .transact(dbconnector)                 // IO[List[String]]
      .unsafeRunSync                         // List[String]
      .foreach(x => sendTuple(x.toString())) //send each transaction to the routers

    startBlock += batchSize                                   //increment batch for the next query
    if (startBlock > maxblock) stop()                         //if we have reached the max block we stop querying the database
    AllocateSpoutTask(Duration(1, MILLISECONDS), "nextBatch") // line up the next batch
  }
} 

package com.raphtory.examples.blockchain.spouts

import java.net.InetAddress
import java.util.NoSuchElementException

import com.raphtory.core.components.Spout.SpoutTrait
import com.raphtory.core.utils.Utils
import com.raphtory.tests.EtherAPITest.baseRequest
import com.raphtory.tests.EtherAPITest.currentBlock
import com.raphtory.tests.EtherAPITest.request

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.Duration
import scala.concurrent.duration.MILLISECONDS
import scala.concurrent.duration.NANOSECONDS
import scala.concurrent.duration.SECONDS
import scala.language.postfixOps
import scala.sys.process._
import scalaj.http.Http
import scalaj.http.HttpRequest
import spray.json._
import akka.http.scaladsl.unmarshalling.Unmarshal
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
import akka.stream.ActorMaterializer
import spray.json.DefaultJsonProtocol._

import scala.concurrent.duration._
import scala.concurrent.Await

case class EthResult(blockHash:Option[String],blockNumber:Option[String],from:Option[String],gas:Option[String],gasPrice:Option[String],hash:Option[String],input:Option[String],nonce:Option[String],r:Option[String],s:Option[String],to:Option[String],transactionIndex:Option[String],v:Option[String],value:Option[String])
case class EthTransaction(id:Option[String],jsonrpc:Option[String],result:EthResult)
class EthereumGethSpout extends SpoutTrait {
  var currentBlock = System.getenv().getOrDefault("SPOUT_ETHEREUM_START_BLOCK_INDEX", "9014194").trim.toInt
  var highestBlock = System.getenv().getOrDefault("SPOUT_ETHEREUM_MAXIMUM_BLOCK_INDEX", "10026447").trim.toInt
  val nodeIP       = System.getenv().getOrDefault("SPOUT_ETHEREUM_IP_ADDRESS", "127.0.0.1").trim
  val nodePort     = System.getenv().getOrDefault("SPOUT_ETHEREUM_PORT", "8545").trim
  val baseRequest  = requestBuilder()

  implicit val materializer = ActorMaterializer()
  implicit val EthFormat = jsonFormat14(EthResult)
  implicit val EthTransactionFormat = jsonFormat3(EthTransaction)
  if (nodeIP.matches(Utils.IPRegex))
    println(s"Connecting to Ethereum RPC \n Address:$nodeIP \n Port:$nodePort")
  else
    println(s"Connecting to Ethereum RPC \n Address:${hostname2Ip(nodeIP)} \n Port:$nodePort")

  override protected def ProcessSpoutTask(message: Any): Unit = message match {
    case StartSpout  => pullNextBlock()
    case "nextBlock" => pullNextBlock()
  }

  def pullNextBlock(): Unit = {
    if (currentBlock > highestBlock)
      return
    try {
      log.debug(s"Trying block $currentBlock")
      val transactionCountHex = executeRequest("eth_getBlockTransactionCountByNumber", "\"0x" + currentBlock.toHexString + "\"")
      val transactionCount = Integer.parseInt(transactionCountHex.fields("result").toString().drop(3).dropRight(1), 16)
      if(transactionCount>0){
        var transactions = "["
        for (i <- 0 until transactionCount)
          transactions = transactions + batchRequestBuilder("eth_getTransactionByBlockNumberAndIndex",s""""0x${currentBlock.toHexString}","0x${i.toHexString}"""")+","
        val trasnactionBlock = executeBatchRequest(transactions.dropRight(1)+"]")
        val transList = trasnactionBlock.parseJson.convertTo[List[EthTransaction]]
        transList.foreach(t => { //try needed to ignore contracts //todo include them
          try{sendTuple(s"${t.result.blockNumber.get},${t.result.from.get},${t.result.to.get},${t.result.value.get}")}
          catch {case e:NoSuchElementException =>}

        })

      }
      currentBlock += 1
      AllocateSpoutTask(Duration(1, NANOSECONDS), "nextBlock")
    } catch {
      case e: NumberFormatException => AllocateSpoutTask(Duration(1, SECONDS), "nextBlock")
      case e: Exception             => e.printStackTrace(); AllocateSpoutTask(Duration(1, SECONDS), "nextBlock")
    }
  }


  def batchRequestBuilder(command:String,params:String):String = s"""{"jsonrpc": "2.0", "id":"100", "method": "$command", "params": [$params]}"""
  def executeBatchRequest(data: String) = requestBatch(data).execute().body.toString
  def requestBatch(data: String): HttpRequest = baseRequest.postData(data)
  def requestBuilder() =
    if (nodeIP.matches(Utils.IPRegex))
      Http("http://" + nodeIP + ":" + nodePort).header("content-type", "application/json")
    else
      Http("http://" + hostname2Ip(nodeIP) + ":" + nodePort).header("content-type", "application/json")
  def request(command: String, params: String = ""): HttpRequest =
    baseRequest.postData(s"""{"jsonrpc": "2.0", "id":"100", "method": "$command", "params": [$params]}""")
  def executeRequest(command: String, params: String = "") =
    request(command, params).execute().body.toString.parseJson.asJsObject

  def hostname2Ip(hostname: String): String = InetAddress.getByName(hostname).getHostAddress()

} 

package ee.cone.c4actor

import com.typesafe.scalalogging.LazyLogging
import ee.cone.c4actor.EachTestProtocol.D_Item
import ee.cone.c4actor.Types.SrcId
import ee.cone.c4assemble.{IndexUtil, Single, assemble, by, c4assemble}
import ee.cone.c4assemble.Types._
import ee.cone.c4di.c4
import ee.cone.c4proto.{Id, protocol}

import scala.concurrent.Await
import scala.concurrent.duration.{Duration, SECONDS}

//  new EachTestNotEffectiveAssemble :: // 25s vs 1s for 3K 1-item-tx-s

@protocol("EachTestApp") object EachTestProtocol   {
  @Id(0x0001) case class D_Item(@Id(0x0001) srcId: String, @Id(0x0002) parent: String)
}

case class EachTestItem(item: D_Item, valueItem: D_Item)

@c4assemble("EachTestApp") class EachTestAssembleBase   {
  type ByParent = SrcId
  def joinByVal(
    key: SrcId,
    item: Each[D_Item]
  ): Values[(ByParent, D_Item)] = List(item.parent -> item)
  def join(
    key: SrcId,
    vItem: Each[D_Item],
    @by[ByParent] item: Each[D_Item]
  ): Values[(SrcId,EachTestItem)] = {
    List(WithPK(EachTestItem(item,vItem)))
  }
}



    def measure[R](f: =>R): R = {
      val startTime = System.currentTimeMillis
      val res = f
      logger.info(s"${System.currentTimeMillis - startTime}")
      res
    }

    IgnoreTestContext(Function.chain[Context](Seq(
      txAdd.add(LEvent.update(D_Item(s"V",""))),
      l => measure(Function.chain[Context](
        (1 to 3000).map(n=>txAdd.add(LEvent.update(D_Item(s"$n","V"))))
      )(l)),
      { (l:Context) =>
        val r = eachTestItems.ofA(l)
        assert(r.keys.size==3000)
        assert(r.values.forall(_.valueItem.parent.isEmpty))
        l
      }
    ))(voidContext))
    println("TEST 2 OK")

    execution.complete()
  }
} 

package ee.cone.c4gate

import java.lang.Math.toIntExact
import java.net.InetSocketAddress
import java.util.concurrent.TimeUnit

import com.sun.net.httpserver.{HttpExchange, HttpHandler, HttpServer}
import ee.cone.c4actor.{Executable, Execution, FinallyClose, Observer, Trace}
import ee.cone.c4gate.HttpProtocol.N_Header
import okio.ByteString

import scala.concurrent.{Await, ExecutionContext, Future}
import scala.concurrent.duration.{Duration, SECONDS}
import scala.collection.JavaConverters.mapAsScalaMapConverter
import scala.collection.JavaConverters.iterableAsScalaIterableConverter



class SunReqHandler(handler: FHttpHandler, executionContext: ExecutionContext) extends HttpHandler {
  def handle(httpExchange: HttpExchange) =
    Trace{ FinallyClose[HttpExchange,Unit](_.close())(httpExchange) { ex =>
      val method = httpExchange.getRequestMethod
      val path = httpExchange.getRequestURI.getPath
      val reqHeaders: List[N_Header] = httpExchange.getRequestHeaders.asScala
        .flatMap{ case(k,l)=>l.asScala.map(v=>N_Header(k,v)) }.toList
      val buffer = (new okio.Buffer).readFrom(httpExchange.getRequestBody)
      val body = buffer.readByteString()
      val request = FHttpRequest(method, path, reqHeaders, body)
      val responseF = handler.handle(request)(executionContext)
      val response = Await.result(responseF,Duration(600,SECONDS))
      val headers = httpExchange.getResponseHeaders
      response.headers.foreach(header=>headers.add(header.key,header.value))
      val bytes = response.body.toByteArray
      httpExchange.sendResponseHeaders(toIntExact(response.status), bytes.length)
      if(bytes.nonEmpty) httpExchange.getResponseBody.write(bytes)
    } }
}

class SunHttpServer(port: Int, handler: FHttpHandler, execution: Execution) extends Executable {
  def run(): Unit = concurrent.blocking{
    val pool = execution.newExecutorService("http-",None) //newWorkStealingPool
    execution.onShutdown("Pool",()=>{
      val tasks = pool.shutdownNow()
      pool.awaitTermination(Long.MaxValue,TimeUnit.SECONDS)
    })
    val executionContext: ExecutionContext = ExecutionContext.fromExecutor(pool)
    val server: HttpServer = HttpServer.create(new InetSocketAddress(port),0)
    execution.onShutdown("HttpServer",()=>server.stop(Int.MaxValue))
    server.setExecutor(pool)
    server.createContext("/", new SunReqHandler(handler,executionContext))
    server.start()
  }
}

class MutableStatefulReceiver[Message](execution: Execution, inner: List[Observer[Message]]) extends StatefulReceiver[Message] {
  var state: Future[List[Observer[Message]]] = Future.successful(inner)
  def send(message: Message): Unit = execution.fatal{ implicit ec =>
    synchronized{
      state = state.map(_.flatMap(_.activate(message)))
      state
    }
  }
}

class MutableStatefulReceiverFactory(execution: Execution) extends StatefulReceiverFactory {
  def create[Message](inner: List[Observer[Message]])(implicit executionContext: ExecutionContext): Future[StatefulReceiver[Message]] =
    Future.successful(new MutableStatefulReceiver[Message](execution,inner))
} 

package crawler

import java.net.URL

import scala.collection.JavaConverters._

import org.jsoup.nodes._
import org.jsoup._

import akka.actor._, akka.pattern._
import akka.util.Timeout
import scala.concurrent.duration.SECONDS
import scala.concurrent.ExecutionContext.Implicits.global

import Main.fetch
import Protocol._

object ActorBased {
  implicit val timeout: Timeout = Timeout(30, SECONDS)

  def main(args: Array[String]): Unit = {
    val system = ActorSystem("PiSystem")
    val root   = system actorOf Worker.workerProps

    (root ? Job(new URL("http://mvnrepository.com/"), 1)).onSuccess {
      case Result(res) =>
        println("Crawling finished successfully")
        println(res.take(10).mkString("\n"))
        println(res.size)

    }
  }
}

class Worker extends Actor {
  var buffer  : Set[URL]      = Set()
  var children: Set[ActorRef] = Set()
  var replyTo : Option[ActorRef] = None

  var answered = 0

  def receive = awaitingForTasks

  def dispatch(lnk: URL, depth: Int, visited: Set[URL]): Unit = {
    val child = context actorOf Worker.workerProps
    children += child
    child ! Job(lnk, depth, visited)
  }

  def awaitingForTasks: Receive = {
    case Job(url, depth, visited) =>
      replyTo = Some(sender)

      val links = fetch(url).getOrElse(Set()).filter(!visited(_))
      buffer    = links

      if (depth > 0) {
        println(s"Processing links of $url, descending now")

        children = Set()
        answered = 0

        for { l <- links } dispatch(l, depth - 1, visited)
        context become processing
      }
      else {
        println(s"Reached maximal depth on $url - returning its links only")
        sender ! Result(buffer)
        context stop self
      }
  }


  def processing: Receive = {
    case Result(urls) =>
      replyTo match {
        case Some(to) =>
          answered += 1
          println(s"$self: $answered actors responded of ${children.size}")
          buffer ++= urls
          if (answered == children.size) {
            to ! Result(buffer)
            context stop self
          }

        case None => println("replyTo actor is None, something went wrong")
      }
  }
}

object Worker {
  def workerProps: Props = Props(classOf[Worker])
}


object Protocol {
  sealed trait Msg
  case class Job(url : URL, depth: Int, visited: Set[URL] = Set()) extends Msg
  case class Result (urls: Set[URL]) extends Msg
}