scala.language.postfixOps Scala Examples
The following examples show how to use scala.language.postfixOps.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
Example 1
| Source File: WordEmbedding.scala From scorch with BSD 2-Clause "Simplified" License | 5 votes |
|
package scorch.sandbox.rnn
import botkop.numsca.Tensor
import botkop.{numsca => ns}
import scorch.autograd._
import scorch.nn.Module
import scala.language.postfixOps
case class WordEmbedding(w: Variable) extends Module(Seq(w)) {
override def forward(x: Variable): Variable =
WordEmbeddingFunction(x, w).forward()
}
object WordEmbedding {
def apply(v: Int, d: Int): WordEmbedding = {
val w = ns.randn(v, d)
WordEmbedding(Variable(w))
}
}
case class WordEmbeddingFunction(x: Variable, w: Variable) extends Function {
val List(n, t) = x.shape
val List(v, d) = w.shape
override def forward(): Variable = {
// todo: write idiomatic implementation in numsca
val out = for {
i <- 0 until n
j <- 0 until t
} yield {
val wix = x.data(i, j).squeeze().toInt
w.data(wix).data
}
val td = Tensor(out.toArray.flatten).reshape(n, t, d)
Variable(td, Some(this))
}
override def backward(gradOutput: Variable): Unit = {
// x = n * t
// w = v * d
// g = n * t * d
val dW = ns.zerosLike(w.data)
for {
i <- 0 until n
j <- 0 until t
} {
val gWord = gradOutput.data(i, j).reshape(1, d)
val wordIx = x.data(i, j).squeeze().toInt
dW(wordIx) += gWord
}
w.backward(Variable(dW))
x.backward()
}
}
Example 2
| Source File: SampleActorSpec.scala From coral with Apache License 2.0 | 5 votes |
|
package io.coral.actors.transform
import akka.actor.{ActorSystem, Props}
import akka.testkit.{ImplicitSender, TestActorRef, TestKit, TestProbe}
import akka.util.Timeout
import io.coral.lib.{NotSoRandom, Random}
import org.json4s._
import org.json4s.jackson.JsonMethods._
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}
import scala.concurrent.duration._
import scala.language.postfixOps
class SampleActorSpec(_system: ActorSystem)
extends TestKit(_system)
with ImplicitSender
with WordSpecLike
with Matchers
with BeforeAndAfterAll
with ScalaFutures {
def this() = this(ActorSystem("SampleActorSpec"))
override def afterAll() {
TestKit.shutdownActorSystem(system)
}
def arbitrarySampleActor(): SampleActor = {
val json = parse(
"""{ "type": "sample",
| "params": { "fraction": 0.707 } }
""".stripMargin)
val props = SampleActor(json).get
TestActorRef[SampleActor](props).underlyingActor
}
def notSoRandomSampleActor(fraction: Double, randoms: Double*): SampleActor = {
val json = parse(
s"""{ "type": "sample", "params": { "fraction": ${fraction} } }
""".stripMargin)
val source = NotSoRandom(randoms: _*)
val props = Props(classOf[SampleActor], json, Random(source))
TestActorRef[SampleActor](props).underlyingActor
}
implicit val timeout = Timeout(100 millis)
"A SampleActor" should {
"Be instantiated with sample fraction" in {
val json = parse("""{ "type": "sample", "params": { "fraction": 0.5 }}""".stripMargin)
val props = SampleActor(json).get
props.actorClass() should be(classOf[SampleActor])
val actor = TestActorRef[SampleActor](props).underlyingActor
actor.fraction should be(0.5)
}
"Not be instantiated without fraction or percentage" in {
val json = parse("""{ "type": "sample", "params": { "bla": "blabla" }}""".stripMargin)
SampleActor(json) should be(None)
}
"Be constructible with a io.coral.lib.Random for random boolean stream" in {
val actor = notSoRandomSampleActor(fraction = 0.5, randoms = 0.1, 0.49, 0.50, 0.51, 0.8, 0.4)
actor.next() should be(true)
actor.next() should be(true)
actor.next() should be(false)
actor.next() should be(false)
actor.next() should be(false)
actor.next() should be(true)
}
"Should trigger true or false according to random binomial sequence" in {
val actor = notSoRandomSampleActor(fraction = 0.7, randoms = 0.8, 0.6)
val json = parse( """{ "something": "whatever" }""").asInstanceOf[JObject]
val result1 = actor.simpleEmitTrigger(json)
result1 should be(Some(JNothing))
val result2 = actor.simpleEmitTrigger(json)
result2 should be(Some(json))
}
"Should have trigger and emit cooperate" in {
val actor = notSoRandomSampleActor(fraction = 0.7, randoms = 0.6, 0.8)
val ref = actor.self
val json = parse( """{ "something": "whatever" }""").asInstanceOf[JObject]
val probe = TestProbe()
actor.emitTargets += probe.ref
ref ! json
probe.expectMsg(json)
ref ! json
probe.expectNoMsg(100 millis)
}
}
}
Example 3
| Source File: ReadTreesTest.scala From apache-spark-test with Apache License 2.0 | 5 votes |
|
package com.github.dnvriend.spark.dataset
import com.github.dnvriend.TestSpec
import com.github.dnvriend.spark._
import scala.language.postfixOps
// note: in dutch, a tree is called 'boom'
class ReadTreesTest extends TestSpec {
val oldestTree =
Tree("BOMEN.fid-42c8dacd_14bfe306016_aff", "950572", "boom in verharding", "Onbekend", Some("overig"), Some(8586), "Verharding", "POINT (147106.879 483519.092)")
it should "get the oldest tree using sql" in withTrees { spark => trees =>
import spark.implicits._
trees.createOrReplaceTempView("trees")
trees.show(10, truncate = false)
// number of trees in the dataset
trees.count shouldBe 148648 // 148k trees in the city
trees.sqlContext.sql("FROM trees ORDER BY jaar DESC LIMIT 1").as[Tree].head shouldBe oldestTree
}
it should "get the oldest tree using dsl" in withTrees { spark => trees =>
import spark.implicits._
trees
.orderBy('jaar.desc)
.limit(1)
.head shouldBe oldestTree
}
it should "get the tree family there is most of" in withTrees { spark => trees =>
import spark.implicits._
import org.apache.spark.sql.functions._
trees
.filter('soort isNotNull)
.filter('soort notEqual "overig")
.groupBy('soort)
.agg(count('soort).as("count"))
.orderBy('count.desc)
.select('soort)
.limit(1)
.as[String].head shouldBe "Populier"
}
}
Example 4
| Source File: VwLabelRowCreatorTest.scala From aloha with MIT License | 5 votes |
|
package com.eharmony.aloha.dataset.vw.labeled
import com.eharmony.aloha.dataset.SparseFeatureExtractorFunction
import com.eharmony.aloha.semantics.func.GenFunc.f0
import org.junit.Assert._
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.BlockJUnit4ClassRunner
import scala.language.{postfixOps, implicitConversions}
@RunWith(classOf[BlockJUnit4ClassRunner])
final class VwLabelRowCreatorTest {
private[this] val lab = 3d
private[this] val imp0 = 0d
private[this] val imp1 = 1d
private[this] val imp2 = 2d
private[this] val emptyTag = ""
private[this] val tag = "t"
private[this] implicit def liftToOption[A](a: A): Option[A] = Option(a)
private[this] def spec(lab: Option[Double] = None, imp: Option[Double] = None, tag: Option[String] = None): VwLabelRowCreator[Any] = {
val fef = new SparseFeatureExtractorFunction[Any](Vector("f1" -> f0("Empty", _ => Nil)))
VwLabelRowCreator(fef, 0 to 0 toList, Nil, None, f0("", _ => lab), f0("", _ => imp), f0("", _ => tag))
}
private[this] def testLabelRemoval(spec: VwLabelRowCreator[Any], exp: String = ""): Unit = assertEquals(exp, spec(())._2.toString)
// All of these should return empty label because the Label function returns a missing label.
@Test def testS___() = testLabelRemoval(spec())
@Test def testS__e() = testLabelRemoval(spec(tag = emptyTag))
@Test def testS__t() = testLabelRemoval(spec(tag = tag))
@Test def testS_0_() = testLabelRemoval(spec(imp = imp0))
@Test def testS_0e() = testLabelRemoval(spec(imp = imp0, tag = emptyTag))
@Test def testS_0t() = testLabelRemoval(spec(imp = imp0, tag = tag))
@Test def testS_1_() = testLabelRemoval(spec(imp = imp1))
@Test def testS_1e() = testLabelRemoval(spec(imp = imp1, tag = emptyTag))
@Test def testS_1t() = testLabelRemoval(spec(imp = imp1, tag = tag))
@Test def testS_2_() = testLabelRemoval(spec(imp = imp2))
@Test def testS_2e() = testLabelRemoval(spec(imp = imp2, tag = emptyTag))
@Test def testS_2t() = testLabelRemoval(spec(imp = imp2, tag = tag))
// Importance not provided makes entire label vanish
@Test def testS1_e() = testLabelRemoval(spec(lab = lab, tag = emptyTag))
@Test def testS1_t() = testLabelRemoval(spec(lab = lab, tag = tag))
// Importance of zero is given explicitly.
@Test def testS10_() = testLabelRemoval(spec(lab = lab, imp = imp0), "3 0 |")
@Test def testS10e() = testLabelRemoval(spec(lab = lab, imp = imp0, tag = emptyTag), "3 0 |")
@Test def testS10t() = testLabelRemoval(spec(lab = lab, imp = imp0, tag = tag), "3 0 t|")
// Importance of 1 is omitted.
@Test def testS11_() = testLabelRemoval(spec(lab = lab, imp = imp1), "3 |")
@Test def testS11e() = testLabelRemoval(spec(lab = lab, imp = imp1, tag = emptyTag), "3 |")
@Test def testS11t() = testLabelRemoval(spec(lab = lab, imp = imp1, tag = tag), "3 t|")
@Test def testS12_() = testLabelRemoval(spec(lab = lab, imp = imp2), "3 2 |")
@Test def testS12e() = testLabelRemoval(spec(lab = lab, imp = imp2, tag = emptyTag), "3 2 |")
@Test def testS12t() = testLabelRemoval(spec(lab = lab, imp = imp2, tag = tag), "3 2 t|")
@Test def testStringLabel() {
val spec = new VwLabelRowCreator(
new SparseFeatureExtractorFunction(Vector("f1" -> f0("Empty", (_: Double) => Nil))),
0 to 0 toList,
Nil,
None,
f0("", (s: Double) => Option(s)), // Label
f0("", (_: Double) => Option(1d)), // Importance
f0("", (_: Double) => None)) // Tag
val values = Seq(
-1.0 -> "-1",
-0.99999999999999999 -> "-1",
-0.9999999999999999 -> "-0.9999999999999999",
-1.0E-16 -> "-0.0000000000000001",
-1.0E-17 -> "-0.00000000000000001",
-1.0E-18 -> "-0",
0.0 -> "0",
1.0E-18 -> "0",
1.0E-17 -> "0.00000000000000001",
1.0E-16 -> "0.0000000000000001",
0.9999999999999999 -> "0.9999999999999999",
0.99999999999999999 -> "1",
1.0 -> "1"
)
values foreach { case(v, ex) => assertEquals(s"for line: $v", Option(ex), spec.stringLabel(v)) }
}
}
Example 5
| Source File: MappingsHelper.scala From sbt-jib with Apache License 2.0 | 5 votes |
|
package de.gccc.jib
import java.io.File
import sbt._
import sbt.io.{ IO, PathFinder }
import scala.language.postfixOps
def fromClasspath(entries: Seq[Attributed[File]],
target: String,
includeArtifact: Artifact => Boolean,
includeOnNoArtifact: Boolean = false): Seq[(File, String)] =
entries.filter(attr => attr.get(sbt.Keys.artifact.key) map includeArtifact getOrElse includeOnNoArtifact).map {
attribute =>
val file = attribute.data
file -> s"$target/${file.getName}"
}
}
Example 6
| Source File: NetworkTest.scala From jvm-toxcore-c with GNU General Public License v3.0 | 5 votes |
|
package im.tox.tox4j.core
import im.tox.core.network.Port
import im.tox.tox4j.DhtNodeSelector.node
import im.tox.tox4j.TestConstants.Timeout
import im.tox.tox4j._
import im.tox.tox4j.core.NetworkTest.logger
import im.tox.tox4j.core.data.ToxPublicKey
import im.tox.tox4j.impl.jni.ToxCoreImplFactory.{ withToxUnit, withToxes }
import org.scalatest.FlatSpec
import org.scalatest.concurrent.Timeouts
import org.slf4j.LoggerFactory
import scala.language.postfixOps
object NetworkTest {
private val logger = LoggerFactory.getLogger(classOf[NetworkTest])
private val ToxCount = 10
}
@SuppressWarnings(Array("org.wartremover.warts.While"))
final class NetworkTest extends FlatSpec with Timeouts {
// TODO(iphydf): Figure out why the bootstrap tests all fail on Travis.
"LAN discovery" should "connect all nodes" in {
failAfter(Timeout) {
withToxes(NetworkTest.ToxCount) { toxes =>
val action = s"Connecting all of ${toxes.size} toxes with LAN discovery"
logger.info(action)
val start = System.currentTimeMillis
while (!toxes.isAllConnected) {
toxes.iterate()
Thread.sleep(toxes.iterationInterval)
}
val end = System.currentTimeMillis
logger.info(s"$action took ${end - start} ms")
}
}
}
it should "connect at least one instance" in {
failAfter(Timeout) {
withToxes(NetworkTest.ToxCount) { toxes =>
val action = s"Connecting one of ${toxes.size} toxes with LAN discovery"
logger.info(action)
val start = System.currentTimeMillis
while (!toxes.isAnyConnected) {
toxes.iterate()
try {
Thread.sleep(toxes.iterationInterval)
} catch {
case e: InterruptedException =>
}
}
val end = System.currentTimeMillis
logger.info(s"$action took ${end - start} ms")
}
}
}
}
Example 7
| Source File: ExampleApi.scala From caliban with Apache License 2.0 | 5 votes |
|
package caliban
import scala.language.postfixOps
import caliban.ExampleData._
import caliban.ExampleService.ExampleService
import caliban.GraphQL.graphQL
import caliban.schema.Annotations.{ GQLDeprecated, GQLDescription }
import caliban.schema.GenericSchema
import caliban.wrappers.ApolloTracing.apolloTracing
import caliban.wrappers.Wrappers.{ maxDepth, maxFields, printSlowQueries, timeout }
import zio.URIO
import zio.clock.Clock
import zio.console.Console
import zio.duration._
import zio.stream.ZStream
object ExampleApi extends GenericSchema[ExampleService] {
case class Queries(
@GQLDescription("Return all characters from a given origin")
characters: CharactersArgs => URIO[ExampleService, List[Character]],
@GQLDeprecated("Use `characters`")
character: CharacterArgs => URIO[ExampleService, Option[Character]]
)
case class Mutations(deleteCharacter: CharacterArgs => URIO[ExampleService, Boolean])
case class Subscriptions(characterDeleted: ZStream[ExampleService, Nothing, String])
implicit val roleSchema = gen[Role]
implicit val characterSchema = gen[Character]
implicit val characterArgsSchema = gen[CharacterArgs]
implicit val charactersArgsSchema = gen[CharactersArgs]
val api: GraphQL[Console with Clock with ExampleService] =
graphQL(
RootResolver(
Queries(
args => ExampleService.getCharacters(args.origin),
args => ExampleService.findCharacter(args.name)
),
Mutations(args => ExampleService.deleteCharacter(args.name)),
Subscriptions(ExampleService.deletedEvents)
)
) @@
maxFields(200) @@ // query analyzer that limit query fields
maxDepth(30) @@ // query analyzer that limit query depth
timeout(3 seconds) @@ // wrapper that fails slow queries
printSlowQueries(500 millis) @@ // wrapper that logs slow queries
apolloTracing // wrapper for https://github.com/apollographql/apollo-tracing
}
Example 8
| Source File: ParallelModuleSpec.scala From scorch with BSD 2-Clause "Simplified" License | 5 votes |
|
package scorch.nn
import botkop.{numsca => ns}
import org.nd4j.linalg.api.buffer.DataBuffer
import org.nd4j.linalg.factory.Nd4j
import org.scalatest.{FlatSpec, Matchers}
import scorch.autograd.Variable
import scorch._
import scorch.autograd.Variable._
import scorch.TestUtil._
import scorch.nn.ParallelModule.ParallelizeFunction
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.util.Random
class ParallelModuleSpec extends FlatSpec with Matchers {
Nd4j.setDataType(DataBuffer.Type.DOUBLE)
ns.rand.setSeed(231)
Random.setSeed(231)
"A Parallelization module" should "correctly reconstitute output in forward pass" in {
val batchSize = 32
val numFeatures = 20
case class Net() extends Module {
override def forward(x: Variable): Variable = Variable(x.data.copy())
}
val net = Net()
val p = ParallelModule(net, parallelism = 4)
val input = Variable(ns.randn(batchSize, numFeatures))
val yHat = p(input)
assert(yHat.data.sameElements(input.data))
assert(yHat.data.sameShape(input.data))
}
it should "update the gradients of the module" in {
val batchSize = 32
val numFeatures = 20
val numClasses = 10
case class Net() extends Module {
val fc = Linear(numFeatures, numClasses)
override def forward(x: Variable): Variable = x ~> fc ~> relu
}
val net = Net()
val parCopies = net.parameters.map { p =>
Variable(p.data.copy())
}
val p = ParallelModule(net, parallelism = 4)
val input = Variable(ns.randn(batchSize, numFeatures))
val yHat = p(input)
val target = Variable(ns.randint(numClasses, Array(batchSize, 1)))
val loss = softmaxLoss(yHat, target)
loss.backward()
net.parameters.map(_.grad).foreach(println)
net.parameters.foreach { p =>
assert(p.grad.data.sameShape(p.data))
assert(ns.sum(p.grad.data) != 0.0)
}
}
"The parallelization function" should "calulate the correct gradients" in {
val batchSize = 90
val numFeatures = 20
val numClasses = 10
case class Net() extends Module {
val fc = Linear(numFeatures, numClasses)
override def forward(x: Variable): Variable = x ~> fc ~> relu
}
val net = Net()
def f(a: Variable) =
ParallelizeFunction(a,
module = net,
parallelism = 4)
.forward()
val input = Variable(ns.randn(batchSize, numFeatures))
oneOpGradientCheck(f, input, 1e-7)
}
}
Example 9
| Source File: DataLoaderSpec.scala From scorch with BSD 2-Clause "Simplified" License | 5 votes |
|
package scorch.data.loader
import botkop.numsca.Tensor
import botkop.{numsca => ns}
import org.nd4j.linalg.api.buffer.DataBuffer
import org.nd4j.linalg.factory.Nd4j
import org.scalatest.{FlatSpec, Matchers}
import scorch._
import scorch.autograd.Variable
import scorch.nn.cnn.{Conv2d, MaxPool2d}
import scorch.nn.{Linear, Module}
import scorch.optim.Adam
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.util.Random
class DataLoaderSpec extends FlatSpec with Matchers {
Nd4j.setDataType(DataBuffer.Type.DOUBLE)
ns.rand.setSeed(231)
Random.setSeed(231)
"A cifar-10 loader" should "load data" in {
val miniBatchSize = 8
val take = 10
val loader =
new Cifar10DataLoader(miniBatchSize = miniBatchSize,
mode = "train",
take = Some(take * miniBatchSize)).toSeq
assert(loader.size == take)
loader.foreach {
case (x, y) =>
assert(x.shape.head == miniBatchSize)
// assert(x.shape(1) == 3 * 32 * 32)
assert(y.shape.head == miniBatchSize)
assert(y.shape(1) == 1)
}
}
it should "feed a network" in {
val batchSize = 16
val numBatches = 2
val numEpochs = 5
val parallelism = 8
val (numChannels, imageSize) = (3, 32)
val inputShape = List(batchSize, numChannels, imageSize, imageSize)
val numClasses = 10
case class Net() extends Module {
val conv = Conv2d(numChannels = 3,
numFilters = 4,
filterSize = 5,
weightScale = 1e-3,
stride = 1,
pad = 1)
val pool = MaxPool2d(poolSize = 2, stride = 2)
val numFlatFeatures: Int =
pool.outputShape(conv.outputShape(inputShape)).tail.product
def flatten(v: Variable): Variable = v.reshape(-1, numFlatFeatures)
val fc = Linear(numFlatFeatures, numClasses)
override def forward(x: Variable): Variable =
x ~> conv ~> relu ~> pool ~> flatten ~> fc ~> relu
}
val net = Net().par(parallelism)
val optimizer = Adam(net.parameters, lr = 0.01)
val loader = new Cifar10DataLoader(miniBatchSize = batchSize,
mode = "train",
take = Some(numBatches * batchSize))
val seq = loader.toSeq
for (epoch <- 1 to numEpochs) {
seq.zipWithIndex
.foreach {
case ((x, y), i) =>
optimizer.zeroGrad()
val yHat = net(x)
val loss = softmaxLoss(yHat, y)
val guessed = ns.argmax(yHat.data, axis = 1)
val accuracy = ns.sum(y.data == guessed) / batchSize
println(
s"$epoch:$i: loss: ${loss.data.squeeze()} accuracy: $accuracy")
loss.backward()
optimizer.step()
}
}
}
}
Example 10
| Source File: recfun.scala From bolts with Apache License 2.0 | 5 votes |
|
package recfun
import stainless.lang._
import stainless.collection._
import scala.language.postfixOps
object Main {
// Exercise 1
def pascal(c: BigInt, r: BigInt): BigInt = {
require(c>=0 && r>=0 && c<=r)
decreases(r)
if(c==0) 1
else if(r==c) 1
else pascal(c-1, r-1) + pascal(c, r-1)
}
def checkPascal(c: BigInt): Boolean = {
c<0 || pascal(c, c) == 1
}.holds
def checkPascal2(c: BigInt, r: BigInt): Boolean = {
require(c>=0 && r>=0 && c<=r)
pascal(c,r)==1 || pascal(c,r) == pascal(c-1,r-1) + pascal(c,r-1)
}.holds
// Exercise 2
def balance(chars: List[Char]): Boolean = {
def recbalance(chars: List[Char], n: BigInt) : Boolean = {
decreases(chars)
if (n < 0) false
else if (chars.isEmpty) {
if (n == 0) true
else false
}
else if (chars.head == '(') recbalance(chars.tail, n+1)
else if (chars.head == ')') recbalance(chars.tail, n-1)
else recbalance(chars.tail, n)
}
recbalance(chars, 0)
}
def checkBalance(chars: List[Char]) : Boolean = {
!balance(')' :: chars)
}.holds
// Exercise 3
def distinct[T](l: List[T]): Boolean = {
decreases(l)
l match {
case Nil() => true
case Cons(x,xs) => !xs.contains(x) && distinct(xs)
}
}
def max(x: BigInt, y: BigInt) = if (x > y) x else y
def countChange(money: BigInt, coins: List[BigInt]): BigInt = {
require(coins.forall(_ > 0) && distinct(coins))
decreases(max(0, money), coins)
if (money == 0) BigInt(1)
else if (money < 0 || coins.isEmpty) BigInt(0)
else countChange(money-coins.head, coins) + countChange(money, coins.tail)
} ensuring(_ >= 0)
}
Example 11
| Source File: MnistDataLoader.scala From scorch with BSD 2-Clause "Simplified" License | 5 votes |
|
package scorch.data.loader
import botkop.numsca.Tensor
import com.typesafe.scalalogging.LazyLogging
import scorch.autograd.Variable
import scala.io.Source
import scala.language.postfixOps
import scala.util.Random
class MnistDataLoader(val mode: String,
miniBatchSize: Int,
take: Option[Int] = None,
seed: Long = 231)
extends DataLoader
with LazyLogging {
val file: String = mode match {
case "train" => "data/mnist/mnist_train.csv"
case "validate" => "data/mnist/mnist_test.csv"
}
val numEntries: Int =
Source.fromFile(file).getLines().length
override val numSamples: Int = take match {
case Some(n) => math.min(n, numEntries)
case None => numEntries
}
override val numBatches: Int =
(numSamples / miniBatchSize) +
(if (numSamples % miniBatchSize == 0) 0 else 1)
// this should fit in memory
val data: Seq[(Variable, Variable)] = Random
.shuffle(
Source
.fromFile(file)
.getLines()
)
.take(take.getOrElse(numSamples))
.sliding(miniBatchSize, miniBatchSize)
.map { lines =>
val (xData, yData) = lines
.foldLeft(List.empty[Float], List.empty[Float]) {
case ((xs, ys), line) =>
val tokens = line.split(",")
val (y, x) =
(tokens.head.toFloat, tokens.tail.map(_.toFloat / 255).toList)
(x ::: xs, y :: ys)
}
val x = Variable(Tensor(xData.toArray).reshape(yData.length, 784))
val y = Variable(Tensor(yData.toArray).reshape(yData.length, 1))
(x, y)
} toSeq
override def iterator: Iterator[(Variable, Variable)] = data.toIterator
}
Example 12
| Source File: JSCarousels.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.carousel
import scala.language.postfixOps
import scala.scalajs.js
import rx.Rx
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.JSRenderingContext
import com.karasiq.bootstrap.icons.Icons
import com.karasiq.bootstrap.jquery.BootstrapJQueryContext
import com.karasiq.bootstrap.utils.Utils
trait JSCarousels extends UniversalCarousels { self: JSRenderingContext with Carousels with Utils with Icons with BootstrapComponents with BootstrapJQueryContext ⇒
import scalaTags.all._
type Carousel = JSCarousel
object Carousel extends CarouselFactory {
def apply(data: Rx[Seq[Modifier]], id: String = Bootstrap.newId): JSCarousel = {
new JSCarousel(id, data)
}
def slide(image: String, content: Modifier*): Modifier = {
UniversalCarousel.slide(image, content)
}
}
class JSCarousel(carouselId: String, content: Rx[Seq[Modifier]]) extends UniversalCarousel(carouselId, content) {
def create(interval: Int = 5000, pause: String = "hover",
wrap: Boolean = true, keyboard: Boolean = true,
modifiers: Modifier = Bootstrap.noModifier): Element = {
val element = carousel(modifiers).render
val options = js.Object().asInstanceOf[JSCarouselOptions]
options.interval = interval
options.pause = pause
options.wrap = wrap
options.keyboard = keyboard
jQuery(element).carousel(options)
element
}
override def render(md: Modifier*): Modifier = {
create(modifiers = md)
}
}
}
Example 13
| Source File: JSModals.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.modal
import scala.language.postfixOps
import scala.scalajs.js
import scala.scalajs.js.|
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.JSRenderingContext
import com.karasiq.bootstrap.jquery.BootstrapJQueryContext
trait JSModals { self: JSRenderingContext with Modals with BootstrapComponents with BootstrapJQueryContext ⇒
implicit class JSModalOps(modal: Modal) {
def show(backdrop: Boolean | String = true, keyboard: Boolean = true,
show: Boolean = true, events: Map[String, js.Any] = Map.empty): Unit = {
val dialog = jQuery(modal.renderTag().render)
val options = js.Object().asInstanceOf[JSModalOptions]
options.backdrop = backdrop
options.keyboard = keyboard
options.show = show
dialog.modal(options)
dialog.on("hidden.bs.modal", () ⇒ {
// Remove from DOM
dialog.remove()
})
events.foreach { case (event, handler) ⇒
dialog.on(event, handler)
}
dialog.modal("show")
}
}
}
Example 14
| Source File: JSPopovers.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.popover
import scala.language.postfixOps
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.JSRenderingContext
import com.karasiq.bootstrap.jquery.BootstrapJQueryContext
import com.karasiq.bootstrap.tooltip.Tooltips
trait JSPopovers { self: JSRenderingContext with BootstrapComponents with Popovers with Tooltips with BootstrapJQueryContext ⇒
import scalaTags.all._
class JSPopover(val options: PopoverOptions) extends Popover {
def toggle: Modifier = new Modifier {
def applyTo(t: Element): Unit = {
val jsOptions = scalajs.js.Object().asInstanceOf[JSPopoverOptions]
def set(value: String, f: String ⇒ Unit) = if (value.nonEmpty) f(value)
jsOptions.animation = options.animation
jsOptions.content = options.content.render
jsOptions.title = options.title.render
jsOptions.html = options.html
jsOptions.placement = options.placement.toString
set(options.container, jsOptions.container = _)
set(options.delay, jsOptions.delay = _)
set(options.selector, jsOptions.selector = _)
set(options.template, jsOptions.template = _)
set(options.trigger, jsOptions.trigger = _)
set(options.viewport, jsOptions.viewport = _)
jQuery(t).popover(jsOptions)
}
}
override def render(md: Modifier*): Modifier = {
toggle +: md
}
}
object Popover extends PopoverFactory {
def apply(title: Frag, content: Frag, placement: TooltipPlacement = TooltipPlacement.auto): JSPopover = {
new JSPopover(PopoverOptions(html = true, title = title, content = content, placement = placement))
}
}
}
Example 15
| Source File: JSBootstrapBundle.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap
import scala.language.postfixOps
import rx.Ctx
import com.karasiq.bootstrap.carousel.JSCarousels
import com.karasiq.bootstrap.context.JSRenderingContext
import com.karasiq.bootstrap.jquery.BootstrapJQueryContext
import com.karasiq.bootstrap.modal.JSModals
import com.karasiq.bootstrap.navbar.JSNavigationBars
import com.karasiq.bootstrap.popover.JSPopovers
import com.karasiq.bootstrap.tooltip.JSTooltips
// JS components implementation
trait JSBootstrapBundle extends UniversalBootstrapBundle with JSRenderingContext with JSModals with JSTooltips
with JSPopovers with JSNavigationBars with JSCarousels with BootstrapJQueryContext
object JSBootstrapBundle {
def apply()(implicit rx: Ctx.Owner): JSBootstrapBundle = {
new JSBootstrapBundle {
implicit val scalaRxContext = rx
}
}
}
Example 16
| Source File: Buttons.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.buttons
import scala.language.postfixOps
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.{ClassModifiers, RenderingContext}
import com.karasiq.bootstrap.utils.Utils
trait Buttons extends ButtonStyles with ButtonGroups with ButtonStates { self: RenderingContext with BootstrapComponents with ClassModifiers with Utils ⇒
import scalaTags.all._
type Button <: AbstractButton
val Button: ButtonFactory
trait AbstractButton extends BootstrapHtmlComponent {
def style: ButtonStyle
def size: ButtonSize
def block: Boolean
def active: Boolean
def disabled: Boolean
}
trait ButtonFactory {
def apply(style: ButtonStyle = ButtonStyle.default, size: ButtonSize = ButtonSize.default,
block: Boolean = false, active: Boolean = false, disabled: Boolean = false): Button
}
}
Example 17
| Source File: UniversalButtons.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.buttons
import scala.language.postfixOps
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.{ClassModifiers, RenderingContext}
trait UniversalButtons extends UniversalButtonStates with UniversalButtonGroups { self: RenderingContext with BootstrapComponents with ClassModifiers with Buttons ⇒
import scalaTags.all._
type Button = ButtonBuilder
object Button extends ButtonFactory {
case class ButtonBuilder(style: ButtonStyle = ButtonStyle.default, size: ButtonSize = ButtonSize.default,
block: Boolean = false, active: Boolean = false, disabled: Boolean = false)
extends UniversalButton {
def withStyle(style: ButtonStyle): ButtonBuilder = copy(style = style)
def withSize(size: ButtonSize): ButtonBuilder = copy(size = size)
}
}
Example 18
| Source File: PagedTables.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.table
import scala.language.postfixOps
import rx._
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.{ClassModifiers, RenderingContext}
import com.karasiq.bootstrap.pagination.PageSelectors
trait PagedTables { self: RenderingContext with BootstrapComponents with Tables with PageSelectors with ClassModifiers ⇒
import scalaTags.all._
type PagedTable <: AbstractPagedTable with BootstrapHtmlComponent
val PagedTable: PagedTableFactory
trait AbstractPagedTable {
def table: Table
def pageSelector: PageSelector
}
trait PagedTableFactory {
def apply(heading: Rx[Seq[Modifier]], content: Rx[Seq[TableRow]], perPage: Int = 20): PagedTable
def static(heading: Seq[Modifier], content: Seq[TableRow], perPage: Int = 20): PagedTable = {
this.apply(Var(heading), Var(content), perPage)
}
private[table] def pagesRx(allContent: Rx[Seq[TableRow]], rowsPerPage: Int): Rx[Int] = {
Rx {
val data = allContent()
val result = if (data.isEmpty) {
1
} else if (data.length % rowsPerPage == 0) {
data.length / rowsPerPage
} else {
data.length / rowsPerPage + 1
}
result
}
}
private[table] def pagedDataRx(allContent: Rx[Seq[TableRow]], currentPage: Rx[Int], rowsPerPage: Int): Rx[Seq[TableRow]] = {
Rx {
val data = allContent()
val page = currentPage()
data.slice(rowsPerPage * (page - 1), rowsPerPage * (page - 1) + rowsPerPage)
}
}
}
trait AbstractStaticPagedTable extends AbstractPagedTable {
def allContent: Rx[Seq[TableRow]]
def rowsPerPage: Int
}
}
Example 19
| Source File: ProxyRequestCodec.scala From aws-lambda-scala with MIT License | 5 votes |
|
package io.github.mkotsur.aws.codecs
import java.io.ByteArrayInputStream
import cats.syntax.either.catsSyntaxEither
import io.circe.generic.auto._
import io.github.mkotsur.aws.handler.CanDecode
import io.github.mkotsur.aws.proxy.ProxyRequest
import shapeless.Generic
import scala.language.{higherKinds, postfixOps}
private[aws] trait ProxyRequestCodec extends AllCodec with FutureCodec {
def GenericProxyRequestOf[T] = shapeless.Generic[ProxyRequest[T]]
implicit def canDecodeProxyRequest[T](implicit canDecode: CanDecode[T]) = CanDecode.instance[ProxyRequest[T]] { is =>
{
def extractBody(s: ProxyRequest[String]) = s.body match {
case Some(bodyString) => canDecode.readStream(new ByteArrayInputStream(bodyString.getBytes)).map(Option.apply)
case None => Right(None)
}
def produceProxyResponse(decodedRequestString: ProxyRequest[String], bodyOption: Option[T]) = {
val reqList = Generic[ProxyRequest[String]].to(decodedRequestString)
Generic[ProxyRequest[T]].from((bodyOption :: reqList.reverse.tail).reverse)
}
for (decodedRequest$String <- CanDecode[ProxyRequest[String]].readStream(is);
decodedBodyOption <- extractBody(decodedRequest$String))
yield produceProxyResponse(decodedRequest$String, decodedBodyOption)
}
}
}
Example 20
| Source File: FutureCodec.scala From aws-lambda-scala with MIT License | 5 votes |
|
package io.github.mkotsur.aws.codecs
import java.io.ByteArrayOutputStream
import java.nio.charset.Charset
import io.circe.Encoder
import io.github.mkotsur.aws.handler.CanEncode
import io.github.mkotsur.aws.proxy.ProxyResponse
import io.circe.generic.auto._
import io.circe.syntax._
import cats.syntax.either.catsSyntaxEither
import scala.concurrent.{Await, Future}
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.util.{Failure, Success, Try}
private[aws] trait FutureCodec {
implicit def canEncodeFuture[I: Encoder](implicit canEncode: Encoder[I]) =
CanEncode.instance[Future[I]]((os, responseEither, ctx) => {
(for {
response <- responseEither.toTry
futureResult <- Try(Await.result(response, ctx.getRemainingTimeInMillis millis))
json <- Try(canEncode(futureResult).noSpaces.getBytes)
_ <- Try(os.write(json))
} yield {
()
}) match {
case Success(v) => Right(v)
case Failure(e) => Left(e)
}
})
implicit def canEncodeProxyResponse[T](implicit canEncode: CanEncode[T]) = CanEncode.instance[ProxyResponse[T]](
(output, proxyResponseEither, ctx) => {
def writeBody(bodyOption: Option[T]): Either[Throwable, Option[String]] =
bodyOption match {
case None => Right(None)
case Some(body) =>
val os = new ByteArrayOutputStream()
val result = canEncode.writeStream(os, Right(body), ctx)
os.close()
result.map(_ => Some(os.toString()))
}
val proxyResposeOrError = for {
proxyResponse <- proxyResponseEither
bodyOption <- writeBody(proxyResponse.body)
} yield
ProxyResponse[String](
proxyResponse.statusCode,
proxyResponse.headers,
bodyOption
)
val response = proxyResposeOrError match {
case Right(proxyRespose) =>
proxyRespose
case Left(e) =>
ProxyResponse[String](
500,
Some(Map("Content-Type" -> s"text/plain; charset=${Charset.defaultCharset().name()}")),
Some(e.getMessage)
)
}
output.write(response.asJson.noSpaces.getBytes)
Right(())
}
)
}
Example 21
| Source File: Lambda.scala From aws-lambda-scala with MIT License | 5 votes |
|
package io.github.mkotsur.aws.handler
import java.io.{InputStream, OutputStream}
import com.amazonaws.services.lambda.runtime.{Context, RequestStreamHandler}
import io.circe.generic.auto._
import io.github.mkotsur.aws.codecs._
import io.github.mkotsur.aws.proxy.{ProxyRequest, ProxyResponse}
import org.slf4j.LoggerFactory
import cats.syntax.either.catsSyntaxEither
import io.github.mkotsur.aws.handler.Lambda.HandleResult
import scala.language.{higherKinds, postfixOps}
import scala.util.{Failure, Success, Try}
object Lambda extends AllCodec with ProxyRequestCodec {
type Handle[I, O] = (I, Context) => HandleResult[O]
type HandleResult[O] = Either[Throwable, O]
type Proxy[I, O] = Lambda[ProxyRequest[I], ProxyResponse[O]]
object Proxy {
type Handle[I, O] = (ProxyRequest[I], Context) => HandleResult[O]
type HandleResult[O] = Either[Throwable, ProxyResponse[O]]
private type CanDecodeProxyRequest[A] = CanDecode[ProxyRequest[A]]
private type CanEncodeProxyRequest[A] = CanEncode[ProxyResponse[A]]
def instance[I: CanDecodeProxyRequest, O: CanEncodeProxyRequest](
doHandle: Proxy.Handle[I, O]): Lambda[ProxyRequest[I], ProxyResponse[O]] =
new Lambda.Proxy[I, O] {
override protected def handle(i: ProxyRequest[I], c: Context) = doHandle(i, c)
}
}
def instance[I: CanDecode, O: CanEncode](doHandle: Handle[I, O]) =
new Lambda[I, O] {
override protected def handle(i: I, c: Context): Either[Throwable, O] = {
super.handle(i, c)
doHandle(i, c)
}
}
type ReadStream[I] = InputStream => Either[Throwable, I]
type ObjectHandler[I, O] = I => Either[Throwable, O]
type WriteStream[O] = (OutputStream, Either[Throwable, O], Context) => Either[Throwable, Unit]
private val logger = LoggerFactory.getLogger(getClass)
}
abstract class Lambda[I: CanDecode, O: CanEncode] extends RequestStreamHandler {
final def handle(input: InputStream, output: OutputStream, context: Context): Unit =
handleRequest(input, output, context)
// This function will ultimately be used as the external handler
final def handleRequest(input: InputStream, output: OutputStream, context: Context): Unit = {
val read = implicitly[CanDecode[I]].readStream(input)
val handled = read.flatMap { input =>
Try(handle(input, context)) match {
case Success(v) => v
case Failure(e) =>
Lambda.logger.error(s"Error while executing lambda handler: ${e.getMessage}", e)
Left(e)
}
}
val written = implicitly[CanEncode[O]].writeStream(output, handled, context)
output.close()
written.left.foreach(e => throw e)
}
}
Example 22
| Source File: RandomStateSpec.scala From Scalaprof with GNU General Public License v2.0 | 5 votes |
|
package edu.neu.coe.csye._7200.random
import org.scalatest.{FlatSpec, Matchers}
import scala.language.postfixOps
class RandomStateSpec extends FlatSpec with Matchers {
def stdDev(xs: Seq[Double]): Double = math.sqrt(xs.reduceLeft((a, x) => a + x * x)) / xs.length
def mean(xs: Seq[Double]) = xs.sum / xs.length
// XXX Clearly, this doesn't look good. We will soon learn how to write
// generic methods like sum and mean. But for now, this is what we've got.
def sumU(xs: Seq[UniformDouble]): Double = xs.foldLeft(0.0)((a, x) => x + a)
def meanU(xs: Seq[UniformDouble]) = sumU(xs) / xs.length
"RandomState(0L)" should "match case RandomState(4804307197456638271)" in {
val r: RandomState[Long] = RandomState(0L)
r.next should matchPattern { case JavaRandomState(4804307197456638271L, _) => }
}
it should "match case RandomState(-1034601897293430941) on next" in {
val r: RandomState[Long] = RandomState(0L)
r.next.next should matchPattern { case JavaRandomState(-1034601897293430941L, _) => }
}
"7th element of RandomState(0)" should "match case RandomState(5082315122564986995L)" in {
val lrs = RandomState(0).toStream.slice(6, 7)
(lrs head) should matchPattern { case 5082315122564986995L => }
}
"Double stream" should "have zero mean" in {
val xs = RandomState(0).map(RandomState.longToDouble).toStream take 10001 toList
val mu = mean(xs) //xs.sum/xs.length
math.abs(mu) shouldBe <= (2E-2)
}
"0..1 stream" should "have mean = 0.5" in {
val xs = RandomState(0).map(RandomState.longToDouble).map(RandomState.doubleToUniformDouble).toStream take 1001 toList;
math.abs(meanU(xs) - 0.5) shouldBe <=(5E-3)
}
"map" should "work" in {
val rLong: RandomState[Long] = RandomState(0)
val rInt = rLong.map(_.toInt)
rInt.get shouldBe -723955400
val next = rInt.next
next.get shouldBe 406937919
val next2 = next.next
next2.get shouldBe 1407270755
}
it should "work with map of map" in {
val rLong: RandomState[Long] = RandomState(0L)
val rInt = rLong.map(_.toInt)
val rBoolean = rInt.map(_ % 2 == 0)
rBoolean.get shouldBe true
}
"flatMap" should "work" in {
val r1 = RandomState(0)
val r2 = r1.flatMap(RandomState(_))
r2.get shouldBe 4804307197456638271L
}
"for comprehension" should "work" in {
val r1 = RandomState(0)
// TODO this looks wrong: y is never used
val z: RandomState[Double] = for (x <- r1; y <- RandomState(x)) yield x.toDouble/Long.MaxValue
z.get shouldBe -0.5380644352028887 +- 0.0001
}
}
Example 23
| Source File: HedgeFund.scala From Scalaprof with GNU General Public License v2.0 | 5 votes |
|
package com.phasmid.hedge_fund
import com.phasmid.hedge_fund.model._
import com.phasmid.hedge_fund.actors._
import com.phasmid.hedge_fund.portfolio.{Portfolio,PortfolioParser}
import akka.actor.{ Actor, ActorSystem, Props, ActorRef }
import com.typesafe.config.{ ConfigFactory, Config }
import scala.io.Source
import scala.concurrent.ExecutionContext.Implicits.global
object HedgeFund extends App {
val config = ConfigFactory.load()
implicit val system = ActorSystem("HedgeFund")
println(s"""${config.getString("name")}, ${config.getString("appVersion")}""")
val engines: Seq[Query] = config.getString("engine") match {
case "YQL" => Seq(YQLQuery(config.getString("format"), false))
case "Google" => Seq(GoogleQuery("NASDAQ"))
case "YQL,Google" => Seq(YQLQuery(config.getString("format"), false),GoogleQuery("NASDAQ"))
case _ => Seq()
}
println(s"engines: $engines")
val portfolio = getPortfolio(config)
val blackboard = system.actorOf(Props.create(classOf[HedgeFundBlackboard]), "blackboard")
val symbols = getSymbols(config,portfolio)
for (engine <- engines) blackboard ! ExternalLookup(engine.getProtocol, engine.createQuery(symbols))
val optionEngine = new GoogleOptionQuery
symbols foreach {
s => blackboard ! ExternalLookup(optionEngine.getProtocol, optionEngine.createQuery(List(s)))
}
blackboard ! PortfolioUpdate(portfolio)
import scala.language.postfixOps
def getSymbols(config: Config, portfolio: Portfolio) = {
// TODO add in the symbols from the portfolio
config.getString("symbols") split ("\\,") toList;
}
def getPortfolio(config: Config): Portfolio = {
val json = Source.fromFile(config.getString("portfolio")) mkString
val portfolio = PortfolioParser.decode(json)
println(s"portfolio: $portfolio")
portfolio
}
}
Example 24
| Source File: UpdateLogger.scala From Scalaprof with GNU General Public License v2.0 | 5 votes |
|
package com.phasmid.hedge_fund.actors
import akka.actor.{ ActorRef, Props }
import akka.pattern.ask
import akka.util.Timeout
import scala.concurrent.duration._
import scala.concurrent.Await
import scala.language.postfixOps
import com.phasmid.hedge_fund.model.Model
import com.phasmid.hedge_fund.portfolio._
class UpdateLogger(blackboard: ActorRef) extends BlackboardActor(blackboard) {
var portfolio = new Portfolio("", Nil)
override def receive =
{
case Confirmation(id, model, attrs) =>
log.debug(s"update for identifier: $id")
if (model.isOption)
processOption(id, model, attrs)
else
processStock(id, model)
case PortfolioUpdate(p) =>
log.debug(s"portfolio update for: ${p.name}")
portfolio = p
showPortfolio
case m => super.receive(m)
}
implicit val timeout = Timeout(5 seconds)
def processStock(identifier: String, model: Model) = {
model.getKey("price") match {
case Some(p) => {
// sender is the MarketData actor
val future = (sender ? SymbolQuery(identifier, List(p))).mapTo[QueryResponse]
// TODO why are we waiting for this here?
val result = Await.result(future, timeout.duration)
result match {
case QueryResponseValid(k,a) =>
a map {
case (k, v) => log.info(s"$identifier attribute $k has been updated to: $v")
}
case _ =>
}
}
case None => log.warning(s"'price' not defined in model")
}
}
def processOption(identifier: String, model: Model, attributes: Map[String, Any]) = {
val key = "underlying"
attributes.get(key) match {
case Some(value) =>
val future = (blackboard ? OptionQuery("id", value)).mapTo[QueryResponse]
// TODO why are we waiting for this here?
val result = Await.result(future, timeout.duration)
result match {
case QueryResponseValid(k,a) =>
println(s"Action Required: re: qualifying option $identifier with underlying symbol: $k and attributes: $a")
case _ =>
}
case None => log.warning(s"processOption: value not present for $key")
}
}
def showPortfolio {
println(s"Portfolio for ${portfolio.name}")
portfolio.positions foreach { showPosition(_) }
}
def showPosition(position: Position) {
println(s"position for ${position.symbol}: quantity=${position.quantity}; options=")
position.contracts foreach { showContract(_) }
}
def showContract(contract: Contract) {
println(s"contract: $contract")
}
}
Example 25
| Source File: models.scala From Scalaprof with GNU General Public License v2.0 | 5 votes |
|
package com.phasmid.hedge_fund.rss
import java.net.URL
import scala.language.postfixOps
import java.util.Date
case class RssUrl(url: URL) {
override def toString = "RSS: " + url.toString
}
trait RssFeed {
val link: String
val title: String
val desc: String
val items: Seq[RssItem]
override def toString = title + "\n" + desc + "\n**"
def latest = items sortWith ((a, b) => a.date.compareTo(b.date) > 0) head
}
case class AtomRssFeed(title: String, link: String, desc: String, items: Seq[RssItem]) extends RssFeed
case class XmlRssFeed(title: String, link: String, desc: String, language: String, items: Seq[RssItem]) extends RssFeed
case class RssItem(title: String, link: String, desc: String, date: Date, guid: String) {
override def toString = date + " " + title
}
Example 26
| Source File: JsonYQLParserSpec.scala From Scalaprof with GNU General Public License v2.0 | 5 votes |
|
package com.phasmid.hedge_fund.actors
import akka.actor.{ ActorSystem, Actor, Props, ActorRef }
import akka.testkit._
import org.scalatest.{ WordSpecLike, Matchers, BeforeAndAfterAll }
import scala.io.Source
import scala.concurrent.duration._
import spray.http._
import spray.http.MediaTypes._
import org.scalatest.Inside
import scala.language.postfixOps
import spray.http.ContentType.apply
class JsonYQLParserSpec(_system: ActorSystem) extends TestKit(_system) with ImplicitSender
with WordSpecLike with Matchers with Inside with BeforeAndAfterAll {
def this() = this(ActorSystem("JsonYQLParserSpec"))
override def afterAll {
TestKit.shutdownActorSystem(system)
}
import scala.language.postfixOps
val json = Source.fromFile("src/test/resources/yqlExample.json") mkString
"json conversion" in {
val body = HttpEntity(MediaTypes.`application/json`, json.getBytes())
val ok = JsonYQLParser.decode(body) match {
case Right(x) =>
val count = x.query.count
count should equal(4)
x.query.results.quote.length should equal(count)
x.query.results.get(count - 1, "symbol") should matchPattern { case Some("MSFT") => }
case Left(x) =>
fail("decoding error: " + x)
}
}
"send back" in {
val blackboard = system.actorOf(Props.create(classOf[MockYQLBlackboard], testActor), "blackboard")
val entityParser = _system.actorOf(Props.create(classOf[EntityParser], blackboard), "entityParser")
val entity = HttpEntity(MediaTypes.`application/json`, json.getBytes())
entityParser ! EntityMessage("json:YQL", entity)
val msg = expectMsgClass(3.seconds, classOf[QueryResponseValid])
println("msg received: " + msg)
msg should matchPattern {
case QueryResponseValid("MSFT", _) =>
}
inside(msg) {
case QueryResponseValid(symbol, attributes) => attributes.get("Ask") should matchPattern { case Some("46.17") => }
}
}
}
import akka.pattern.ask
import akka.util.Timeout
import scala.concurrent.duration._
import scala.concurrent.Await
import com.phasmid.hedge_fund.model.Model
class MockYQLUpdateLogger(blackboard: ActorRef) extends UpdateLogger(blackboard) {
override def processStock(identifier: String, model: Model) = {
model.getKey("price") match {
case Some(p) => {
// sender is the MarketData actor
val future = sender ? SymbolQuery(identifier, List(p))
val result = Await.result(future, timeout.duration).asInstanceOf[QueryResponseValid]
result.attributes map {
case (k, v) =>
log.info(s"$identifier attribute $k has been updated to: $v")
blackboard ! result
}
}
case None => log.warning(s"'price' not defined in model")
}
}
}
class MockYQLBlackboard(testActor: ActorRef) extends Blackboard(Map(classOf[KnowledgeUpdate] -> "marketData", classOf[SymbolQuery] -> "marketData", classOf[OptionQuery] -> "marketData", classOf[CandidateOption] -> "optionAnalyzer", classOf[Confirmation] -> "updateLogger"),
Map("marketData" -> classOf[MarketData], "optionAnalyzer" -> classOf[OptionAnalyzer], "updateLogger" -> classOf[MockYQLUpdateLogger])) {
override def receive =
{
case msg: Confirmation => msg match {
// Cut down on the volume of messages
case Confirmation("MSFT", _, _) => super.receive(msg)
case _ =>
}
case msg: QueryResponseValid => testActor forward msg
case msg => super.receive(msg)
}
}
Example 27
| Source File: NotebookRestClient.scala From seahorse-workflow-executor with Apache License 2.0 | 5 votes |
|
package io.deepsense.commons.rest.client
import java.net.URL
import scala.concurrent._
import scala.concurrent.duration._
import scala.language.postfixOps
import akka.actor._
import akka.util.Timeout
import spray.client.pipelining._
import spray.http.{HttpResponse, StatusCodes}
import spray.httpx.SprayJsonSupport
import io.deepsense.commons.json.NotebookRestClientProtocol._
import io.deepsense.commons.models.Id
import io.deepsense.commons.rest.client.req.NotebookClientRequest
import io.deepsense.commons.utils.Logging
case class NotebookHttpException(
httpResponse: HttpResponse,
msg: String,
cause: Throwable = null)
extends Exception(msg, cause)
class NotebookRestClient(
notebooksServerAddress: URL,
workflowId: Id,
nodeId: Id,
pollInterval: FiniteDuration,
retryCountLimit: Int
)(implicit override val as: ActorSystem)
extends Logging with RestClient with SprayJsonSupport {
def apiUrl: java.net.URL = new URL(notebooksServerAddress, "/jupyter/")
def credentials: Option[spray.http.HttpCredentials] = None
def userId: Option[java.util.UUID] = None
def userName: Option[String] = None
implicit val timeout: Timeout = 70 minutes
private val filenameExtension = "html"
private val postPath = endpointPath("HeadlessNotebook")
private val getPath = endpointPath(s"HeadlessNotebook/${workflowId}_$nodeId.$filenameExtension")
private val poller = NotebookPoller(this, pollInterval, retryCountLimit, workflowId, nodeId, getPath)
def pollForNotebookData(): Future[Array[Byte]] = poller.tryWork
def generateNotebookData(language: String): Future[HttpResponse] = {
val req = NotebookClientRequest(workflowId, nodeId, language)
fetchHttpResponse(Post(postPath, req)).flatMap { resp => resp.status match {
case StatusCodes.Success(_) => Future.successful(resp)
case statusCode => Future.failed(NotebookHttpException(resp,
s"Notebook server responded with $statusCode when asked to generate notebook data"
))
}}
}
def generateAndPollNbData(language: String): Future[Array[Byte]] = {
generateNotebookData(language).flatMap(_ => pollForNotebookData())
}
def toFactory: NotebooksClientFactory =
new NotebooksClientFactory(notebooksServerAddress, pollInterval, retryCountLimit)
}
class NotebooksClientFactory(notebooksServerAddress: URL, pollInterval: FiniteDuration, retryCountLimit: Int)
(implicit system: ActorSystem) {
def createNotebookForNode(workflow: Id, node: Id): NotebookRestClient = {
new NotebookRestClient(notebooksServerAddress, workflow, node, pollInterval, retryCountLimit)
}
}
Example 28
| Source File: RetrySpec.scala From seahorse-workflow-executor with Apache License 2.0 | 5 votes |
|
package io.deepsense.commons.utils
import scala.concurrent.{Await, Future}
import scala.concurrent.duration._
import scala.language.postfixOps
import akka.actor.ActorSystem
import akka.util.Timeout
import org.scalatest.{Matchers, WordSpec}
import io.deepsense.commons.utils.RetryActor.{RetriableException, RetryLimitReachedException}
class RetrySpec extends WordSpec with Matchers {
val uutName = classOf[Retry[_]].getSimpleName.filterNot(_ == '$')
trait Setup {
def generateUUT[T](retryLimitCount: Int)(toDo: => Future[T]): Retry[T] = new {
override val workDescription = Some("test work")
override val actorSystem: ActorSystem = ActorSystem()
override val retryInterval = 1 nano
override val retryLimit = retryLimitCount
override val timeout = Timeout(1 minute)
} with Retry[T] {
override def work: Future[T] = toDo
}
}
s"A $uutName" should {
"complete its work" when {
"no exceptions are thrown" in {
new Setup {
val uut = generateUUT(0) {
Future.successful(2 * 3 + 8)
}
Await.result(
uut.tryWork, Duration.Inf) shouldBe 14
}
}
"only retriable exceptions are thrown and retry limit is not reached" in {
new Setup {
var count = 3
val uut = generateUUT(3) {
if (count > 0) {
count -= 1
Future.failed(RetriableException(s"Thrown because count is ${count + 1}", None))
} else {
Future.successful("success")
}
}
Await.result(
uut.tryWork, Duration.Inf
) shouldBe "success"
count shouldBe 0
}
}
}
"fail" when {
"retry limit is reached" in {
new Setup {
val uut = generateUUT(10) {
Future.failed(RetriableException(s"This will never succeed, yet we keep trying", None))
}
a [RetryLimitReachedException] shouldBe thrownBy (Await.result(uut.tryWork, Duration.Inf))
}
}
"unexpected exception is thrown" in {
var count = 1
new Setup {
val uut = generateUUT(10) {
if (count == 0) {
Future.failed(new RuntimeException("Thrown because counter reached zero"))
} else {
count -= 1
Future.failed(RetriableException(s"Thrown because counter was ${count + 1}", None))
}
}
a [RuntimeException] shouldBe thrownBy (Await.result(uut.tryWork, Duration.Inf))
count shouldBe 0
}
}
}
}
}
Example 29
| Source File: thunk.scala From Scala-High-Performance-Programming with MIT License | 5 votes |
|
package highperfscala.free
import scala.language.{higherKinds, implicitConversions, postfixOps}
import scalaz.{-\/, Free, Functor, \/, \/-}
case class LimitMs(value: Long) extends AnyVal
sealed trait Thunk[A]
case class Timed[A](
whenActive: () => A,
whenExpired: () => A,
limit: LimitMs) extends Thunk[A]
case class StartProcessing[A](
whenActive: BboUpdated => A,
whenExpired: BboUpdated => A,
limit: LimitMs) extends Thunk[A]
case class TradingDecision[A](
makeDecision: TradingStrategy => A) extends Thunk[A]
object Thunk {
implicit val functor: Functor[Thunk] = new Functor[Thunk] {
def map[A, B](t: Thunk[A])(f: (A) => B): Thunk[B] = t match {
case Timed(whenActive, whenExpired, limit) =>
Timed(() => f(whenActive()), () => f(whenExpired()), limit)
case StartProcessing(whenActive, whenExpired, limit) =>
StartProcessing(c => f(whenActive(c)), c => f(whenExpired(c)), limit)
case TradingDecision(makeDecision) => TradingDecision(
(makeDecision.apply _).andThen(f))
}
}
def timed[L, R](
f: () => R,
exp: () => L,
limit: LimitMs): Free[Thunk, L \/ R] = Free.liftF(
Timed(() => \/-(f()), () => -\/(exp()), limit))
def startProcessing[L, R](
f: BboUpdated => R,
exp: BboUpdated => L,
limit: LimitMs): Free[Thunk, L \/ R] =
Free.liftF(StartProcessing(f.andThen(\/-(_)), exp.andThen(-\/(_)), limit))
def tradingDecision[L, R](f: TradingStrategy => R): Free[Thunk, L \/ R] =
Free.liftF(TradingDecision((f.apply _).andThen(\/-(_))))
}
Example 30
| Source File: QuickSort2.scala From learning-scala with Apache License 2.0 | 5 votes |
|
package com.es.scala.chapter02
import scala.language.postfixOps
class QuickSort2 {
def sort(xs: Array[Int]): Array[Int] = {
if (xs.length <= 1) xs
else {
val pivot = xs(xs.length / 2)
Array.concat(
sort(xs filter (pivot >)),
xs filter (pivot ==),
sort(xs filter (pivot <)))
}
}
}
Example 31
| Source File: BasicAuthenticationSpec.scala From scruid with Apache License 2.0 | 5 votes |
|
package ing.wbaa.druid.auth.basic
import scala.concurrent.duration._
import scala.language.postfixOps
import akka.http.scaladsl.model.StatusCodes
import ing.wbaa.druid.{ DruidConfig, QueryHost, TimeSeriesQuery }
import ing.wbaa.druid.client.{ DruidAdvancedHttpClient, HttpStatusException }
import ing.wbaa.druid.definitions._
import io.circe.generic.auto._
import org.scalatest.concurrent._
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
class BasicAuthenticationSpec extends AnyWordSpec with Matchers with ScalaFutures {
override implicit def patienceConfig: PatienceConfig = PatienceConfig(5 minutes, 100 millis)
private val totalNumberOfEntries = 39244
private val basicAuthenticationAddition =
new BasicAuthenticationExtension(username = "user", password = "aloha")
case class TimeseriesCount(count: Int)
"TimeSeriesQuery without Basic Auth" should {
implicit val config = DruidConfig(
clientBackend = classOf[DruidAdvancedHttpClient],
clientConfig = DruidAdvancedHttpClient
.ConfigBuilder()
.build(),
hosts = Seq(QueryHost("localhost", 8088))
)
"get 401 Auth Required when querying Druid without Authentication config" in {
val request = TimeSeriesQuery(
aggregations = List(
CountAggregation(name = "count")
),
granularity = GranularityType.Hour,
intervals = List("2011-06-01/2017-06-01")
).execute
whenReady(request.failed) { throwable =>
throwable shouldBe a[HttpStatusException]
throwable.asInstanceOf[HttpStatusException].status shouldBe StatusCodes.Unauthorized
}
}
}
"TimeSeriesQuery with Basic Auth" should {
implicit val config = DruidConfig(
clientBackend = classOf[DruidAdvancedHttpClient],
clientConfig = DruidAdvancedHttpClient
.ConfigBuilder()
.withRequestInterceptor(basicAuthenticationAddition)
.build(),
hosts = Seq(QueryHost("localhost", 8088))
)
"successfully query Druid when an Authentication config is set" in {
val request = TimeSeriesQuery(
aggregations = List(
CountAggregation(name = "count")
),
granularity = GranularityType.Hour,
intervals = List("2011-06-01/2017-06-01")
).execute
whenReady(request) { response =>
response.list[TimeseriesCount].map(_.count).sum shouldBe totalNumberOfEntries
}
}
}
}
Example 32
| Source File: IdeaSources.scala From sbt-idea-plugin with Apache License 2.0 | 5 votes |
|
package org.jetbrains.sbtidea.download.idea
import java.net.URL
import java.nio.file.{Files, Path}
import org.jetbrains.sbtidea.download.FileDownloader
import org.jetbrains.sbtidea.{PluginLogger, pathToPathExt}
import sbt._
import org.jetbrains.sbtidea.download.api._
import scala.language.postfixOps
abstract class IdeaSources extends IdeaArtifact {
override type R = IdeaSources
override protected def usedInstaller: Installer[IdeaSources] = new Installer[IdeaSources] {
override def isInstalled(art: IdeaSources)(implicit ctx: InstallContext): Boolean =
ctx.baseDirectory / "sources.zip" exists
override def downloadAndInstall(art: IdeaSources)(implicit ctx: InstallContext): Unit = {
val file = FileDownloader(ctx.baseDirectory.getParent).download(art.dlUrl, optional = true)
Files.move(file, ctx.baseDirectory.resolve("sources.zip"))
PluginLogger.info(s"${caller.buildInfo.edition.name} sources installed")
}
}
}
class IdeaSourcesImpl(override val caller: AbstractIdeaDependency, dlUrlProvider: () => URL) extends IdeaSources {
override def dlUrl: URL = dlUrlProvider()
}
object IdeaSourcesImpl {
def apply(caller: AbstractIdeaDependency, dlUrlProvider: () => URL): IdeaSourcesImpl = new IdeaSourcesImpl(caller, dlUrlProvider)
}
Example 33
| Source File: IntellijPluginParserTest.scala From sbt-idea-plugin with Apache License 2.0 | 5 votes |
|
package org.jetbrains.sbtidea.download.plugin
import org.jetbrains.sbtidea.Keys.IntellijPlugin._
import org.jetbrains.sbtidea.Keys._
import org.scalatest.{FunSuite, Matchers}
import scala.language.postfixOps
final class IntellijPluginParserTest extends FunSuite with Matchers {
private def id(str: String) = str.toPlugin.asInstanceOf[Id]
test("Id parser should parse simple id") {
"org.jetbrains.scala".toPlugin shouldBe a [Id]
}
test("plugin url should parse as Url") {
"https://foo.bar/a.zip".toPlugin shouldBe an [Url]
"http://foo.bar/a.zip".toPlugin shouldBe an [Url]
"plugin:http://foo.bar/a.zip".toPlugin shouldBe an [Url]
}
test("id should parse with with mixed segments") {
id("foo").id shouldBe "foo"
id("foo:").id shouldBe "foo"
id("foo::").id shouldBe "foo"
id("foo::baz").id shouldBe "foo"
id("foo:123:").id shouldBe "foo"
id("foo:123:baz").id shouldBe "foo"
}
test("parser should not parse invalid strings") {
assertThrows[RuntimeException]("::".toPlugin)
assertThrows[RuntimeException](":123:foo".toPlugin)
assertThrows[RuntimeException](":123:".toPlugin)
}
}
Example 34
| Source File: UniversalButtons.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4.buttons
import scala.language.postfixOps
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.{ClassModifiers, RenderingContext}
trait UniversalButtons extends UniversalButtonStates with UniversalButtonGroups { self: RenderingContext with BootstrapComponents with ClassModifiers with Buttons ⇒
import scalaTags.all._
type Button = ButtonBuilder
object Button extends ButtonFactory {
case class ButtonBuilder(style: ButtonStyle = ButtonStyle.default, size: ButtonSize = ButtonSize.default,
block: Boolean = false, active: Boolean = false, disabled: Boolean = false)
extends UniversalButton {
def withStyle(style: ButtonStyle): ButtonBuilder = copy(style = style)
def withSize(size: ButtonSize): ButtonBuilder = copy(size = size)
}
}
Example 35
| Source File: TodoList.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.test.frontend
import java.util.UUID
import scala.language.postfixOps
import rx._
import com.karasiq.bootstrap.Bootstrap.default._
import scalaTags.all._
object TodoList {
sealed abstract class ItemPriority(val style: TableRowStyle)
object ItemPriority {
case object Low extends ItemPriority(TableRowStyle.success)
case object Normal extends ItemPriority(TableRowStyle.info)
case object High extends ItemPriority(TableRowStyle.danger)
def fromString(s: String): ItemPriority = Seq(Low, Normal, High).find(_.toString == s).get
}
case class Item(title: String, priority: ItemPriority = ItemPriority.Normal, completed: Boolean = false)
def apply(): TodoList = {
new TodoList()
}
}
final class TodoList extends BootstrapHtmlComponent {
import TodoList._
val items: Var[Seq[Var[Item]]] = Var(Nil)
def removeCompleted(): Unit = {
items() = items.now.filterNot(_.now.completed)
}
def addTestData(): Unit = {
items() = items.now ++ (for (_ <- 1 to 20) yield Var(Item(s"Test ${UUID.randomUUID()}", ItemPriority.Low)))
}
private[this] def showDialog(title: String, priority: ItemPriority)(onApply: (String, ItemPriority) ⇒ Unit): Unit = {
val titleText = Var(title)
val prioritySelect = FormInput.simpleSelect("Priority", "Low", "Normal", "High")
prioritySelect.selected.update(Seq(priority.toString))
Modal("Add/edit item")
.withBody(Form(
FormInputGroup(FormInputGroup.label("Title"), FormInputGroup.addon("file-text-o".fontAwesome(FontAwesome.fixedWidth)), FormInputGroup.text(placeholder := "Write description", titleText.reactiveInput)),
prioritySelect
))
.withButtons(Modal.closeButton("Cancel"), Modal.button("Apply", Modal.dismiss, onclick := Callback.onClick { _ ⇒
onApply(titleText.now, ItemPriority.fromString(prioritySelect.selected.now.head))
}))
.show(backdrop = false)
}
def showEditDialog(item: Var[Item]): Unit = {
showDialog(item.now.title, item.now.priority) { (title, priority) ⇒
item() = item.now.copy(title, priority)
}
}
def showAddDialog(): Unit = {
showDialog("", ItemPriority.Normal) { (title, priority) ⇒
items() = items.now :+ Var(Item(title, priority))
}
}
private[this] def renderItem(item: Var[Item]): TableRow = {
def todoTitle = Rx(if (item().completed) s(item().title, color.gray) else b(item().title))
def buttons = ButtonGroup(ButtonGroupSize.small,
Button(ButtonStyle.primary)("Edit", onclick := Callback.onClick(_ ⇒ showEditDialog(item))),
Button(ButtonStyle.danger)("Remove", onclick := Callback.onClick(_ ⇒ items.update(items.now.filter(_.ne(item)))))
)
TableRow(
Seq(
Seq[Modifier](todoTitle, GridSystem.col(10), onclick := Callback.onClick(_ ⇒ item.update(item.now.copy(completed = !item.now.completed)))),
Seq[Modifier](buttons, GridSystem.col(2), textAlign.center)
),
Rx(`class` := {
if (item().completed) "" else item().priority.style.styleClass.getOrElse("")
}).auto
)
}
override def renderTag(md: ModifierT*): TagT = {
val heading = Rx(Seq[Modifier](
Seq[Modifier]("Description", GridSystem.col(10)),
Seq[Modifier]("Actions", GridSystem.col(2)))
)
val table = PagedTable(heading, items.map(_.map(renderItem)), 5)
Panel(style = PanelStyle.success)
.withHeader(Panel.title("th-list".glyphicon, span("Scala.js Todo", Bootstrap.nbsp, Rx(Bootstrap.badge(items().count(i ⇒ !i().completed)))), Panel.buttons(
Panel.button("plus".glyphicon, onclick := Callback.onClick(_ ⇒ showAddDialog())),
Panel.button("trash".glyphicon, onclick := Callback.onClick(_ ⇒ removeCompleted())),
Panel.button("flash".glyphicon, onclick := Callback.onClick(_ ⇒ addTestData()))
)))
.renderTag(table.renderTag(TableStyle.bordered, TableStyle.hover, TableStyle.striped, TableStyle.condensed))
}
}
Example 36
| Source File: TestHtmlPage.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.test.frontend
import scala.language.postfixOps
import rx._
import com.karasiq.bootstrap.Bootstrap.text._
import scalaTags.all._
object TestHtmlPage {
def apply(): String = {
"<!doctype html>" + html(head(
base(href := "/"),
meta(httpEquiv := "content-type", content := "text/html; charset=utf-8"),
meta(name := "viewport", content := "width=device-width, initial-scale=1.0"),
script(src := "https://code.jquery.com/jquery-1.12.0.js"),
raw(bootstrapCdnLinks),
scalaTags.tags2.style(raw(fontAwesomeCss)),
script(raw(activateTooltipScript)),
scalaTags.tags2.title("Bootstrap text page")
), body(
new TestContainer
))
}
private[this] class TestContainer extends BootstrapComponent {
def render(md: ModifierT*): ModifierT = {
val testModal = this.createModal
val rxText = Var("ERROR") // Pseudo-reactive binding
val navigationBar = NavigationBar()
.withBrand(rxText, href := "http://getbootstrap.com/components/#navbar")
.withTabs(
NavigationTab("Table", "table", "table".faFwIcon, this.createTable),
NavigationTab("Carousel", "carousel", "picture".glyphicon, this.createCarousel),
NavigationTab("Buttons", "empty", "address-book".faFwIcon, Bootstrap.jumbotron(
Bootstrap.button("Modal", testModal.toggle)
))
)
.withContentContainer(e ⇒ GridSystem.container(GridSystem.mkRow(e), marginTop := 60.px))
.build()
rxText() = "Scala.js Bootstrap Test"
Seq[ModifierT](testModal, navigationBar)
}
private[this] def createTable = {
val table = PagedTable(Rx(Seq("Number", "Square")), Rx(TableRow(Seq(1, 1), Tooltip(b("First row")),
onclick := Callback.onClick(_ ⇒ println("Pseudo callback"))) +: (2 to 100).map(i ⇒ TableRow.data(i, i * i))))
table.renderTag(TableStyle.bordered, TableStyle.hover, TableStyle.striped)
}
private[this] def createModal = {
Modal()
.withBody("Text-rendered modal")
}
private[this] def createCarousel = {
TestCarousel("https://upload.wikimedia.org/wikipedia/commons/9/9e/Scorpius_featuring_Mars_and_Saturn._%2828837147345%29.jpg")
}
}
private[this] def bootstrapCdnLinks: String =
"""
|<!-- Latest compiled and minified CSS -->
|<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
|
|<!-- Optional theme -->
|<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap-theme.min.css" integrity="sha384-rHyoN1iRsVXV4nD0JutlnGaslCJuC7uwjduW9SVrLvRYooPp2bWYgmgJQIXwl/Sp" crossorigin="anonymous">
|
|<!-- Latest compiled and minified JavaScript -->
|<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
|""".stripMargin
private[this] def fontAwesomeCss: String =
"""
|
|@import url('https://use.fontawesome.com/releases/v4.7.0/css/font-awesome-css.min.css');
|@font-face {
| font-family: 'FontAwesome';
| src: url('https://use.fontawesome.com/releases/v4.7.0/fonts/fontawesome-webfont.eot');
| src: url('https://use.fontawesome.com/releases/v4.7.0/fonts/fontawesome-webfont.eot?#iefix') format('embedded-opentype'),
| url('https://use.fontawesome.com/releases/v4.7.0/fonts/fontawesome-webfont.woff2') format('woff2'),
| url('https://use.fontawesome.com/releases/v4.7.0/fonts/fontawesome-webfont.woff') format('woff'),
| url('https://use.fontawesome.com/releases/v4.7.0/fonts/fontawesome-webfont.ttf') format('truetype'),
| url('https://use.fontawesome.com/releases/v4.7.0/fonts/fontawesome-webfont.svg#fontawesomeregular') format('svg');
| font-weight: normal;
| font-style: normal;
|}
""".stripMargin
private[this] def activateTooltipScript: String =
"""
|$(function () {
| $('[data-toggle="tooltip"]').tooltip()
|})
""".stripMargin
}
Example 37
| Source File: TestHtmlPage.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4.test.frontend
import scala.language.postfixOps
import rx._
import com.karasiq.bootstrap4.Bootstrap.text._
import scalaTags.all._
object TestHtmlPage {
def apply(): String = {
"<!doctype html>" + html(head(
base(href := "/"),
meta(httpEquiv := "content-type", content := "text/html; charset=utf-8"),
meta(name := "viewport", content := "width=device-width, initial-scale=1.0"),
script(src := "https://code.jquery.com/jquery-3.2.1.js"),
raw(bootstrapCdnLinks),
scalaTags.tags2.style(raw(fontAwesomeCss)),
script(raw(activateTooltipScript)),
scalaTags.tags2.title("Bootstrap text page")
), body(
new TestContainer
))
}
private[this] class TestContainer extends BootstrapComponent {
def render(md: ModifierT*): ModifierT = {
val testModal = this.createModal
val rxText = Var("ERROR") // Pseudo-reactive binding
val navigationBar = NavigationBar()
.withBrand(rxText, href := "http://getbootstrap.com/components/#navbar")
.withTabs(
NavigationTab("Table", "table", "table".faFwIcon, this.createTable),
NavigationTab("Carousel", "carousel", "file-image-o".faFwIcon, this.createCarousel),
NavigationTab("Buttons", "empty", "address-book".faFwIcon, Bootstrap.jumbotron(
Bootstrap.button("Modal", testModal.toggle)
))
)
.withContentContainer(e ⇒ GridSystem.container(GridSystem.mkRow(e), marginTop := 60.px))
.build()
rxText() = "Scala.js Bootstrap Test"
Seq[ModifierT](testModal, navigationBar)
}
private[this] def createTable = {
val table = PagedTable(Rx(Seq("Number", "Square")), Rx(TableRow(Seq(1, 1), Tooltip(b("First row")),
onclick := Callback.onClick(_ ⇒ println("Pseudo callback"))) +: (2 to 100).map(i ⇒ TableRow.data(i, i * i))))
table.renderTag(TableStyle.bordered, TableStyle.hover, TableStyle.striped)
}
private[this] def createModal = {
Modal()
.withBody("Text-rendered modal")
}
private[this] def createCarousel = {
TestCarousel("https://upload.wikimedia.org/wikipedia/commons/9/9e/Scorpius_featuring_Mars_and_Saturn._%2828837147345%29.jpg")
}
}
private[this] def bootstrapCdnLinks: String =
"""
|<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta.2/css/bootstrap.min.css">
|""".stripMargin
private[this] def fontAwesomeCss: String =
"""
|
|@import url('https://use.fontawesome.com/releases/v4.7.0/css/font-awesome-css.min.css');
|@font-face {
| font-family: 'FontAwesome';
| src: url('https://use.fontawesome.com/releases/v4.7.0/fonts/fontawesome-webfont.eot');
| src: url('https://use.fontawesome.com/releases/v4.7.0/fonts/fontawesome-webfont.eot?#iefix') format('embedded-opentype'),
| url('https://use.fontawesome.com/releases/v4.7.0/fonts/fontawesome-webfont.woff2') format('woff2'),
| url('https://use.fontawesome.com/releases/v4.7.0/fonts/fontawesome-webfont.woff') format('woff'),
| url('https://use.fontawesome.com/releases/v4.7.0/fonts/fontawesome-webfont.ttf') format('truetype'),
| url('https://use.fontawesome.com/releases/v4.7.0/fonts/fontawesome-webfont.svg#fontawesomeregular') format('svg');
| font-weight: normal;
| font-style: normal;
|}
""".stripMargin
private[this] def activateTooltipScript: String =
"""
|$(function () {
| $('[data-toggle="tooltip"]').tooltip()
|})
""".stripMargin
}
Example 38
| Source File: JSCarousels.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4.carousel
import scala.language.postfixOps
import scala.scalajs.js
import rx.Rx
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.JSRenderingContext
import com.karasiq.bootstrap.jquery.BootstrapJQueryContext
import com.karasiq.bootstrap4.icons.Icons
import com.karasiq.bootstrap4.utils.Utils
trait JSCarousels extends UniversalCarousels { self: JSRenderingContext with Carousels with Utils with Icons with BootstrapComponents with BootstrapJQueryContext ⇒
import scalaTags.all._
type Carousel = JSCarousel
object Carousel extends CarouselFactory {
def apply(data: Rx[Seq[Modifier]], id: String = Bootstrap.newId): JSCarousel = {
new JSCarousel(id, data)
}
def slide(image: String, content: Modifier*): Modifier = {
UniversalCarousel.slide(image, content)
}
}
class JSCarousel(carouselId: String, content: Rx[Seq[Modifier]]) extends UniversalCarousel(carouselId, content) {
def create(interval: Int = 5000, pause: String = "hover",
wrap: Boolean = true, keyboard: Boolean = true,
modifiers: Modifier = Bootstrap.noModifier): Element = {
val element = carousel(modifiers).render
val options = js.Object().asInstanceOf[JSCarouselOptions]
options.interval = interval
options.pause = pause
options.wrap = wrap
options.keyboard = keyboard
jQuery(element).carousel(options)
element
}
override def render(md: Modifier*): Modifier = {
create(modifiers = md)
}
}
}
Example 39
| Source File: JSModals.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4.modal
import scala.language.postfixOps
import scala.scalajs.js
import scala.scalajs.js.|
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.JSRenderingContext
import com.karasiq.bootstrap.jquery.BootstrapJQueryContext
trait JSModals { self: JSRenderingContext with Modals with BootstrapComponents with BootstrapJQueryContext ⇒
implicit class JSModalOps(modal: Modal) {
def show(backdrop: Boolean | String = true, keyboard: Boolean = true,
show: Boolean = true, events: Map[String, js.Any] = Map.empty): Unit = {
val dialog = jQuery(modal.renderTag().render)
val options = js.Object().asInstanceOf[JSModalOptions]
options.backdrop = backdrop
options.keyboard = keyboard
options.show = show
dialog.modal(options)
dialog.on("hidden.bs.modal", () ⇒ {
// Remove from DOM
dialog.remove()
})
events.foreach { case (event, handler) ⇒
dialog.on(event, handler)
}
dialog.modal("show")
}
}
}
Example 40
| Source File: JSPopovers.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4.popover
import scala.language.postfixOps
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.JSRenderingContext
import com.karasiq.bootstrap.jquery.BootstrapJQueryContext
import com.karasiq.bootstrap4.tooltip.Tooltips
trait JSPopovers { self: JSRenderingContext with BootstrapComponents with Popovers with Tooltips with BootstrapJQueryContext ⇒
import scalaTags.all._
class JSPopover(val options: PopoverOptions) extends Popover {
def toggle: Modifier = new Modifier {
def applyTo(t: Element): Unit = {
val jsOptions = scalajs.js.Object().asInstanceOf[JSPopoverOptions]
def set(value: String, f: String ⇒ Unit) = if (value.nonEmpty) f(value)
jsOptions.animation = options.animation
jsOptions.content = options.content.render
jsOptions.title = options.title.render
jsOptions.html = options.html
jsOptions.placement = options.placement.toString
set(options.container, jsOptions.container = _)
set(options.delay, jsOptions.delay = _)
set(options.selector, jsOptions.selector = _)
set(options.template, jsOptions.template = _)
set(options.trigger, jsOptions.trigger = _)
set(options.viewport, jsOptions.viewport = _)
jQuery(t).popover(jsOptions)
}
}
override def render(md: Modifier*): Modifier = {
toggle +: md
}
}
object Popover extends PopoverFactory {
def apply(title: Frag, content: Frag, placement: TooltipPlacement = TooltipPlacement.auto): JSPopover = {
new JSPopover(PopoverOptions(html = true, title = title, content = content, placement = placement))
}
}
}
Example 41
| Source File: JSBootstrapBundle.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4
import scala.language.postfixOps
import rx.Ctx
import com.karasiq.bootstrap.context.JSRenderingContext
import com.karasiq.bootstrap.jquery.BootstrapJQueryContext
import com.karasiq.bootstrap4.carousel.JSCarousels
import com.karasiq.bootstrap4.modal.JSModals
import com.karasiq.bootstrap4.navbar.JSNavigationBars
import com.karasiq.bootstrap4.popover.JSPopovers
import com.karasiq.bootstrap4.tooltip.JSTooltips
// JS components implementation
trait JSBootstrapBundle extends UniversalBootstrapBundle with JSRenderingContext with JSModals with JSTooltips
with JSPopovers with JSNavigationBars with JSCarousels with BootstrapJQueryContext
object JSBootstrapBundle {
def apply()(implicit rx: Ctx.Owner): JSBootstrapBundle = {
new JSBootstrapBundle {
implicit val scalaRxContext = rx
}
}
}
Example 42
| Source File: Buttons.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4.buttons
import scala.language.postfixOps
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.{ClassModifiers, RenderingContext}
import com.karasiq.bootstrap4.utils.Utils
trait Buttons extends ButtonStyles with ButtonGroups with ButtonStates { self: RenderingContext with BootstrapComponents with ClassModifiers with Utils ⇒
import scalaTags.all._
type Button <: AbstractButton
val Button: ButtonFactory
trait AbstractButton extends BootstrapHtmlComponent {
def style: ButtonStyle
def size: ButtonSize
def block: Boolean
def active: Boolean
def disabled: Boolean
}
trait ButtonFactory {
def apply(style: ButtonStyle = ButtonStyle.default, size: ButtonSize = ButtonSize.default,
block: Boolean = false, active: Boolean = false, disabled: Boolean = false): Button
}
}
Example 43
| Source File: BootstrapTestApp.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.test.frontend
import scala.language.postfixOps
import org.scalajs.dom
import org.scalajs.dom.window
import rx._
import com.karasiq.bootstrap.Bootstrap.default._
import scalaTags.all._
import com.karasiq.bootstrap.jquery.BootstrapJQueryContext
object BootstrapTestApp {
def main(args: Array[String]): Unit = {
BootstrapJQueryContext.useNpmImports()
jQuery(() ⇒ {
// Table tab will appear after 3 seconds
val tableVisible = Var(false)
val tabTitle = Var("Wait...")
// Show table tab in 3 seconds
window.setTimeout(() ⇒ {
tableVisible.update(true)
window.setTimeout(() ⇒ { tabTitle() = "Table" }, 1000)
}, 3000)
val tabs = Var(Seq[NavigationTab](
NavigationTab(tabTitle, "table", "table".faFwIcon, TestTable(), tableVisible.reactiveShow),
NavigationTab("Carousel", "carousel", "picture".glyphicon, TestCarousel("https://upload.wikimedia.org/wikipedia/commons/thumb/5/5e/Big_Wood%2C_N2.JPG/1280px-Big_Wood%2C_N2.JPG")),
NavigationTab("ToDo list", "todo", "fort-awesome".faFwIcon, TodoList()),
NavigationTab("Text rendering", "text", "envelope".glyphicon, Bootstrap.jumbotron(
FormInput.textArea(a("Text rendering", href := "./serverside.html"), rows := 30, readonly, TestHtmlPage())
))
))
val navigationBar = NavigationBar()
.withBrand("Scala.js Bootstrap Test", href := "https://github.com/Karasiq/scalajs-bootstrap")
.withTabs(tabs)
.withContentContainer(content ⇒ GridSystem.container(id := "main-container", GridSystem.mkRow(content)))
.withStyles(NavigationBarStyle.inverse, NavigationBarStyle.fixedTop)
.build()
// Render page
navigationBar.applyTo(dom.document.body)
// Reactive navbar test
tabs() = tabs.now :+ NavigationTab("Buttons", "buttons", "log-in".glyphicon, TestPanel("Serious business panel", PanelStyle.warning))
navigationBar.selectTab(2)
})
}
}
Example 44
| Source File: PagedTables.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4.table
import scala.language.postfixOps
import rx._
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.{ClassModifiers, RenderingContext}
import com.karasiq.bootstrap4.pagination.PageSelectors
trait PagedTables { self: RenderingContext with BootstrapComponents with Tables with PageSelectors with ClassModifiers ⇒
import scalaTags.all._
type PagedTable <: AbstractPagedTable with BootstrapHtmlComponent
val PagedTable: PagedTableFactory
trait AbstractPagedTable {
def table: Table
def pageSelector: PageSelector
}
trait PagedTableFactory {
def apply(heading: Rx[Seq[Modifier]], content: Rx[Seq[TableRow]], perPage: Int = 20): PagedTable
def static(heading: Seq[Modifier], content: Seq[TableRow], perPage: Int = 20): PagedTable = {
this.apply(Var(heading), Var(content), perPage)
}
private[table] def pagesRx(allContent: Rx[Seq[TableRow]], rowsPerPage: Int): Rx[Int] = {
Rx {
val data = allContent()
val result = if (data.isEmpty) {
1
} else if (data.length % rowsPerPage == 0) {
data.length / rowsPerPage
} else {
data.length / rowsPerPage + 1
}
result
}
}
private[table] def pagedDataRx(allContent: Rx[Seq[TableRow]], currentPage: Rx[Int], rowsPerPage: Int): Rx[Seq[TableRow]] = {
Rx {
val data = allContent()
val page = currentPage()
data.slice(rowsPerPage * (page - 1), rowsPerPage * (page - 1) + rowsPerPage)
}
}
}
trait AbstractStaticPagedTable extends AbstractPagedTable {
def allContent: Rx[Seq[TableRow]]
def rowsPerPage: Int
}
}
Example 45
| Source File: SortableTables.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4.table
import scala.language.{higherKinds, postfixOps}
import rx._
import com.karasiq.bootstrap.context.RenderingContext
import com.karasiq.bootstrap4.utils.Utils
trait SortableTables extends TableCols { self: RenderingContext with PagedTables with Utils ⇒
import scalaTags.all._
type SortableTable[T] <: AbstractSortableTable[T] with BootstrapHtmlComponent
val SortableTable: AbstractSortableTableFactory
trait AbstractSortableTableFactory {
def apply[T](items: Rx[Seq[T]], columns: Rx[Seq[TableCol[T, _]]],
rowModifiers: T ⇒ Modifier = (_: T) ⇒ Bootstrap.noModifier,
filterItem: (T, String) ⇒ Boolean = (i: T, f: String) ⇒ i.toString.contains(f)): SortableTable[T]
}
trait AbstractSortableTable[T] {
def items: Rx[Seq[T]]
def columns: Rx[GenTableCols[T]]
def sortByColumn: Var[GenTableCol[T]]
def reverseOrdering: Var[Boolean]
def filter: Var[String]
def filterItem(item: T, filter: String): Boolean
def rowModifiers(item: T): Modifier
def setOrdering(column: TableCol[T, _]): Unit = {
if (sortByColumn.now == column) reverseOrdering() = !reverseOrdering.now
else sortByColumn() = column
}
def pagedTable: PagedTable
}
}
Example 46
| Source File: Tables.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4.table
import scala.language.postfixOps
import rx.{Rx, Var}
import com.karasiq.bootstrap.context.RenderingContext
import com.karasiq.bootstrap4.utils.Utils
trait Tables extends TableRows with TableStyles { self: RenderingContext with Utils ⇒
import scalaTags.all._
type Table <: AbstractTable with BootstrapHtmlComponent
val Table: TableFactory
trait AbstractTable {
def heading: Rx[Seq[Modifier]]
def content: Rx[Seq[TableRow]]
}
trait TableFactory {
def apply(heading: Rx[Seq[Modifier]], content: Rx[Seq[TableRow]]): Table
def static(heading: Seq[Modifier], content: Seq[TableRow]): Table = {
apply(Var(heading), Var(content))
}
}
}
Example 47
| Source File: TextCarousels.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4.carousel
import scala.language.postfixOps
import rx.Rx
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.RenderingContext
import com.karasiq.bootstrap4.icons.Icons
import com.karasiq.bootstrap4.utils.Utils
trait TextCarousels extends UniversalCarousels { self: RenderingContext with Carousels with Utils with Icons with BootstrapComponents ⇒
import scalaTags.all._
type Carousel = UniversalCarousel
object Carousel extends CarouselFactory {
def apply(data: Rx[Seq[Modifier]], id: String = Bootstrap.newId): UniversalCarousel = {
new UniversalCarousel(id, data)
}
def slide(image: String, content: Modifier*): Modifier = {
UniversalCarousel.slide(image, content)
}
}
}
Example 48
| Source File: TextPopovers.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4.popover
import scala.language.postfixOps
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.RenderingContext
import com.karasiq.bootstrap4.tooltip.Tooltips
import com.karasiq.bootstrap4.utils.Utils
trait TextPopovers { self: RenderingContext with BootstrapComponents with Tooltips with Popovers with Utils ⇒
import scalaTags.all._
import BootstrapAttrs._
class TextPopover(val options: PopoverOptions) extends Popover {
override def render(md: ModifierT*): ModifierT = {
(`data-toggle` := "popover") +: Bootstrap.dataProps(options.toStrings: _*) +: md
}
}
object Popover extends PopoverFactory {
def apply(title: Frag, content: Frag, placement: TooltipPlacement = TooltipPlacement.auto): Popover = {
val options = PopoverOptions(html = true, title = title, content = content, placement = placement)
new TextPopover(options)
}
}
}
Example 49
| Source File: Popovers.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4.popover
import scala.language.postfixOps
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.RenderingContext
import com.karasiq.bootstrap4.tooltip.Tooltips
import com.karasiq.bootstrap4.utils.Utils
trait Popovers { self: RenderingContext with BootstrapComponents with Tooltips with Utils ⇒
import scalaTags.all._
case class PopoverOptions(animation: Boolean = true, container: String = "", content: Frag = "", delay: String = "",
html: Boolean = false, placement: TooltipPlacement = TooltipPlacement.right, selector: String = "", template: String = "",
title: Frag = "", trigger: String = "", viewport: String = "") {
def toStrings: Seq[(String, String)] = {
def opt[T](name: String, value: T, default: T) = Option(name → value).filterNot(_._2 == default)
Seq(
opt("animation", animation, true),
opt("container", container, ""),
opt("content", content, ""),
opt("delay", delay, ""),
opt("html", html, false),
opt("placement", placement, TooltipPlacement.right),
opt("selector", selector, ""),
opt("template", template, ""),
opt("title", title, ""),
opt("trigger", trigger, ""),
opt("viewport", viewport, "")
).flatten.map(kv ⇒ kv._1 → kv._2.toString)
}
}
trait Popover extends BootstrapComponent {
def options: PopoverOptions
}
trait PopoverFactory {
def apply(title: Frag, content: Frag, placement: TooltipPlacement = TooltipPlacement.auto): Popover
}
val Popover: PopoverFactory
}
Example 50
| Source File: TextBootstrapBundle.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4
import scala.language.postfixOps
import rx.Ctx
import com.karasiq.bootstrap.context.TextRenderingContext
import com.karasiq.bootstrap4.carousel.TextCarousels
import com.karasiq.bootstrap4.popover.TextPopovers
import com.karasiq.bootstrap4.tooltip.TextTooltips
// Text components implementation
trait TextBootstrapBundle extends UniversalBootstrapBundle with TextRenderingContext
with TextCarousels with TextTooltips with TextPopovers
object TextBootstrapBundle {
def apply()(implicit ctx: Ctx.Owner): TextBootstrapBundle = {
new TextBootstrapBundle {
implicit val scalaRxContext = ctx
}
}
}
Example 51
| Source File: BootstrapBundle.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4
import scala.language.postfixOps
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.{ClassModifiers, RenderingContext}
import com.karasiq.bootstrap4.alert.{Alerts, UniversalAlerts}
import com.karasiq.bootstrap4.buttons.{Buttons, UniversalButtons}
import com.karasiq.bootstrap4.card.{Cards, UniversalCards}
import com.karasiq.bootstrap4.carousel.{Carousels, UniversalCarousels}
import com.karasiq.bootstrap4.collapse.{Collapses, UniversalCollapses}
import com.karasiq.bootstrap4.dropdown.{Dropdowns, UniversalDropdowns}
import com.karasiq.bootstrap4.form.{Forms, UniversalForms}
import com.karasiq.bootstrap4.grid.{Grids, UniversalGrids}
import com.karasiq.bootstrap4.icons.{Icons, UniversalIcons}
import com.karasiq.bootstrap4.modal.{Modals, UniversalModals}
import com.karasiq.bootstrap4.navbar.{NavigationBars, UniversalNavigationBars}
import com.karasiq.bootstrap4.pagination.{PageSelectors, UniversalPageSelectors}
import com.karasiq.bootstrap4.popover.Popovers
import com.karasiq.bootstrap4.progressbar.{ProgressBars, UniversalProgressBars}
import com.karasiq.bootstrap4.table._
import com.karasiq.bootstrap4.tooltip.Tooltips
import com.karasiq.bootstrap4.utils.{UniversalUtils, Utils}
// Abstract components
trait BootstrapBundle extends RenderingContext with BootstrapComponents with ClassModifiers with Alerts with Buttons
with Carousels with Collapses with Dropdowns with Forms with Grids with Icons with Modals with NavigationBars
with Cards with Popovers with ProgressBars with Tables with PageSelectors with PagedTables with SortableTables
with Tooltips with Utils
// Default components implementation
trait UniversalBootstrapBundle extends BootstrapBundle
with UniversalPageSelectors with UniversalTables with UniversalPagedTables with UniversalSortableTables
with UniversalProgressBars with UniversalCards with UniversalNavigationBars with UniversalModals
with UniversalIcons with UniversalGrids with UniversalForms with UniversalDropdowns with UniversalCollapses
with UniversalCarousels with UniversalButtons with UniversalAlerts with UniversalUtils
Example 52
| Source File: TestPanel.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.test.frontend
import scala.concurrent.duration._
import scala.language.postfixOps
import org.scalajs.dom.window
import rx._
import com.karasiq.bootstrap.Bootstrap.default._
import scalaTags.all.{span, _}
object TestPanel {
def apply(panelTitle: String, style: PanelStyle): TestPanel = {
new TestPanel(panelTitle, style)
}
}
final class TestPanel(panelTitle: String, style: PanelStyle) extends BootstrapHtmlComponent {
override def renderTag(md: ModifierT*): TagT = {
val titleVar = Var[Frag]("ERROR")
val successButton = Button(ButtonStyle.success)("Win 10000000$", onclick := Callback.onClick(_ ⇒ TestModal().show()), Tooltip(i("Press me"), TooltipPlacement.left)).render
val dangerButton = Button(ButtonStyle.danger)("Format C:\\", Popover(span(titleVar), "Popover test", TooltipPlacement.right)).render
titleVar() = i("Boom")
val toggleButton = ToggleButton(Bootstrap.button("Toggle me"))
val disabledButton = DisabledButton(Bootstrap.button("Heavy computation"))
disabledButton.state.foreach { pressed ⇒
if (pressed) {
window.setTimeout(() ⇒ {
window.alert(s"Answer: ${if (toggleButton.state.now) 321 else 123}")
disabledButton.state() = false
}, 1000)
}
}
// Render panel
val panelId = Bootstrap.newId
val collapseBtnTitle = Var("ERROR")
val panel = Panel(panelId, style)
.withHeader(Panel.title("euro".glyphicon, Panel.collapse(panelId, panelTitle, Bootstrap.nbsp, Bootstrap.badge("42")), Panel.buttons(
Panel.button("plus".glyphicon, onclick := Callback.onClick(_ ⇒ window.alert("Panel add"))),
Panel.button("minus".glyphicon, onclick := Callback.onClick(_ ⇒ window.alert("Panel remove")))
)))
.renderTag(
new TestProgressBar(ProgressBarStyle.success, 300 millis),
Navigation.tabs(
NavigationTab("Simple buttons", Bootstrap.newId, "remove".glyphicon, Bootstrap.well(
GridSystem.mkRow(
small("Hint: press the green button for reactive forms test", Bootstrap.textStyle.info)
),
GridSystem.mkRow(
ButtonGroup(ButtonGroupSize.default, successButton, dangerButton)
),
GridSystem.mkRow(Collapse(collapseBtnTitle)(
GridSystem.row(
GridSystem.col(6)(Dropdown("Dropdown", Dropdown.item("Test 1", onclick := Callback.onClick(_ ⇒ window.alert("Test 1"))), Dropdown.item("Test 2"))),
GridSystem.col(6)(Dropdown.dropup("Dropup", Dropdown.item("Test 3", onclick := Callback.onClick(_ ⇒ window.alert("Test 3"))), Dropdown.item("Test 4")))
)
))
)),
NavigationTab("Reactive buttons", Bootstrap.newId, "play-circle".glyphicon, Bootstrap.well(
ButtonGroup(ButtonGroupSize.large, toggleButton, disabledButton)
))
)
)
collapseBtnTitle() = "Dropdowns"
panel
}
}
Example 53
| Source File: BootstrapTestApp.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4.test.frontend
import scala.language.postfixOps
import org.scalajs.dom
import org.scalajs.dom.window
import rx._
import com.karasiq.bootstrap4.Bootstrap.default._
import scalaTags.all._
import com.karasiq.bootstrap.jquery.BootstrapJQueryContext
object BootstrapTestApp {
def main(args: Array[String]): Unit = {
BootstrapJQueryContext.useNpmImports()
jQuery(() ⇒ {
// Table tab will appear after 3 seconds
val tableVisible = Var(false)
val tabTitle = Var("Wait...")
// Show table tab in 3 seconds
window.setTimeout(() ⇒ {
tableVisible.update(true)
window.setTimeout(() ⇒ { tabTitle() = "Table" }, 1000)
}, 3000)
val tabs = Var(Seq[NavigationTab](
NavigationTab(tabTitle, "table", "table".faFwIcon, TestTable(), tableVisible.reactiveShow),
NavigationTab("Carousel", "carousel", "file-image-o".faFwIcon, TestCarousel("https://upload.wikimedia.org/wikipedia/commons/thumb/5/5e/Big_Wood%2C_N2.JPG/1280px-Big_Wood%2C_N2.JPG")),
NavigationTab("ToDo list", "todo", "fort-awesome".faFwIcon, TodoList()),
NavigationTab("Text rendering", "text", "file-text-o".faFwIcon, Bootstrap.jumbotron(
FormInput.textArea(a("Text rendering", href := "./serverside.html"), rows := 30, readonly, TestHtmlPage())
))
))
val navigationBar = NavigationBar()
.withBrand("Scala.js Bootstrap Test", href := "https://github.com/Karasiq/scalajs-bootstrap")
.withTabs(tabs)
.withContentContainer(content ⇒ GridSystem.container(id := "main-container", GridSystem.mkRow(content)))
.build()
// Render page
navigationBar.applyTo(dom.document.body)
// Reactive navbar test
tabs() = tabs.now :+ NavigationTab("Buttons", "buttons", "hand-o-right".faFwIcon, TestPanel("Serious business panel"))
navigationBar.selectTab(2)
})
}
}
Example 54
| Source File: TestPanel.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap4.test.frontend
import scala.concurrent.duration._
import scala.language.postfixOps
import org.scalajs.dom.window
import rx._
import com.karasiq.bootstrap4.Bootstrap.default._
import scalaTags.all.{span, _}
object TestPanel {
def apply(panelTitle: String): TestPanel = {
new TestPanel(panelTitle)
}
}
final class TestPanel(panelTitle: String) extends BootstrapHtmlComponent {
override def renderTag(md: ModifierT*): TagT = {
val titleVar = Var[Frag]("ERROR")
val successButton = Button(ButtonStyle.success)("Win 10000000$", onclick := Callback.onClick(_ ⇒ TestModal().show()), Tooltip(i("Press me"), TooltipPlacement.left)).render
val dangerButton = Button(ButtonStyle.danger)("Format C:\\", Popover(span(titleVar), "Popover test", TooltipPlacement.right)).render
titleVar() = i("Boom")
val toggleButton = ToggleButton(Bootstrap.button("Toggle me"))
val disabledButton = DisabledButton(Bootstrap.button("Heavy computation"))
disabledButton.state.foreach { pressed ⇒
if (pressed) {
window.setTimeout(() ⇒ {
window.alert(s"Answer: ${if (toggleButton.state.now) 321 else 123}")
disabledButton.state() = false
}, 1000)
}
}
// Render panel
val panelId = Bootstrap.newId
val collapseBtnTitle = Var("ERROR")
val panel = Card(panelId)
.withHeader("euro".faFwIcon, Card.collapse(panelId, panelTitle, Bootstrap.nbsp, Bootstrap.badge("42")), Card.buttons(
Card.button("plus".faFwIcon, onclick := Callback.onClick(_ ⇒ window.alert("Panel add"))),
Card.button("minus".faFwIcon, onclick := Callback.onClick(_ ⇒ window.alert("Panel remove")))
))
.withBody(
new TestProgressBar(Bootstrap.background.success, 200 millis),
Navigation.tabs(
NavigationTab("Simple buttons", Bootstrap.newId, "remove".faFwIcon, Card().withBody(
GridSystem.mkRow(
small("Hint: press the green button for reactive forms test", Bootstrap.textStyle.info)
),
GridSystem.mkRow(
ButtonGroup(ButtonGroupSize.default, successButton, dangerButton)
),
GridSystem.mkRow(Collapse(collapseBtnTitle)(
GridSystem.row(
GridSystem.col(6).asDiv(Dropdown("Dropdown", Dropdown.item("Test 1", onclick := Callback.onClick(_ ⇒ window.alert("Test 1"))), Dropdown.item("Test 2"))),
GridSystem.col(6).asDiv(Dropdown.dropup("Dropup", Dropdown.item("Test 3", onclick := Callback.onClick(_ ⇒ window.alert("Test 3"))), Dropdown.item("Test 4")))
)
))
)),
NavigationTab("Reactive buttons", Bootstrap.newId, "play-circle".faFwIcon, Card().withBody(ButtonGroup(ButtonGroupSize.large, toggleButton, disabledButton)))
)
)
.renderTag(Bootstrap.borderStyle.warning, Bootstrap.textStyle.info)
collapseBtnTitle() = "Dropdowns"
panel
}
}
Example 55
| Source File: BootstrapTestApp.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.test.backend
import java.util.concurrent.TimeUnit
import scala.concurrent.Await
import scala.concurrent.duration._
import scala.language.postfixOps
import akka.actor._
import akka.io.IO
import akka.pattern.ask
import akka.util.Timeout
import spray.can.Http
import spray.http._
import spray.routing.HttpService
import com.karasiq.bootstrap.test.frontend.TestHtmlPage
object BootstrapTestApp extends App {
final class AppHandler extends Actor with HttpService {
override def receive: Actor.Receive = runRoute {
get {
{
// Server-rendered page
path("serverside.html") {
complete(HttpResponse(entity = HttpEntity(ContentType(MediaTypes.`text/html`), TestHtmlPage())))
} ~
// Index page
(pathSingleSlash & respondWithMediaType(MediaTypes.`text/html`)) {
getFromResource("webapp/index.html")
} ~
// Other resources
getFromResourceDirectory("webapp")
}
}
}
override def actorRefFactory: ActorRefFactory = context
}
def startup(): Unit = {
implicit val timeout = Timeout(20 seconds)
implicit val actorSystem = ActorSystem("bootstrap-test")
Runtime.getRuntime.addShutdownHook(new Thread(new Runnable {
override def run(): Unit = {
Await.result(actorSystem.terminate(), FiniteDuration(5, TimeUnit.MINUTES))
}
}))
val service = actorSystem.actorOf(Props[AppHandler], "webService")
IO(Http) ? Http.Bind(service, interface = "localhost", port = 9000)
}
startup()
}
Example 56
| Source File: NavigationBars.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.navbar
import scala.language.{implicitConversions, postfixOps}
import rx.Rx
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.{ClassModifiers, RenderingContext}
import com.karasiq.bootstrap.grid.Grids
import com.karasiq.bootstrap.icons.Icons
import com.karasiq.bootstrap.utils.Utils
trait NavigationBars extends NavigationBarStyles { self: RenderingContext with Icons with Grids with Utils with BootstrapComponents with ClassModifiers ⇒
import scalaTags.all._
// -----------------------------------------------------------------------
// Tabs
// -----------------------------------------------------------------------
case class NavigationTab(name: Modifier, id: String, icon: IconModifier, content: Modifier, modifiers: Modifier*)
case class NavigationTabs(tabs: Rx[Seq[NavigationTab]])
object NavigationTabs {
implicit def toRxSeq(nt: NavigationTabs): Rx[Seq[NavigationTab]] = nt.tabs
implicit def fromRxSeq(seq: Rx[Seq[NavigationTab]]): NavigationTabs = new NavigationTabs(seq)
implicit def fromSeq(seq: Seq[NavigationTab]): NavigationTabs = fromRxSeq(Rx(seq))
}
// -----------------------------------------------------------------------
// Component definitions
// -----------------------------------------------------------------------
type Navigation <: AbstractNavigation with BootstrapComponent
val Navigation: NavigationFactory
val NavigationBar: NavigationBarFactory
type NavigationBar <: AbstractNavigationBar with BootstrapComponent
trait NavComponent {
val navId: String
val navTabs: NavigationTabs
def tabId(id: String): String = s"$navId-$id-tab"
}
trait AbstractNavigation extends NavComponent {
def navType: String
}
trait AbstractNavigationBar extends NavComponent
trait NavigationFactory {
def tabs(tabs: NavigationTab*): Navigation
def pills(tabs: NavigationTab*): Navigation
}
trait NavigationBarFactory {
def apply(tabs: Seq[NavigationTab] = Nil, barId: String = Bootstrap.newId, brand: Modifier = "Navigation",
styles: Seq[NavigationBarStyle] = Seq(NavigationBarStyle.default, NavigationBarStyle.fixedTop),
container: Modifier ⇒ Modifier = md ⇒ GridSystem.container(md),
contentContainer: Modifier ⇒ Modifier = md ⇒ GridSystem.container(GridSystem.mkRow(md))): NavigationBar
}
}
Example 57
| Source File: Tooltips.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.tooltip
import scala.language.{implicitConversions, postfixOps}
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.RenderingContext
trait Tooltips extends TooltipStyles { self: RenderingContext with BootstrapComponents ⇒
import scalaTags.all._
type Tooltip <: AbstractTooltip
val Tooltip: TooltipFactory
case class TooltipOptions(animation: Boolean = true, container: String = "", delay: String = "",
html: Boolean = false, placement: TooltipPlacement = TooltipPlacement.right, selector: String = "", template: String = "",
title: Frag = "", trigger: String = "", viewport: String = "") {
def toStrings: Seq[(String, String)] = {
def opt[T](name: String, value: T, default: T) = Option(name → value).filterNot(_._2 == default)
Seq(
opt("animation", animation, true),
opt("container", container, ""),
opt("delay", delay, ""),
opt("html", html, false),
opt("placement", placement.placement, TooltipPlacement.right),
opt("selector", selector, ""),
opt("template", template, ""),
opt("title", title, ""),
opt("trigger", trigger, ""),
opt("viewport", viewport, "")
).flatten.map(kv ⇒ kv._1 → kv._2.toString)
}
}
trait AbstractTooltip extends BootstrapComponent {
def options: TooltipOptions
}
trait TooltipFactory {
def apply(content: Frag, placement: TooltipPlacement = TooltipPlacement.auto): AbstractTooltip
}
}
Example 58
| Source File: TextTooltips.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.tooltip
import scala.language.postfixOps
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.RenderingContext
import com.karasiq.bootstrap.utils.Utils
trait TextTooltips { self: RenderingContext with BootstrapComponents with Tooltips with Utils ⇒
import scalaTags.all._
import BootstrapAttrs._
class TextTooltip(val options: TooltipOptions) extends AbstractTooltip {
override def render(md: ModifierT*): ModifierT = {
(`data-toggle` := "tooltip") +: Bootstrap.dataProps(options.toStrings:_*) +: md
}
}
object Tooltip extends TooltipFactory {
def apply(content: Frag, placement: TooltipPlacement): AbstractTooltip = {
new TextTooltip(TooltipOptions(html = true, title = content, placement = placement))
}
}
}
Example 59
| Source File: Grids.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.grid
import scala.language.{implicitConversions, postfixOps}
import com.karasiq.bootstrap.context.RenderingContext
trait Grids { self: RenderingContext ⇒
import scalaTags.all._
type GridSystem <: AbstractGridSystem
val GridSystem: GridSystem
trait AbstractGridSystem {
type ContainerT = Tag
type RowT = Tag
def container: ContainerT
def containerFluid: ContainerT
def row: RowT
def col: AbstractColumnFactory
def hidden: AbstractGridVisibilityFactory
def visible: AbstractGridVisibilityFactory
def mkRow(md: Modifier*): Tag = {
row(col(col.maxSize).asDiv(md))
}
def visibility(xs: Boolean = true, sm: Boolean = true, md: Boolean = true, lg: Boolean = true): Modifier = {
Array[Modifier](
if (xs) visible.xs else hidden.xs,
if (sm) visible.sm else hidden.sm,
if (md) visible.md else hidden.md,
if (lg) visible.lg else hidden.lg
)
}
}
trait AbstractColumn extends ModifierFactory {
def size: Int
def asDiv: Tag
def apply(md: ModifierT*): Tag = this.asDiv.apply(md:_*)
}
trait AbstractColumnFactory {
def minSize: Int
def maxSize: Int
def xs(size: Int): AbstractColumn
def sm(size: Int): AbstractColumn
def md(size: Int): AbstractColumn
def lg(size: Int): AbstractColumn
def responsive(xsSize: Int, smSize: Int, mdSize: Int, lgSize: Int): AbstractColumn = new AbstractColumn {
override val createModifier: ModifierT = Seq[ModifierT](xs(xsSize), sm(smSize), md(mdSize), lg(lgSize))
def size: Int = lgSize
def asDiv: Tag = div(createModifier)
}
def apply(size: Int): AbstractColumn = this.responsive(size, size, size, size)
}
trait AbstractGridVisibility extends ModifierFactory {
def screenSize: String
def hidden: Boolean
}
trait AbstractGridVisibilityFactory {
def xs: AbstractGridVisibility
def sm: AbstractGridVisibility
def md: AbstractGridVisibility
def lg: AbstractGridVisibility
def print: AbstractGridVisibility
}
}
Example 60
| Source File: BootstrapBundle.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap
import scala.language.postfixOps
import com.karasiq.bootstrap.alert.{Alerts, UniversalAlerts}
import com.karasiq.bootstrap.buttons.{Buttons, UniversalButtons}
import com.karasiq.bootstrap.carousel.{Carousels, UniversalCarousels}
import com.karasiq.bootstrap.collapse.{Collapses, UniversalCollapses}
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.{ClassModifiers, RenderingContext}
import com.karasiq.bootstrap.dropdown.{Dropdowns, UniversalDropdowns}
import com.karasiq.bootstrap.form.{Forms, UniversalForms}
import com.karasiq.bootstrap.grid.{Grids, UniversalGrids}
import com.karasiq.bootstrap.icons.{Icons, UniversalIcons}
import com.karasiq.bootstrap.modal.{Modals, UniversalModals}
import com.karasiq.bootstrap.navbar.{NavigationBars, UniversalNavigationBars}
import com.karasiq.bootstrap.pagination.{PageSelectors, UniversalPageSelectors}
import com.karasiq.bootstrap.panel.{Panels, UniversalPanels}
import com.karasiq.bootstrap.popover.Popovers
import com.karasiq.bootstrap.progressbar.{ProgressBars, UniversalProgressBars}
import com.karasiq.bootstrap.table._
import com.karasiq.bootstrap.tooltip.Tooltips
import com.karasiq.bootstrap.utils.{UniversalUtils, Utils}
// Abstract components
trait BootstrapBundle extends RenderingContext with BootstrapComponents with ClassModifiers with Alerts with Buttons
with Carousels with Collapses with Dropdowns with Forms with Grids with Icons with Modals with NavigationBars
with Panels with Popovers with ProgressBars with Tables with PageSelectors with PagedTables with SortableTables
with Tooltips with Utils
// Default components implementation
trait UniversalBootstrapBundle extends BootstrapBundle
with UniversalPageSelectors with UniversalTables with UniversalPagedTables with UniversalSortableTables
with UniversalProgressBars with UniversalPanels with UniversalNavigationBars with UniversalModals
with UniversalIcons with UniversalGrids with UniversalForms with UniversalDropdowns with UniversalCollapses
with UniversalCarousels with UniversalButtons with UniversalAlerts with UniversalUtils
Example 61
| Source File: TextBootstrapBundle.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap
import scala.language.postfixOps
import rx.Ctx
import com.karasiq.bootstrap.carousel.TextCarousels
import com.karasiq.bootstrap.context.TextRenderingContext
import com.karasiq.bootstrap.popover.TextPopovers
import com.karasiq.bootstrap.tooltip.TextTooltips
// Text components implementation
trait TextBootstrapBundle extends UniversalBootstrapBundle with TextRenderingContext
with TextCarousels with TextTooltips with TextPopovers
object TextBootstrapBundle {
def apply()(implicit ctx: Ctx.Owner): TextBootstrapBundle = {
new TextBootstrapBundle {
implicit val scalaRxContext = ctx
}
}
}
Example 62
| Source File: Popovers.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.popover
import scala.language.postfixOps
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.RenderingContext
import com.karasiq.bootstrap.tooltip.Tooltips
import com.karasiq.bootstrap.utils.Utils
trait Popovers { self: RenderingContext with BootstrapComponents with Tooltips with Utils ⇒
import scalaTags.all._
case class PopoverOptions(animation: Boolean = true, container: String = "", content: Frag = "", delay: String = "",
html: Boolean = false, placement: TooltipPlacement = TooltipPlacement.right, selector: String = "", template: String = "",
title: Frag = "", trigger: String = "", viewport: String = "") {
def toStrings: Seq[(String, String)] = {
def opt[T](name: String, value: T, default: T) = Option(name → value).filterNot(_._2 == default)
Seq(
opt("animation", animation, true),
opt("container", container, ""),
opt("content", content, ""),
opt("delay", delay, ""),
opt("html", html, false),
opt("placement", placement, TooltipPlacement.right),
opt("selector", selector, ""),
opt("template", template, ""),
opt("title", title, ""),
opt("trigger", trigger, ""),
opt("viewport", viewport, "")
).flatten.map(kv ⇒ kv._1 → kv._2.toString)
}
}
trait Popover extends BootstrapComponent {
def options: PopoverOptions
}
trait PopoverFactory {
def apply(title: Frag, content: Frag, placement: TooltipPlacement = TooltipPlacement.auto): Popover
}
val Popover: PopoverFactory
}
Example 63
| Source File: TextPopovers.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.popover
import scala.language.postfixOps
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.RenderingContext
import com.karasiq.bootstrap.tooltip.Tooltips
import com.karasiq.bootstrap.utils.Utils
trait TextPopovers { self: RenderingContext with BootstrapComponents with Tooltips with Popovers with Utils ⇒
import scalaTags.all._
import BootstrapAttrs._
class TextPopover(val options: PopoverOptions) extends Popover {
override def render(md: ModifierT*): ModifierT = {
(`data-toggle` := "popover") +: Bootstrap.dataProps(options.toStrings: _*) +: md
}
}
object Popover extends PopoverFactory {
def apply(title: Frag, content: Frag, placement: TooltipPlacement = TooltipPlacement.auto): Popover = {
val options = PopoverOptions(html = true, title = title, content = content, placement = placement)
new TextPopover(options)
}
}
}
Example 64
| Source File: TextCarousels.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.carousel
import scala.language.postfixOps
import rx.Rx
import com.karasiq.bootstrap.components.BootstrapComponents
import com.karasiq.bootstrap.context.RenderingContext
import com.karasiq.bootstrap.icons.Icons
import com.karasiq.bootstrap.utils.Utils
trait TextCarousels extends UniversalCarousels { self: RenderingContext with Carousels with Utils with Icons with BootstrapComponents ⇒
import scalaTags.all._
type Carousel = UniversalCarousel
object Carousel extends CarouselFactory {
def apply(data: Rx[Seq[Modifier]], id: String = Bootstrap.newId): UniversalCarousel = {
new UniversalCarousel(id, data)
}
def slide(image: String, content: Modifier*): Modifier = {
UniversalCarousel.slide(image, content)
}
}
}
Example 65
| Source File: Tables.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.table
import scala.language.postfixOps
import rx.{Rx, Var}
import com.karasiq.bootstrap.context.{ClassModifiers, RenderingContext}
import com.karasiq.bootstrap.utils.Utils
trait Tables extends TableRows with TableStyles { self: RenderingContext with ClassModifiers with Utils ⇒
import scalaTags.all._
type Table <: AbstractTable with BootstrapHtmlComponent
val Table: TableFactory
trait AbstractTable {
def heading: Rx[Seq[Modifier]]
def content: Rx[Seq[TableRow]]
}
trait TableFactory {
def apply(heading: Rx[Seq[Modifier]], content: Rx[Seq[TableRow]]): Table
def static(heading: Seq[Modifier], content: Seq[TableRow]): Table = {
apply(Var(heading), Var(content))
}
}
}
Example 66
| Source File: SortableTables.scala From scalajs-bootstrap with MIT License | 5 votes |
|
package com.karasiq.bootstrap.table
import scala.language.{higherKinds, postfixOps}
import rx._
import com.karasiq.bootstrap.context.RenderingContext
import com.karasiq.bootstrap.utils.Utils
trait SortableTables extends TableCols { self: RenderingContext with PagedTables with Utils ⇒
import scalaTags.all._
type SortableTable[T] <: AbstractSortableTable[T] with BootstrapHtmlComponent
val SortableTable: AbstractSortableTableFactory
trait AbstractSortableTableFactory {
def apply[T](items: Rx[Seq[T]], columns: Rx[Seq[TableCol[T, _]]],
rowModifiers: T ⇒ Modifier = (_: T) ⇒ Bootstrap.noModifier,
filterItem: (T, String) ⇒ Boolean = (i: T, f: String) ⇒ i.toString.contains(f)): SortableTable[T]
}
trait AbstractSortableTable[T] {
def items: Rx[Seq[T]]
def columns: Rx[GenTableCols[T]]
def sortByColumn: Var[GenTableCol[T]]
def reverseOrdering: Var[Boolean]
def filter: Var[String]
def filterItem(item: T, filter: String): Boolean
def rowModifiers(item: T): Modifier
def setOrdering(column: TableCol[T, _]): Unit = {
if (sortByColumn.now == column) reverseOrdering() = !reverseOrdering.now
else sortByColumn() = column
}
def pagedTable: PagedTable
}
}
Example 67
| Source File: ParallelOrSpec.scala From chymyst-core with Apache License 2.0 | 5 votes |
|
package io.chymyst.test
import io.chymyst.jc._
import scala.annotation.tailrec
import scala.concurrent.duration._
import scala.language.postfixOps
class ParallelOrSpec extends LogSpec {
behavior of "test"
def firstResult[T](b1: B[Unit, T], b2: B[Unit, T], tp: Pool): B[Unit, T] = {
val get = b[Unit, T]
val res = b[Unit, T]
val res1 = m[Unit]
val res2 = m[Unit]
val done = m[T]
site(tp)(
go { case res1(_) => val x = b1(); done(x) }, // IntelliJ 2016.3 CE insists on `b1(())` here, but scalac is fine with `b1()`.
go { case res2(_) => val x = b2(); done(x) }
)
site(tp)(go { case get(_, r) + done(x) => r(x) })
site(tp)(go { case res(_, r) => res1() + res2(); val x = get(); r(x) })
res
}
@tailrec
final def neverReturn[T]: T = {
Thread.sleep(1000000)
neverReturn[T]
}
it should "implement the First Result operation correctly" in {
val never = b[Unit, String]
val fast = b[Unit, String]
val slow = b[Unit, String]
val tp = FixedPool(30)
site(tp)(
go { case never(_, r) => r(neverReturn[String]) },
go { case fast(_, r) => Thread.sleep(10); r("fast") },
go { case slow(_, r) => Thread.sleep(200); r("slow") }
)
firstResult(fast, fast, tp)() shouldEqual "fast"
firstResult(fast, slow, tp)() shouldEqual "fast"
firstResult(slow, fast, tp)() shouldEqual "fast"
firstResult(slow, slow, tp)() shouldEqual "slow"
firstResult(fast, never, tp)() shouldEqual "fast"
firstResult(never, slow, tp)() shouldEqual "slow"
tp.shutdownNow()
}
}
Example 68
| Source File: KafkaStreamSuite.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.kafka
import scala.collection.mutable
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.util.Random
import kafka.serializer.StringDecoder
import org.scalatest.BeforeAndAfterAll
import org.scalatest.concurrent.Eventually
import org.apache.spark.{SparkConf, SparkFunSuite}
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Milliseconds, StreamingContext}
class KafkaStreamSuite extends SparkFunSuite with Eventually with BeforeAndAfterAll {
private var ssc: StreamingContext = _
private var kafkaTestUtils: KafkaTestUtils = _
override def beforeAll(): Unit = {
kafkaTestUtils = new KafkaTestUtils
kafkaTestUtils.setup()
}
override def afterAll(): Unit = {
if (ssc != null) {
ssc.stop()
ssc = null
}
if (kafkaTestUtils != null) {
kafkaTestUtils.teardown()
kafkaTestUtils = null
}
}
test("Kafka input stream") {
val sparkConf = new SparkConf().setMaster("local[4]").setAppName(this.getClass.getSimpleName)
ssc = new StreamingContext(sparkConf, Milliseconds(500))
val topic = "topic1"
val sent = Map("a" -> 5, "b" -> 3, "c" -> 10)
kafkaTestUtils.createTopic(topic)
kafkaTestUtils.sendMessages(topic, sent)
val kafkaParams = Map("zookeeper.connect" -> kafkaTestUtils.zkAddress,
"group.id" -> s"test-consumer-${Random.nextInt(10000)}",
"auto.offset.reset" -> "smallest")
val stream = KafkaUtils.createStream[String, String, StringDecoder, StringDecoder](
ssc, kafkaParams, Map(topic -> 1), StorageLevel.MEMORY_ONLY)
val result = new mutable.HashMap[String, Long]()
stream.map(_._2).countByValue().foreachRDD { r =>
r.collect().foreach { kv =>
result.synchronized {
val count = result.getOrElseUpdate(kv._1, 0) + kv._2
result.put(kv._1, count)
}
}
}
ssc.start()
eventually(timeout(10000 milliseconds), interval(100 milliseconds)) {
assert(result.synchronized { sent === result })
}
}
}
Example 69
| Source File: FlumeStreamSuite.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.flume
import java.util.concurrent.ConcurrentLinkedQueue
import scala.collection.JavaConverters._
import scala.concurrent.duration._
import scala.language.postfixOps
import org.jboss.netty.channel.ChannelPipeline
import org.jboss.netty.channel.socket.SocketChannel
import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory
import org.jboss.netty.handler.codec.compression._
import org.scalatest.{BeforeAndAfter, Matchers}
import org.scalatest.concurrent.Eventually._
import org.apache.spark.{SparkConf, SparkFunSuite}
import org.apache.spark.internal.Logging
import org.apache.spark.network.util.JavaUtils
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Milliseconds, StreamingContext, TestOutputStream}
class FlumeStreamSuite extends SparkFunSuite with BeforeAndAfter with Matchers with Logging {
val conf = new SparkConf().setMaster("local[4]").setAppName("FlumeStreamSuite")
var ssc: StreamingContext = null
test("flume input stream") {
testFlumeStream(testCompression = false)
}
test("flume input compressed stream") {
testFlumeStream(testCompression = true)
}
private class CompressionChannelFactory(compressionLevel: Int)
extends NioClientSocketChannelFactory {
override def newChannel(pipeline: ChannelPipeline): SocketChannel = {
val encoder = new ZlibEncoder(compressionLevel)
pipeline.addFirst("deflater", encoder)
pipeline.addFirst("inflater", new ZlibDecoder())
super.newChannel(pipeline)
}
}
}
Example 70
| Source File: StatusTrackerSuite.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark
import scala.concurrent.duration._
import scala.language.implicitConversions
import scala.language.postfixOps
import org.scalatest.Matchers
import org.scalatest.concurrent.Eventually._
import org.apache.spark.JobExecutionStatus._
class StatusTrackerSuite extends SparkFunSuite with Matchers with LocalSparkContext {
test("basic status API usage") {
sc = new SparkContext("local", "test", new SparkConf(false))
val jobFuture = sc.parallelize(1 to 10000, 2).map(identity).groupBy(identity).collectAsync()
val jobId: Int = eventually(timeout(10 seconds)) {
val jobIds = jobFuture.jobIds
jobIds.size should be(1)
jobIds.head
}
val jobInfo = eventually(timeout(10 seconds)) {
sc.statusTracker.getJobInfo(jobId).get
}
jobInfo.status() should not be FAILED
val stageIds = jobInfo.stageIds()
stageIds.size should be(2)
val firstStageInfo = eventually(timeout(10 seconds)) {
sc.statusTracker.getStageInfo(stageIds(0)).get
}
firstStageInfo.stageId() should be(stageIds(0))
firstStageInfo.currentAttemptId() should be(0)
firstStageInfo.numTasks() should be(2)
eventually(timeout(10 seconds)) {
val updatedFirstStageInfo = sc.statusTracker.getStageInfo(stageIds(0)).get
updatedFirstStageInfo.numCompletedTasks() should be(2)
updatedFirstStageInfo.numActiveTasks() should be(0)
updatedFirstStageInfo.numFailedTasks() should be(0)
}
}
test("getJobIdsForGroup()") {
sc = new SparkContext("local", "test", new SparkConf(false))
// Passing `null` should return jobs that were not run in a job group:
val defaultJobGroupFuture = sc.parallelize(1 to 1000).countAsync()
val defaultJobGroupJobId = eventually(timeout(10 seconds)) {
defaultJobGroupFuture.jobIds.head
}
eventually(timeout(10 seconds)) {
sc.statusTracker.getJobIdsForGroup(null).toSet should be (Set(defaultJobGroupJobId))
}
// Test jobs submitted in job groups:
sc.setJobGroup("my-job-group", "description")
sc.statusTracker.getJobIdsForGroup("my-job-group") should be (Seq.empty)
val firstJobFuture = sc.parallelize(1 to 1000).countAsync()
val firstJobId = eventually(timeout(10 seconds)) {
firstJobFuture.jobIds.head
}
eventually(timeout(10 seconds)) {
sc.statusTracker.getJobIdsForGroup("my-job-group") should be (Seq(firstJobId))
}
val secondJobFuture = sc.parallelize(1 to 1000).countAsync()
val secondJobId = eventually(timeout(10 seconds)) {
secondJobFuture.jobIds.head
}
eventually(timeout(10 seconds)) {
sc.statusTracker.getJobIdsForGroup("my-job-group").toSet should be (
Set(firstJobId, secondJobId))
}
}
test("getJobIdsForGroup() with takeAsync()") {
sc = new SparkContext("local", "test", new SparkConf(false))
sc.setJobGroup("my-job-group2", "description")
sc.statusTracker.getJobIdsForGroup("my-job-group2") shouldBe empty
val firstJobFuture = sc.parallelize(1 to 1000, 1).takeAsync(1)
val firstJobId = eventually(timeout(10 seconds)) {
firstJobFuture.jobIds.head
}
eventually(timeout(10 seconds)) {
sc.statusTracker.getJobIdsForGroup("my-job-group2") should be (Seq(firstJobId))
}
}
test("getJobIdsForGroup() with takeAsync() across multiple partitions") {
sc = new SparkContext("local", "test", new SparkConf(false))
sc.setJobGroup("my-job-group2", "description")
sc.statusTracker.getJobIdsForGroup("my-job-group2") shouldBe empty
val firstJobFuture = sc.parallelize(1 to 1000, 2).takeAsync(999)
val firstJobId = eventually(timeout(10 seconds)) {
firstJobFuture.jobIds.head
}
eventually(timeout(10 seconds)) {
sc.statusTracker.getJobIdsForGroup("my-job-group2") should have size 2
}
}
}
Example 71
| Source File: LauncherBackendSuite.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.launcher
import java.util.concurrent.TimeUnit
import scala.concurrent.duration._
import scala.language.postfixOps
import org.scalatest.Matchers
import org.scalatest.concurrent.Eventually._
import org.apache.spark._
class LauncherBackendSuite extends SparkFunSuite with Matchers {
private val tests = Seq(
"local" -> "local",
"standalone/client" -> "local-cluster[1,1,1024]")
tests.foreach { case (name, master) =>
test(s"$name: launcher handle") {
testWithMaster(master)
}
}
private def testWithMaster(master: String): Unit = {
val env = new java.util.HashMap[String, String]()
env.put("SPARK_PRINT_LAUNCH_COMMAND", "1")
val handle = new SparkLauncher(env)
.setSparkHome(sys.props("spark.test.home"))
.setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, System.getProperty("java.class.path"))
.setConf("spark.ui.enabled", "false")
.setConf(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, s"-Dtest.appender=console")
.setMaster(master)
.setAppResource(SparkLauncher.NO_RESOURCE)
.setMainClass(TestApp.getClass.getName().stripSuffix("$"))
.startApplication()
try {
eventually(timeout(30 seconds), interval(100 millis)) {
handle.getAppId() should not be (null)
}
handle.stop()
eventually(timeout(30 seconds), interval(100 millis)) {
handle.getState() should be (SparkAppHandle.State.KILLED)
}
} finally {
handle.kill()
}
}
}
object TestApp {
def main(args: Array[String]): Unit = {
new SparkContext(new SparkConf()).parallelize(Seq(1)).foreach { i =>
Thread.sleep(TimeUnit.SECONDS.toMillis(20))
}
}
}
Example 72
| Source File: SignInController.scala From play-silhouette-4.0-slick-postgres-seed with Apache License 2.0 | 5 votes |
|
package controllers.auth
import javax.inject.Inject
import com.mohiva.play.silhouette.api.Authenticator.Implicits._
import com.mohiva.play.silhouette.api._
import com.mohiva.play.silhouette.api.exceptions.ProviderException
import com.mohiva.play.silhouette.api.repositories.AuthInfoRepository
import com.mohiva.play.silhouette.api.util.{ Clock, Credentials }
import com.mohiva.play.silhouette.impl.exceptions.IdentityNotFoundException
import com.mohiva.play.silhouette.impl.providers._
import controllers.{ WebJarAssets, auth, pages }
import forms.auth.SignInForm
import models.services.UserService
import net.ceedubs.ficus.Ficus._
import play.api.Configuration
import play.api.i18n.{ I18nSupport, Messages, MessagesApi }
import play.api.libs.concurrent.Execution.Implicits._
import play.api.mvc.{ Action, AnyContent, Controller }
import utils.auth.DefaultEnv
import scala.concurrent.Future
import scala.concurrent.duration._
import scala.language.postfixOps
def submit: Action[AnyContent] = silhouette.UnsecuredAction.async { implicit request =>
SignInForm.form.bindFromRequest.fold(
form => Future.successful(BadRequest(views.html.auth.signIn(form, socialProviderRegistry))),
data => {
val credentials = Credentials(data.email, data.password)
credentialsProvider.authenticate(credentials).flatMap { loginInfo =>
val result = Redirect(pages.routes.ApplicationController.index())
userService.retrieve(loginInfo).flatMap {
case Some(user) if !user.activated =>
Future.successful(Ok(views.html.auth.activateAccount(data.email)))
case Some(user) =>
val c = configuration.underlying
silhouette.env.authenticatorService.create(loginInfo).map {
case authenticator if data.rememberMe =>
authenticator.copy(
expirationDateTime = clock.now + c.as[FiniteDuration]("silhouette.authenticator.rememberMe.authenticatorExpiry"),
idleTimeout = c.getAs[FiniteDuration]("silhouette.authenticator.rememberMe.authenticatorIdleTimeout"),
cookieMaxAge = c.getAs[FiniteDuration]("silhouette.authenticator.rememberMe.cookieMaxAge")
)
case authenticator => authenticator
}.flatMap { authenticator =>
silhouette.env.eventBus.publish(LoginEvent(user, request))
silhouette.env.authenticatorService.init(authenticator).flatMap { v =>
silhouette.env.authenticatorService.embed(v, result)
}
}
case None => Future.failed(new IdentityNotFoundException("Couldn't find user"))
}
}.recover {
case e: ProviderException =>
Redirect(auth.routes.SignInController.view()).flashing("error" -> Messages("invalid.credentials"))
}
}
)
}
}
Example 73
| Source File: ActivateAccountController.scala From play-silhouette-4.0-slick-postgres-seed with Apache License 2.0 | 5 votes |
|
package controllers.auth
import java.net.URLDecoder
import java.util.UUID
import javax.inject.Inject
import com.mohiva.play.silhouette.api._
import com.mohiva.play.silhouette.impl.providers.CredentialsProvider
import controllers.{ WebJarAssets, auth }
import models.services.{ AuthTokenService, UserService }
import play.api.i18n.{ I18nSupport, Messages, MessagesApi }
import play.api.libs.concurrent.Execution.Implicits._
import play.api.libs.mailer.{ Email, MailerClient }
import play.api.mvc.{ Action, AnyContent, Controller }
import utils.auth.DefaultEnv
import scala.concurrent.Future
import scala.language.postfixOps
def activate(token: UUID): Action[AnyContent] = silhouette.UnsecuredAction.async { implicit request =>
authTokenService.validate(token).flatMap {
case Some(authToken) => userService.retrieve(authToken.userID).flatMap {
case Some(user) if user.loginInfo.providerID == CredentialsProvider.ID =>
userService.save(user.copy(activated = true)).map { _ =>
Redirect(auth.routes.SignInController.view()).flashing("success" -> Messages("account.activated"))
}
case _ => Future.successful(Redirect(auth.routes.SignInController.view()).flashing("error" -> Messages("invalid.activation.link")))
}
case None => Future.successful(Redirect(auth.routes.SignInController.view()).flashing("error" -> Messages("invalid.activation.link")))
}
}
}
Example 74
| Source File: AudioLength.scala From jvm-toxcore-api with GNU General Public License v3.0 | 5 votes |
|
package im.tox.tox4j.av.data
import im.tox.core.error.CoreError
import im.tox.core.typesafe.DiscreteValueCompanion
import scala.concurrent.duration._
import scala.language.postfixOps
final case class AudioLength private (value: Duration) extends AnyVal
// scalastyle:off magic.number
case object AudioLength extends DiscreteValueCompanion[Duration, AudioLength](
_.value,
2500 microseconds,
5000 microseconds,
10000 microseconds,
20000 microseconds,
40000 microseconds,
60000 microseconds
) {
protected override def unsafeFromValue(value: Duration): AudioLength = new AudioLength(value)
val Length2_5: AudioLength = new AudioLength(values(0))
val Length5: AudioLength = new AudioLength(values(1))
val Length10: AudioLength = new AudioLength(values(2))
val Length20: AudioLength = new AudioLength(values(3))
val Length40: AudioLength = new AudioLength(values(4))
val Length60: AudioLength = new AudioLength(values(5))
final override def equals(a: AudioLength, b: AudioLength): Boolean = {
a.value == b.value
}
}
Example 75
| Source File: WalLogStat.scala From incubator-s2graph with Apache License 2.0 | 5 votes |
|
package org.apache.s2graph.loader.subscriber
import kafka.producer.KeyedMessage
import kafka.serializer.StringDecoder
import org.apache.s2graph.spark.spark.{WithKafka, SparkApp}
import org.apache.spark.streaming.Durations._
import org.apache.spark.streaming.kafka.HasOffsetRanges
import scala.collection.mutable.{HashMap => MutableHashMap}
import scala.language.postfixOps
object WalLogStat extends SparkApp with WithKafka {
override def run() = {
validateArgument("kafkaZkQuorum", "brokerList", "topics", "intervalInSec", "dbUrl", "statTopic")
val kafkaZkQuorum = args(0)
val brokerList = args(1)
val topics = args(2)
val intervalInSec = seconds(args(3).toLong)
val dbUrl = args(4)
val statTopic = args(5)
val conf = sparkConf(s"$topics: ${getClass.getSimpleName}")
val ssc = streamingContext(conf, intervalInSec)
val sc = ssc.sparkContext
val groupId = topics.replaceAll(",", "_") + "_stat"
val kafkaParams = Map(
"zookeeper.connect" -> kafkaZkQuorum,
"group.id" -> groupId,
"metadata.broker.list" -> brokerList,
"zookeeper.connection.timeout.ms" -> "10000",
"auto.offset.reset" -> "largest")
val stream = getStreamHelper(kafkaParams).createStream[String, String, StringDecoder, StringDecoder](ssc, topics.split(",").toSet)
val statProducer = getProducer[String, String](brokerList)
stream.foreachRDD { (rdd, time) =>
val offsets = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
val ts = time.milliseconds
val elements = rdd.mapPartitions { partition =>
// set executor setting.
val phase = System.getProperty("phase")
GraphSubscriberHelper.apply(phase, dbUrl, "none", brokerList)
partition.map { case (key, msg) =>
GraphSubscriberHelper.g.elementBuilder.toGraphElement(msg) match {
case Some(elem) =>
val serviceName = elem.serviceName
msg.split("\t", 7) match {
case Array(_, operation, log_type, _, _, label, _*) =>
Seq(serviceName, label, operation, log_type).mkString("\t")
case _ =>
Seq("no_service_name", "no_label", "no_operation", "parsing_error").mkString("\t")
}
case None =>
Seq("no_service_name", "no_label", "no_operation", "no_element_error").mkString("\t")
}
}
}
val countByKey = elements.map(_ -> 1L).reduceByKey(_ + _).collect()
val totalCount = countByKey.map(_._2).sum
val keyedMessage = countByKey.map { case (key, value) =>
new KeyedMessage[String, String](statTopic, s"$ts\t$key\t$value\t$totalCount")
}
statProducer.send(keyedMessage: _*)
elements.mapPartitionsWithIndex { (i, part) =>
// commit offset range
val osr = offsets(i)
getStreamHelper(kafkaParams).commitConsumerOffset(osr)
Iterator.empty
}.foreach {
(_: Nothing) => ()
}
}
ssc.start()
ssc.awaitTermination()
}
}
Example 76
| Source File: ExactCounterStreaming.scala From incubator-s2graph with Apache License 2.0 | 5 votes |
|
package org.apache.s2graph.counter.loader.stream
import kafka.serializer.StringDecoder
import org.apache.s2graph.counter.config.S2CounterConfig
import org.apache.s2graph.counter.loader.config.StreamingConfig
import org.apache.s2graph.counter.loader.core.CounterFunctions
import org.apache.s2graph.spark.config.S2ConfigFactory
import org.apache.s2graph.spark.spark.{WithKafka, SparkApp, HashMapParam}
import org.apache.spark.streaming.Durations._
import org.apache.spark.streaming.kafka.KafkaRDDFunctions.rddToKafkaRDDFunctions
import org.apache.spark.streaming.kafka.{HasOffsetRanges, StreamHelper}
import scala.collection.mutable.{HashMap => MutableHashMap}
import scala.language.postfixOps
object ExactCounterStreaming extends SparkApp with WithKafka {
lazy val config = S2ConfigFactory.config
lazy val s2Config = new S2CounterConfig(config)
lazy val className = getClass.getName.stripSuffix("$")
lazy val producer = getProducer[String, String](StreamingConfig.KAFKA_BROKERS)
val inputTopics = Set(StreamingConfig.KAFKA_TOPIC_COUNTER)
val strInputTopics = inputTopics.mkString(",")
val groupId = buildKafkaGroupId(strInputTopics, "counter_v2")
val kafkaParam = Map(
// "auto.offset.reset" -> "smallest",
"group.id" -> groupId,
"metadata.broker.list" -> StreamingConfig.KAFKA_BROKERS,
"zookeeper.connect" -> StreamingConfig.KAFKA_ZOOKEEPER,
"zookeeper.connection.timeout.ms" -> "10000"
)
val streamHelper = StreamHelper(kafkaParam)
override def run() = {
validateArgument("interval", "clear")
val (intervalInSec, clear) = (seconds(args(0).toLong), args(1).toBoolean)
if (clear) {
streamHelper.kafkaHelper.consumerGroupCleanup()
}
val conf = sparkConf(s"$strInputTopics: $className")
val ssc = streamingContext(conf, intervalInSec)
val sc = ssc.sparkContext
implicit val acc: HashMapAccumulable = sc.accumulable(MutableHashMap.empty[String, Long], "Throughput")(HashMapParam[String, Long](_ + _))
// make stream
val stream = streamHelper.createStream[String, String, StringDecoder, StringDecoder](ssc, inputTopics)
stream.foreachRDD { (rdd, ts) =>
val offsets = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
val exactRDD = CounterFunctions.makeExactRdd(rdd, offsets.length)
// for at-least once semantic
exactRDD.foreachPartitionWithIndex { (i, part) =>
// update exact counter
val trxLogs = CounterFunctions.updateExactCounter(part.toSeq, acc)
CounterFunctions.produceTrxLog(trxLogs)
// commit offset range
streamHelper.commitConsumerOffset(offsets(i))
}
}
ssc.start()
ssc.awaitTermination()
}
}
Example 77
| Source File: CollectionCache.scala From incubator-s2graph with Apache License 2.0 | 5 votes |
|
package org.apache.s2graph.counter.util
import java.net.InetAddress
import java.util.concurrent.TimeUnit
import com.google.common.cache.{Cache, CacheBuilder}
import org.slf4j.LoggerFactory
import scala.concurrent.{ExecutionContext, Future}
import scala.language.{postfixOps, reflectiveCalls}
case class CollectionCacheConfig(maxSize: Int, ttl: Int, negativeCache: Boolean = false, negativeTTL: Int = 600)
class CollectionCache[C <: { def nonEmpty: Boolean; def isEmpty: Boolean } ](config: CollectionCacheConfig) {
private val cache: Cache[String, C] = CacheBuilder.newBuilder()
.expireAfterWrite(config.ttl, TimeUnit.SECONDS)
.maximumSize(config.maxSize)
.build[String, C]()
// private lazy val cache = new SynchronizedLruMap[String, (C, Int)](config.maxSize)
private lazy val className = this.getClass.getSimpleName
private lazy val log = LoggerFactory.getLogger(this.getClass)
val localHostname = InetAddress.getLocalHost.getHostName
def size = cache.size
val maxSize = config.maxSize
// cache statistics
def getStatsString: String = {
s"$localHostname ${cache.stats().toString}"
}
def withCache(key: String)(op: => C): C = {
Option(cache.getIfPresent(key)) match {
case Some(r) => r
case None =>
val r = op
if (r.nonEmpty || config.negativeCache) {
cache.put(key, r)
}
r
}
}
def withCacheAsync(key: String)(op: => Future[C])(implicit ec: ExecutionContext): Future[C] = {
Option(cache.getIfPresent(key)) match {
case Some(r) => Future.successful(r)
case None =>
op.map { r =>
if (r.nonEmpty || config.negativeCache) {
cache.put(key, r)
}
r
}
}
}
def purgeKey(key: String) = {
cache.invalidate(key)
}
def contains(key: String): Boolean = {
Option(cache.getIfPresent(key)).nonEmpty
}
}
Example 78
| Source File: Server.scala From incubator-s2graph with Apache License 2.0 | 5 votes |
|
package org.apache.s2graph.http
import java.time.Instant
import scala.language.postfixOps
import scala.concurrent.{Await, ExecutionContext, Future}
import scala.concurrent.duration.Duration
import scala.util.{Failure, Success}
import akka.actor.ActorSystem
import akka.http.scaladsl.Http
import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpResponse, StatusCodes}
import akka.http.scaladsl.server.Route
import akka.http.scaladsl.server.Directives._
import akka.stream.ActorMaterializer
import com.typesafe.config.ConfigFactory
import org.apache.s2graph.core.S2Graph
import org.slf4j.LoggerFactory
object Server extends App
with S2GraphTraversalRoute
with S2GraphAdminRoute
with S2GraphMutateRoute
with S2GraphQLRoute {
implicit val system: ActorSystem = ActorSystem("S2GraphHttpServer")
implicit val materializer: ActorMaterializer = ActorMaterializer()
implicit val executionContext: ExecutionContext = system.dispatcher
val config = ConfigFactory.load()
override val s2graph = new S2Graph(config)
override val logger = LoggerFactory.getLogger(this.getClass)
val port = sys.props.get("http.port").fold(8000)(_.toInt)
val interface = sys.props.get("http.interface").fold("0.0.0.0")(identity)
val startAt = System.currentTimeMillis()
def uptime = System.currentTimeMillis() - startAt
def serverHealth = s"""{ "port": ${port}, "interface": "${interface}", "started_at": ${Instant.ofEpochMilli(startAt)}, "uptime": "${uptime} millis" """
def health = HttpResponse(status = StatusCodes.OK, entity = HttpEntity(ContentTypes.`application/json`, serverHealth))
// Allows you to determine routes to expose according to external settings.
lazy val routes: Route = concat(
pathPrefix("graphs")(traversalRoute),
pathPrefix("mutate")(mutateRoute),
pathPrefix("admin")(adminRoute),
pathPrefix("graphql")(graphqlRoute),
get(complete(health))
)
val binding: Future[Http.ServerBinding] = Http().bindAndHandle(routes, interface, port)
binding.onComplete {
case Success(bound) => logger.info(s"Server online at http://${bound.localAddress.getHostString}:${bound.localAddress.getPort}/")
case Failure(e) => logger.error(s"Server could not start!", e)
}
scala.sys.addShutdownHook { () =>
s2graph.shutdown()
system.terminate()
logger.info("System terminated")
}
Await.result(system.whenTerminated, Duration.Inf)
}
Example 79
| Source File: KafkaTopicAdmin.scala From kafka-configurator with BSD 3-Clause "New" or "Revised" License | 5 votes |
|
package com.sky.kafka.configurator
import java.util.concurrent.ExecutionException
import cats.data.Reader
import com.sky.kafka.configurator.error.TopicNotFound
import org.apache.kafka.clients.admin.AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG
import org.apache.kafka.clients.admin._
import org.apache.kafka.common.config.ConfigResource
import org.apache.kafka.common.errors.UnknownTopicOrPartitionException
import org.zalando.grafter.{ Stop, StopResult }
import scala.collection.JavaConverters._
import scala.language.postfixOps
import scala.util.{ Failure, Success, Try }
object KafkaTopicAdmin {
def apply(adminClient: AdminClient): KafkaTopicAdmin = new KafkaTopicAdmin(adminClient)
def reader: Reader[AppConfig, KafkaTopicAdmin] = Reader { config =>
import com.sky.kafka.utils.MapToJavaPropertiesConversion.mapToProperties
KafkaTopicAdmin(AdminClient.create(Map(BOOTSTRAP_SERVERS_CONFIG -> config.bootstrapServers) ++ config.props))
}
}
class KafkaTopicAdmin(ac: AdminClient) extends TopicReader with TopicWriter with Stop {
override def fetch(topicName: String) = {
def topicDescription = Try {
val allDescriptions = ac.describeTopics(Seq(topicName).asJava).all.get
allDescriptions.get(topicName)
} match {
case Success(result) => Success(result)
case Failure(e: ExecutionException) if e.getCause.isInstanceOf[UnknownTopicOrPartitionException] => Failure(TopicNotFound(topicName))
case other => other
}
def topicConfig = Try {
val allConfigs = ac.describeConfigs(Seq(configResourceForTopic(topicName)).asJava).all.get
allConfigs.get(configResourceForTopic(topicName))
}
for {
desc <- topicDescription
partitions = desc.partitions().size()
replicationFactor = desc.partitions().asScala.head.replicas().size()
config <- topicConfig
} yield Topic(desc.name(), partitions, replicationFactor, config)
}
override def create(topic: Topic) = Try {
val newTopic = new NewTopic(topic.name, topic.partitions, topic.replicationFactor.toShort).configs(topic.config.asJava)
ac.createTopics(Seq(newTopic).asJava).all().get
}
override def updateConfig(topicName: String, config: Map[String, Object]) = Try {
val c = config.map {
case (key, value) => new ConfigEntry(key, value.toString)
}.toList.asJava
ac.alterConfigs(Map(configResourceForTopic(topicName) -> new Config(c)).asJava).all().get
}
override def updatePartitions(topicName: String, numPartitions: Int) = Try {
ac.createPartitions(Map(topicName -> NewPartitions.increaseTo(numPartitions)).asJava).all().get()
}
override def stop = StopResult.eval("KafkaAdminClient")(ac.close())
private def configResourceForTopic(topicName: String) = new ConfigResource(ConfigResource.Type.TOPIC, topicName)
private implicit def kafkaConfigToMap(config: Config): Map[String, String] = config.entries().asScala.map { entry =>
entry.name() -> entry.value()
} toMap
}
Example 80
| Source File: Signing.scala From parquet4s with MIT License | 5 votes |
|
import java.io.File
import sbt.Keys._
import sbt.io.IO
import sbt.librarymanagement.{Artifact, Configuration}
import sbt.{Def, TaskKey, _}
import scala.language.postfixOps
object Signing {
private val SignatureExtension = "asc"
private def createSignatureFile(artifactFile: File): File = {
val signatureFile = file(artifactFile.getAbsolutePath + "." + SignatureExtension)
if (signatureFile.exists()) IO.delete(signatureFile)
val command = "gpg"
val keyRingArgs = Seq("--no-default-keyring", "--keyring", (file(System.getProperty("user.home")) / ".gnupg" / "pubring.kbx").getAbsolutePath)
val actionArgs = Seq("--detach-sign", "--armor")
val passwordArgs = Seq("--pinentry-mode", "loopback", "--passphrase", sys.env("GPG_PASSWORD"))
val outputArgs = Seq("--output", signatureFile.getAbsolutePath)
val targetArgs = Seq(artifactFile.getAbsolutePath)
val args: Seq[String] = keyRingArgs ++ actionArgs ++ passwordArgs ++ outputArgs ++ targetArgs
sys.process.Process(command, args) ! match {
case 0 => ()
case n => sys.error(s"""Failure running '${args.mkString(command + " ", " ", "")}'. Exit code: $n""")
}
signatureFile
}
private def addSignatureArtifact(configuration: Configuration, packageTask: TaskKey[File]): Def.SettingsDefinition = {
val signTaskDef = Def.task {
val (artifact, artifactFile) = packagedArtifact.in(configuration, packageTask).value
streams.value.log.info("Signing: " + artifact)
createSignatureFile(artifactFile)
}
val artifactDef = Def.setting {
val sourceArtifact = artifact.in(configuration, packageTask).value
Artifact(
name = sourceArtifact.name,
`type` = SignatureExtension,
extension = sourceArtifact.extension + "." + SignatureExtension,
classifier = sourceArtifact.classifier,
configurations = Vector.empty,
url = None
)
}
addArtifact(artifactDef, signTaskDef)
}
def signingSettings: Seq[Def.Setting[_]] =
addSignatureArtifact(Compile, packageBin).settings ++
addSignatureArtifact(Compile, packageDoc).settings ++
addSignatureArtifact(Compile, packageSrc).settings ++
addSignatureArtifact(Compile, makePom).settings
}
Example 81
| Source File: Mnemonics.scala From LearningScala with Apache License 2.0 | 5 votes |
|
package _980_problem_solving
import scala.io.Source
import scala.language.postfixOps
def encode(number: String): Set[List[String]] =
if (number.isEmpty) Set(List())
else {
for {
split <- 1 to number.length
word <- wordsForNum(number take split)
rest <- encode(number drop split)
} yield word :: rest
}.toSet
def translate(number: String): Set[String] =
encode(number) map (_ mkString " ")
// testing
println(translate("7225247386"))
println(translate("72252"))
println(translate("783364"))
}
Example 82
| Source File: NQueens.scala From LearningScala with Apache License 2.0 | 5 votes |
|
package _980_problem_solving
import scala.language.postfixOps
object NQueens extends App {
// Tests
println(queens(4))
println((queens(4) map show) mkString "\n")
println
println("==========================")
val queens8 = queens(8)
println(queens8)
println(queens8 size)
println((queens8 take 3 map show) mkString "\n")
println
println("==========================")
// n: # of rows.
def queens(n: Int): Set[List[Int]] = {
def placeQueens(k: Int): Set[List[Int]] =
if (k == 0) Set(List())
else
for {
queens <- placeQueens(k - 1)
col <- 0 until n
if isSafe(col, queens)
} yield col :: queens
def isSafe(col: Int, queens: List[Int]): Boolean = {
val row = queens.length
val queensWithRow = (row - 1 to 0 by -1) zip queens
queensWithRow forall {
case (r, c) => col != c && math.abs(col - c) != row - r
}
}
placeQueens(n)
}
def show(queens: List[Int]): String = {
val lines =
for (col <- queens.reverse)
yield Vector.fill(queens.length)(" * ").updated(col, " X ").mkString
"\n" + (lines mkString "\n")
}
}
Example 83
| Source File: _02_1_HigherOrderFunctions.scala From LearningScala with Apache License 2.0 | 5 votes |
|
package _010_functions
import scala.language.postfixOps
object _02_1_HigherOrderFunctions extends App {
// Functions with functions (Higher order functions)
// function2...
// val f: (Int, Int => Int) => Int = (x: Int, y: Int => Int) => y(x)
val f = (x: Int, y: Int => Int) => y(x)
// all following are the same!
println(f(3, (m: Int) => m + 1))
println(f(3, m => m + 1))
println(f(3, _ + 1))
println(f(3, 1 + _))
println(f(3, 1 +))
val g = (x: Int) => (y: Int) => x + y
println(g(4)(5))
val gg = g(5)
println(gg(10))
}
Example 84
| Source File: _01_2_Functions.scala From LearningScala with Apache License 2.0 | 5 votes |
|
package _010_functions
object _01_2_Functions extends App {
// val f0 = (s: String) => s.length
val f0: String => Int = _.length
print("f0(\"Hello\"): ")
println(f0("Hello"))
println("======================")
{
val tupleFirst = (t: (String, Int)) => t._1
val getFirstThreeLetters = (s: String) => s.substring(0, 3)
val andThenFunction = tupleFirst.andThen(getFirstThreeLetters)
print("andThenFunction(\"Fellow\", 100): ")
println(andThenFunction("Fellow", 100))
println("======================")
}
}
Example 85
| Source File: RationalTest.scala From LearningScala with Apache License 2.0 | 5 votes |
|
package _030_oop_introductory_example
import scala.language.postfixOps
object RationalTest extends App {
println(s"Rational(1, 2): ${Rational(1, 2)}")
println(s"Rational(10, 20): ${Rational(10, 20)}")
println()
val r1 = Rational(1, 2)
val r2 = Rational(3, 4)
val r3 = Rational(6, 16)
println(s"r1: $r1")
println(s"r2: $r2")
println(s"r3: $r3")
println()
println(s"r1 + r2 ==> ${r1 + r2}")
println(s"r1 - r2 ==> ${r1 - r2}")
println(s"r1 * r2 ==> ${r1 * r2}")
println(s"r1 / r2 ==> ${r1 / r2}")
println()
println(s"r1 + 5 ==> ${r1 + 5}")
println(s"r1 - 5 ==> ${r1 - 5}")
println(s"r1 * 5 ==> ${r1 * 5}")
println(s"r1 / 5 ==> ${r1 / 5}")
println()
println(s"r1 + r2 * r3 ==> ${r1 + r2 * r3}")
println(s"r1 * r2 + r3 ==> ${r1 * r2 + r3}")
println()
println(s"r1 > r2 : ${r1 > r2}")
println(s"r1 < r2 : ${r1 < r2}")
println(s"r1 >= r2 : ${r1 >= r2}")
println(s"r1 <= r2 : ${r1 <= r2}")
println(s"r1 == r2 : ${r1 == r2}")
println(s"r1 != r2 : ${r1 != r2}")
println()
// using implicit conversion implemented by the companion object
println(s"5 + r1 ==> ${5 + r1}")
println(s"5 - r1 ==> ${5 - r1}")
println(s"5 * r1 ==> ${5 * r1}")
println(s"5 / r1 ==> ${5 / r1}")
println()
println(s"r1 max r2 ==> ${r1 max r2}")
println(s"r1 min r2 ==> ${r1 min r2}")
println()
println(s"r1 abs ==> ${r1 abs}")
println(s"Rational(-5, 3) abs ==> ${Rational(-5, 3) abs}")
}
Example 86
| Source File: _99_MoreFunctions.scala From LearningScala with Apache License 2.0 | 5 votes |
|
package _021_collections_with_functions
import scala.language.postfixOps
object _99_MoreFunctions extends App {
val a = List(1, 2, 3, 4, 5, 6)
// val b = a partition (x => x % 2 == 0)
val viewResult = (1 to 1000000000).view.map(x => x * 2).take(10).mkString(", ")
println(viewResult)
println()
println("===== by name example =====")
val f = List.fill(10) {
val x = 10
val y = 20
x + y + 19
}
println(f)
}
Example 87
| Source File: _02_Filter_FilterNot_Exsits_Functions.scala From LearningScala with Apache License 2.0 | 5 votes |
|
package _021_collections_with_functions
import scala.language.postfixOps
object _02_Filter_FilterNot_Exsits_Functions extends App {
val a = 1 to 10
println(s"a.filter(x => x % 2 == 0) ==> ${a.filter(x => x % 2 == 0)}")
println(s"a.filter(_ % 2 == 0) ==> ${a.filter(_ % 2 == 0)}")
println()
println(s"a.filterNot(x => x % 2 == 0) ==> ${a.filterNot(x => x % 2 == 0)}")
println(s"a.filterNot(_ % 2 == 0) ==> ${a.filterNot(_ % 2 == 0)}")
println()
println(s"a.exists(x => x % 2 == 0) ==> ${a.exists(x => x % 2 == 0)}")
println(s"a.exists(_ % 2 == 0) ==> ${a.exists(_ % 2 == 0)}")
println()
println(s"a.forall(x => x % 2 == 0) ==> ${a.forall(x => x % 2 == 0)}")
println(s"a.forall(_ % 2 == 0) ==> ${a.forall(_ % 2 == 0)}")
println()
// you may use Regex for this!
def filterVowels(s: String): String = s.toLowerCase.filter(c => Set('a', 'e', 'i', 'o', 'u').contains(c))
print("filterVowels(\"Orange\") ==> ")
println(filterVowels("Orange"))
println()
val b = Set("Brown", "Red", "Green", "purple", "Gray", "Yellow")
println(s"b.filter(s => filterVowels(s).length > 1) ==> ${b.filter(s => filterVowels(s).length > 1)}")
println()
val m = Map(1 -> "One", 2 -> "Two", 3 -> "Three", 4 -> "Four")
println(s"m.filterKeys(_ % 2 == 0) ==> ${m.filterKeys(_ % 2 == 0)}")
println()
println(s"Some(5).filter(_ % 2 == 0) ==> ${Some(5).filter(_ % 2 == 0)}") // THIS IS WOW!
println(s"Some(4).filter(_ % 2 == 0) ==> ${Some(4).filter(_ % 2 == 0)}")
println()
// convert to list only to show contents
println(s"(Array(1, 2, 3, 44) map (2 *)).toList ==> ${(Array(1, 2, 3, 44) map (2 *)).toList}")
println()
print("\"Hello World\" filter (_.isUpper) ==> ")
println("Hello World" filter (_.isUpper))
}
Example 88
| Source File: _02_EncodeList.scala From LearningScala with Apache License 2.0 | 5 votes |
|
package _021_collections_with_functions.examples
import scala.language.postfixOps
object _02_EncodeList extends App {
println(pack(List("a", "a", "a", "b", "c", "c", "a")))
println(packEncode1(List("a", "a", "a", "b", "c", "c", "a")))
println(packEncode2(List("a", "a", "a", "b", "c", "c", "a")))
def pack[T](xs: List[T]): List[List[T]] = xs match {
case Nil => Nil
case x :: _ =>
val (first, rest) = xs span (p => p == x)
first :: pack(rest)
}
def packEncode1[T](xs: List[T]): List[(T, Int)] = xs match {
case Nil => Nil
case x :: _ =>
val (first, rest) = xs span (p => p == x)
(first head, first length) :: packEncode1(rest)
}
def packEncode2[T](xs: List[T]): List[(T, Int)] =
pack(xs) map (ys => (ys head, ys length))
}
Example 89
| Source File: _01_MapFunction.scala From LearningScala with Apache License 2.0 | 5 votes |
|
package _021_collections_with_functions
import scala.language.postfixOps
object _01_MapFunction extends App {
val a = (1 to 5).toList
val f = (x: Int) => x + 1
println(a)
println("*******************")
println(a.map(f))
println(a.map((x: Int) => x + 1))
println(a.map(x => x + 1))
println(a.map(_ + 1))
println(a.map(1 + _))
println(a.map(1 +))
println()
println(a map (x => x * x))
println()
val b = Set("Brown", "Red", "Green", "purple", "Gray", "Yellow")
println(b)
println("********************************************")
println(b.map(x => x.length))
println(b.map(_.length))
println("***************")
println(b.map(x => (x, x.length)))
println("***************")
println()
println()
val fifaMensChamps = Map('Germany -> 4, 'Brazil -> 5, 'Italy -> 4, 'Argentina -> 2)
println(fifaMensChamps.map(t => (Symbol("Team " + t._1.name), t._2)))
println()
println()
println("Hello!".map(x => (x + 1).toChar))
println(Some(4).map(1 +))
println(None.asInstanceOf[Option[Int]].map(1 +))
val age: Option[Int] = None
println(age.map(1 +))
}
Example 90
| Source File: _07_ZipAndUnzip.scala From LearningScala with Apache License 2.0 | 5 votes |
|
package _021_collections_with_functions
import scala.language.postfixOps
object _07_ZipAndUnzip extends App {
val a = List(1, 2, 3, 4)
val b = List(5, 6, 7, 8)
val z = a zip b
println("zip: " + z)
println()
println(s"(1 to 5) zip (6 to 9) ==> ${(1 to 5) zip (6 to 9)}")
println(s"(1 to 2) zip (6 to 9) ==> ${(1 to 2) zip (6 to 9)}")
println()
println(s"a zipWithIndex ==> ${a zipWithIndex}")
println()
println("unzip: " + z.unzip)
println()
val intPair = (List(10, 20), List(3, 4, 5))
val stringLengths = (List("abc", "de"), List(3, 2))
println(s"intPair.zipped.map(_ * _) ==> ${intPair.zipped.map(_ * _)}")
println(s"stringLengths.zipped.forall(_.length == _) ==> ${stringLengths.zipped.forall(_.length == _)}")
println(s"stringLengths.zipped.exists(_.length != _) ==> ${stringLengths.zipped.exists(_.length != _)}")
}
Example 91
| Source File: _05_1_ForComprehensions.scala From LearningScala with Apache License 2.0 | 5 votes |
|
package _021_collections_with_functions
import scala.language.postfixOps
object _05_1_ForComprehensions extends App {
// for (i <- 1 to 10)
// println(i)
//
// println
val result = for (i <- 1 to 10) yield i + 1
println(result)
val result2 = (1 to 10).map(1 +)
println(result2)
val result3 = for (i <- Some(100)) yield i + 40
println(result3)
val result4 = Some(100).map(40 +)
val result5 = for (i <- List(1, 2, 3, 4);
j <- List(5, 6, 7, 8, 9)) yield (i, j)
println(result5)
val result6 = List(1, 2, 3, 4).map(i => List(5, 6, 7, 8).map(j => (i, j)))
println(result6)
val result6b = List(1, 2, 3, 4).flatMap(i => List(5, 6, 7, 8, 9).map(j => (i, j)))
println(result6b)
val result7 = for (i <- List(1, 2, 3, 4) if (i % 2 == 0);
j <- List(5, 6, 7, 8)) yield (i, j)
println(result7)
val result7b = for (i <- List(1, 2, 3, 4) if i % 2 == 0;
j <- List(5, 6, 7, 8)) yield (i, j)
println(result7b)
val result7c = for (i <- List(1, 2, 3, 4)
if i % 2 == 0;
j <- List(5, 6, 7, 8)) yield (i, j)
println(result7c)
val result7d = List(1, 2, 3, 4).filter(i => i % 2 == 0).flatMap(i => List(5, 6, 7, 8).map(j => (i, j)))
println(result7d)
// lazy filter
val result7e = List(1, 2, 3, 4).withFilter(i => i % 2 == 0).flatMap(i => List(5, 6, 7, 8).map(j => (i, j)))
println(result7e)
val result8 = for (i <- List(1, 2, 3, 4);
j <- List(5, 6, 7, 8) if j < 7) yield (i, j)
println(result8)
val result8c = for (i <- List(1, 2, 3, 4);
j <- List(5, 6, 7, 8)
if j < 7) yield (i, j)
println(result8c)
val result8d =
List(1, 2, 3, 4)
.flatMap(i => List(5, 6, 7, 8)
.filter(j => j < 7)
.map(j => (i, j)))
println(result8d)
// lazy filter
val result8e =
List(1, 2, 3, 4)
.flatMap(i => List(5, 6, 7, 8)
.withFilter(j => j < 7)
.map(j => (i, j)))
}
Example 92
| Source File: _06_Tuples.scala From LearningScala with Apache License 2.0 | 5 votes |
|
package _000_intro
import scala.language.postfixOps
object _06_Tuples extends App {
val t = (1, "Hello", 3.2)
println(t._1)
println(t._2)
println(t._3)
// or...
// val t2: (Int, String, Double) = t
// val t3:Tuple3[Int, String, Double] = t
val tuple2 = (1, "Hello")
println(tuple2)
println(tuple2 swap)
println(tuple2.swap)
println(tuple2)
val (x, y) = (1, 2)
println(x)
println(y)
}
Example 93
| Source File: RecommendationIntegrationTest.scala From algoliasearch-client-scala with MIT License | 5 votes |
|
package algolia.integration
import algolia.AlgoliaDsl._
import algolia.AlgoliaTest
import algolia.responses.GetStrategyResponse
import scala.concurrent.Future
import scala.language.postfixOps
class RecommendationIntegrationTest extends AlgoliaTest {
describe("recommendation API test") {
it("should get personalization strategy without failing") {
val task: Future[GetStrategyResponse] = AlgoliaTest.client.execute {
get personalizationRecommendationStrategy
}
whenReady(task) { strategy =>
strategy should not be None
}
}
}
}
Example 94
| Source File: File.scala From nescala with GNU General Public License v2.0 | 5 votes |
|
package com.owlandrews.nescala.helpers
import com.owlandrews.nescala.Console
object File {
import java.io.File
import java.net.URL
import java.io.{FileFilter, FileInputStream, FileOutputStream, ObjectInputStream, ObjectOutputStream}
import javax.imageio.ImageIO
import scala.util.Try
import scala.xml.XML
import scala.language.postfixOps
import sys.process._
import com.typesafe.config.ConfigFactory
def Download(url: String, filename: String) = (for{
url <- Try(new URL(url))
conn <- Try(url.openConnection().connect())
file <- Try(new File(filename))
} yield Try(url #> file !!)) map {x => new File(filename)}
def Writer(filename: String)(op: java.io.PrintWriter => Unit) = {
val p = new java.io.PrintWriter(new File(filename))
try op(p)
finally p.close()
}
def Write(filename: String, content: String) = {
val res = new java.io.PrintWriter(new File(filename))
res.write(content)
res.close()
}
def Filter = new FileFilter {
override def accept(pathname: File): Boolean = pathname.getName.toLowerCase.endsWith(".nes")
}
def Image(file:Try[File]) = file.map(ImageIO.read)
def Image(filename:String) = Try(ImageIO.read(resource(filename)))
def Xml(filename:String) = XML.load(resource("/database.xml"))
def Config(filename:String) = {
val file = new File(filename)
file.exists() match {
case true => ConfigFactory.parseFile(file)
case false => ConfigFactory.empty()
}
}
def SaveState(console:Console) = {
val fos = new FileOutputStream(s"$ApplicationFolder/${console.cartridge.CRC}.save")
val oos = new ObjectOutputStream(fos)
oos.writeObject(console)
oos.close()
}
def LoadState(crc:String):Try[Console] = Try {
val fis = new FileInputStream(s"$ApplicationFolder/$crc.save")
val ois = new ObjectInputStreamWithCustomClassLoader(fis)
val console = ois.readObject.asInstanceOf[Console]
ois.close()
console
}
// Taken from: https://gist.github.com/ramn/5566596
private class ObjectInputStreamWithCustomClassLoader(fileInputStream: FileInputStream) extends ObjectInputStream(fileInputStream) {
override def resolveClass(desc: java.io.ObjectStreamClass): Class[_] = {
try { Class.forName(desc.getName, false, getClass.getClassLoader) }
catch { case ex: ClassNotFoundException => super.resolveClass(desc) }
}
}
lazy val ApplicationFolder: File = {
val settingDirectory = System.getProperty("user.home") + "/.nescala"
val settings = new java.io.File(settingDirectory)
if (!settings.exists()) settings.mkdir()
settings
}
private def resource(filename:String) = getClass.getResourceAsStream(filename)
}
Example 95
| Source File: TaskService.scala From ddd-on-scala with MIT License | 5 votes |
|
package crossroad0201.dddonscala.application.task
import crossroad0201.dddonscala.application._
import crossroad0201.dddonscala.domain.task._
import crossroad0201.dddonscala.domain.user.{UserId, UserRepository}
import crossroad0201.dddonscala.domain.{EntityIdGenerator, EntityMetaDataCreator}
import scala.language.postfixOps
trait TaskService extends TransactionAware {
def createNewTask(name: TaskName, authorId: UserId): Either[ServiceError, Task] =
tx { implicit uow =>
// NOTE: 他集約のエンティティを、自集約のロール型に変換するために、パッケージをimportします。
import crossroad0201.dddonscala.domain.task.{TaskService => TaskDomainService, _}
for {
author <- userRepository.get(authorId) ifNotExists NotFoundError("USER", authorId)
createdTask = author.createTask(name)
exampledTask = TaskDomainService.applyBusinessRuleTo(createdTask.entity)
savedTask <- taskRepository.save(exampledTask) ifFailureThen asServiceError
_ <- taskEventPublisher.publish(createdTask.event) ifFailureThen asServiceError
} yield savedTask
}
def assignToTask(taskId: TaskId, assigneeId: UserId): Either[ServiceError, Task] =
tx { implicit uow =>
import crossroad0201.dddonscala.domain.task._
for {
task <- taskRepository.get(taskId) ifNotExists NotFoundError("TASK", taskId)
assignee <- userRepository.get(assigneeId) ifNotExists NotFoundError("USER", assigneeId)
assignedTask <- assignee.assignTo(task) ifLeftThen asServiceError
savedTask <- taskRepository.save(assignedTask.entity) ifFailureThen asServiceError
_ <- taskEventPublisher.publish(assignedTask.event) ifFailureThen asServiceError
} yield savedTask
}
def unAssignFromTask(taskId: TaskId): Either[ServiceError, Task] =
tx { implicit uow =>
for {
task <- taskRepository.get(taskId) ifNotExists NotFoundError("TASK", taskId)
unAssignedTask <- task.unAssign ifLeftThen asServiceError
savedTask <- taskRepository.save(unAssignedTask.entity) ifFailureThen asServiceError
_ <- taskEventPublisher.publish(unAssignedTask.event) ifFailureThen asServiceError
} yield savedTask
}
def commentToTask(taskId: TaskId, commenterId: UserId, message: CommentMessage): Either[ServiceError, Task] =
tx { implicit uow =>
import crossroad0201.dddonscala.domain.task._
for {
task <- taskRepository.get(taskId) ifNotExists NotFoundError("TASK", taskId)
commenter <- userRepository.get(commenterId) ifNotExists NotFoundError("USER", commenterId)
commentedTask = commenter.commentTo(task, message)
savedTask <- taskRepository.save(commentedTask.entity) ifFailureThen asServiceError
_ <- taskEventPublisher.publish(commentedTask.event) ifFailureThen asServiceError
} yield savedTask
}
def closeTask(taskId: TaskId): Either[ServiceError, Task] =
tx { implicit uow =>
for {
task <- taskRepository.get(taskId) ifNotExists NotFoundError("TASK", taskId)
closedTask <- task.close ifLeftThen asServiceError
savedTask <- taskRepository.save(closedTask.entity) ifFailureThen asServiceError
_ <- taskEventPublisher.publish(closedTask.event) ifFailureThen asServiceError
} yield savedTask
}
def reOpenTask(taskId: TaskId): Either[ServiceError, Task] =
tx { implicit uow =>
for {
task <- taskRepository.get(taskId) ifNotExists NotFoundError("TASK", taskId)
reOpenedTask <- task.reOpen ifLeftThen asServiceError
savedTask <- taskRepository.save(reOpenedTask.entity) ifFailureThen asServiceError
_ <- taskEventPublisher.publish(reOpenedTask.event) ifFailureThen asServiceError
} yield savedTask
}
}
case class IllegalTaskOperationError(task: Task)
extends ApplicationError(ServiceErrorCodes.InvalidTaskOperation, task.id, task.state)
Example 96
| Source File: WebsocketServer.scala From akka_streams_tutorial with MIT License | 5 votes |
|
package alpakka.env
import akka.actor.ActorSystem
import akka.http.scaladsl.Http
import akka.http.scaladsl.model.ws._
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server.Route
import akka.http.scaladsl.server.directives.WebSocketDirectives
import akka.stream.scaladsl.{Flow, Sink, Source}
import scala.concurrent.Await
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.util.{Failure, Success}
object WebsocketServer extends App with WebSocketDirectives {
implicit val system = ActorSystem("WebsocketServer")
implicit val executionContext = system.dispatcher
val (address, port) = ("127.0.0.1", 6002)
server(address, port)
def server(address: String, port: Int) = {
def echoFlow: Flow[Message, Message, Any] =
Flow[Message].mapConcat {
case tm: TextMessage =>
println(s"Server received: $tm")
TextMessage(Source.single("Echo: ") ++ tm.textStream) :: Nil
case bm: BinaryMessage =>
// ignore binary messages but drain content to avoid the stream being clogged
bm.dataStream.runWith(Sink.ignore)
Nil
}
val websocketRoute: Route =
path("echo") {
handleWebSocketMessages(echoFlow)
}
val bindingFuture = Http().bindAndHandle(websocketRoute, address, port)
bindingFuture.onComplete {
case Success(b) =>
println("Server started, listening on: " + b.localAddress)
case Failure(e) =>
println(s"Server could not bind to $address:$port. Exception message: ${e.getMessage}")
system.terminate()
}
sys.addShutdownHook {
println("About to shutdown...")
val fut = bindingFuture.map(serverBinding => serverBinding.terminate(hardDeadline = 3.seconds))
println("Waiting for connections to terminate...")
val onceAllConnectionsTerminated = Await.result(fut, 10.seconds)
println("Connections terminated")
onceAllConnectionsTerminated.flatMap { _ => system.terminate()
}
}
}
}
Example 97
| Source File: LatencyAnalyzer.scala From spatial with MIT License | 5 votes |
|
package spatial.dse
import argon._
import spatial.lang._
import spatial.node._
import spatial.util.spatialConfig
import spatial.util.modeling._
import spatial.traversal._
import spatial.targets._
import java.io.File
import models._
import argon.node._
case class LatencyAnalyzer(IR: State, latencyModel: LatencyModel) extends AccelTraversal {
var cycleScope: List[Double] = Nil
var intervalScope: List[Double] = Nil
var totalCycles: Seq[Long] = Seq()
val batchSize = 1000
def getListOfFiles(d: String):List[String] = {
import java.nio.file.{FileSystems, Files}
import scala.collection.JavaConverters._
val dir = FileSystems.getDefault.getPath(d)
Files.walk(dir).iterator().asScala.filter(Files.isRegularFile(_)).map(_.toString).toList//.foreach(println)
}
override def silence(): Unit = {
super.silence()
}
def test(rewriteParams: Seq[Seq[Any]]): Unit = {
import scala.language.postfixOps
import java.io.File
import sys.process._
val gen_dir = if (config.genDir.startsWith("/")) config.genDir + "/" else config.cwd + s"/${config.genDir}/"
val modelJar = getListOfFiles(gen_dir + "/model").filter(_.contains("RuntimeModel-assembly")).head
totalCycles = rewriteParams.grouped(batchSize).flatMap{params =>
val batchedParams = params.map{rp => "tune " + rp.mkString(" ")}.mkString(" ")
val cmd = s"""java -jar ${modelJar} ni ${batchedParams}"""
// println(s"running cmd: $cmd")
val output = Process(cmd, new File(gen_dir)).!!
output.split("\n").filter(_.contains("Total Cycles for App")).map{r =>
"^.*: ".r.replaceAllIn(r,"").trim.toLong
}.toSeq
}.toSeq
// println(s"DSE Model result: $totalCycles")
}
override protected def preprocess[A](b: Block[A]): Block[A] = {
super.preprocess(b)
}
override protected def postprocess[A](b: Block[A]): Block[A] = {
super.postprocess(b)
}
override protected def visit[A](lhs: Sym[A], rhs: Op[A]): Unit = { }
}
Example 98
| Source File: DotHierarchicalCodegen.scala From spatial with MIT License | 5 votes |
|
package spatial.codegen.dotgen
import argon._
import scala.collection.mutable
import utils.io.files._
import sys.process._
import scala.language.postfixOps
import spatial.metadata.control._
import spatial.metadata.bounds._
trait DotHierarchicalCodegen extends DotCodegen {
override def entryFile: String = s"Top.$ext"
//override def clearGen(): Unit = {} // everything cleared in DotFlatCodegen
override protected def gen(lhs: Sym[_], rhs: Op[_]): Unit = {
currScope.addNode(lhs)
emitNode(lhs)
if (blocks(lhs).nonEmpty) {
scope(Some(lhs)).begin {
open(src"subgraph cluster_${lhs} {")
emit(src"""${graphAttr(lhs).map { case (k,v) => s"$k=$v" }.mkString(" ")}""")
//strMeta(lhs)
rhs.binds.filter(_.isBound).foreach{ b =>
currScope.addNode(b)
emitNode(b)
}
rhs.blocks.foreach(ret)
close(src"}")
}
}
}
def ancestryBetween(a:Sym[_], b:Sym[_]):(List[Sym[_]], List[Sym[_]], List[Sym[_]]) = {
val branchA = ancestors(a)
val branchB = ancestors(b)
val shared = (branchA intersect branchB)
(shared, branchA.filterNot(shared.contains(_)), branchB.filterNot(shared.contains(_)))
}
override def addEdge(from:Sym[_], to:Sym[_], alias:Map[Sym[_], String]=Map.empty):Unit = {
if (!nodes.contains(from)) dbgblk(src"emitEscapeEdge($from, $to):"){
assert(from.vecConst.isEmpty)
val (shared, fromBranch, toBranch) = ancestryBetween(from, to)
dbgs(src"from=$from")
dbgs(src"to=$to")
dbgs(src"shared=$shared")
dbgs(src"fromBranch=$fromBranch")
dbgs(src"toBranch=$toBranch")
toBranch.filterNot{ sym => sym == to }.foreach { ancestor =>
val toNode = toBranch(toBranch.indexOf(ancestor)-1)
scope(Some(ancestor))
.addExternNode(from)
.addEdge(from, to, alias + (to -> getAlias(toNode, alias)))
}
fromBranch.filterNot{ sym => sym == from }.foreach { ancestor =>
val fromNode = fromBranch(fromBranch.indexOf(ancestor)-1)
scope(Some(ancestor))
.addExternNode(to)
.addEdge(from, to, alias + (from -> getAlias(fromNode, alias)))
}
scope(shared.headOption)
.addEdge(from, to, alias + (from -> getAlias(fromBranch.last, alias)) + (to -> getAlias(toBranch.last, alias)))
} else {
currScope.addEdge(from, to, alias)
}
}
override def nodeAttr(lhs:Sym[_]):Map[String,String] = lhs match {
case lhs if blocks(lhs).nonEmpty =>
super.nodeAttr(lhs) + ("URL" -> s""""${scope(Some(lhs)).fileName}.html"""")
case lhs => super.nodeAttr(lhs)
}
override def graphAttr(lhs:Sym[_]):Map[String,String] = lhs match {
case lhs if blocks(lhs).nonEmpty =>
// navigate back to parent graph, or flat dot graph if already at top level
super.graphAttr(lhs) + ("URL" -> s""""${scope(lhs.blk.s).fileName}.html"""")
case lhs => super.graphAttr(lhs)
}
}
Example 99
| Source File: DotFlatCodegen.scala From spatial with MIT License | 5 votes |
|
package spatial.codegen.dotgen
import argon._
import scala.collection.mutable
import utils.io.files._
import sys.process._
import scala.language.postfixOps
trait DotFlatCodegen extends DotCodegen {
override def entryFile: String = s"Main.$ext"
// Remove all dot files. First dot gen
//override def clearGen(): Unit = super.clearGen()
override protected def gen(lhs: Sym[_], rhs: Op[_]): Unit = {
currScope.addNode(lhs)
if (rhs.blocks.nonEmpty) {
open(src"subgraph cluster_${lhs} {")
emit(src"""${graphAttr(lhs).map { case (k,v) => s"$k=$v" }.mkString(" ")}""")
//strMeta(lhs)
if (inputs(lhs).nonEmpty) emitNode(lhs)
rhs.binds.filter(_.isBound).foreach{ b =>
currScope.addNode(b)
emitNode(b)
emitEdge(lhs, b)
}
rhs.blocks.foreach(ret)
close(src"}")
} else {
emitNode(lhs)
}
}
override def graphAttr(lhs:Sym[_]):Map[String,String] = lhs match {
case lhs if blocks(lhs).nonEmpty =>
super.graphAttr(lhs) + ("URL" -> s""""${s"$lhs.html"}"""")
case lhs => super.graphAttr(lhs)
}
}
Example 100
| Source File: Timing.scala From logging with Apache License 2.0 | 5 votes |
|
package demo.test
import java.net.InetAddress
import akka.actor.ActorSystem
import com.persist.logging._
import logging_demo.BuildInfo
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.concurrent.{Future, Await}
import scala.concurrent.ExecutionContext.Implicits._
case class TimeClass() extends ClassLogging with Timing {
def b(id: RequestId): Unit = {
Time(id, "bbb") {
Thread.sleep(100)
}
}
def a(id: RequestId): Unit = {
Time(id, "top") {
Time(id, "aaa1") {
Thread.sleep(200)
}
b(id)
Time(id, "aaa2") {
Thread.sleep(300)
}
b(id)
}
}
}
case class FutureClass() extends ClassLogging with Timing {
def demo(id: RequestId): Future[Int] = {
val token = time.start(id, "top")
val f1 = Future {
Time(id, "f1") {
Thread.sleep(100)
100
}
}
val f2 = f1.map { i =>
val result = Time(id, "f2") {
Thread.sleep(200)
Time(id, "f2a") {
i * 2
}
}
result
}
val f3 = f2.recover{ case ex:Throwable =>
log.error("Timing test failed", ex)
-1
}
f3.map { i =>
time.end(id, "top", token)
i
}
}
}
object Timing {
case class F() extends ClassLogging {
val fc = FutureClass()
val f = fc.demo(RequestId())
val i = Await.result(f, 3 seconds)
log.info(Map("@msg" -> "Future result", "val" -> i))
}
def main(args: Array[String]) {
val system = ActorSystem("test")
val host = InetAddress.getLocalHost.getHostName
val loggingSystem = LoggingSystem(system, BuildInfo.name, BuildInfo.version, host)
val tc = new TimeClass()
tc.a(RequestId())
tc.a(RequestId())
F()
Await.result(loggingSystem.stop, 30 seconds)
Await.result(system.terminate(), 20 seconds)
}
}
Example 101
| Source File: ActorDemo.scala From logging with Apache License 2.0 | 5 votes |
|
package demo.test
import java.net.InetAddress
import akka.actor.{Props, Actor, ActorSystem}
import com.persist.logging._
import logging_demo.BuildInfo
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.concurrent.Await
object DemoActor {
def props() = Props(new DemoActor())
}
class DemoActor() extends Actor with ActorLogging {
println(this.getClass.getSimpleName)
def receive = {
case "foo" => log.info("Saw foo")
case "done" => context.stop(self)
case x: Any => log.error(Map("@msg" -> "Unexpected actor message",
"message" -> x.toString))
}
}
case class ActorDemo(system: ActorSystem) {
def demo(): Unit = {
val a = system.actorOf(DemoActor.props(), name = "Demo")
a ! "foo"
a ! "bar"
a ! "done"
}
}
object ActorDemo {
def main(args: Array[String]) {
val system = ActorSystem("test")
val host = InetAddress.getLocalHost.getHostName
val loggingSystem = LoggingSystem(system, BuildInfo.name, BuildInfo.version, host)
val act = ActorDemo(system)
act.demo()
Await.result(loggingSystem.stop, 30 seconds)
Await.result(system.terminate(), 20 seconds)
}
}
Example 102
| Source File: Exceptions.scala From logging with Apache License 2.0 | 5 votes |
|
package demo.test
import java.net.InetAddress
import akka.actor.ActorSystem
import com.persist.logging._
import logging_demo.BuildInfo
import scala.concurrent.Await
import scala.concurrent.duration._
import scala.language.postfixOps
case class MyException(msg: RichMsg) extends RichException(msg)
case class ExceptionClass() extends ClassLogging {
def demo(): Unit = {
log.error("Test", new Exception("Bad Code"))
log.warn("Rich", RichException(Map("@msg" -> "Fail", "count" -> 23)))
log.error("Special", MyException(Map("@msg" -> "Fail", "count" -> 23)))
try {
throw MyException(Map("@msg" -> "Die", "name" -> "abc"))
} catch {
case ex: Exception => log.error("Caught exception", ex)
}
}
}
object Exceptions {
def main(args: Array[String]) {
val system = ActorSystem("test")
val host = InetAddress.getLocalHost.getHostName
val loggingSystem = LoggingSystem(system, BuildInfo.name, BuildInfo.version, host)
val ec = new ExceptionClass()
ec.demo()
Await.result(loggingSystem.stop, 30 seconds)
Await.result(system.terminate(), 20 seconds)
}
}
Example 103
| Source File: OtherApis.scala From logging with Apache License 2.0 | 5 votes |
|
package demo.test
import java.net.InetAddress
import akka.actor.{Props, Actor, ActorSystem}
import com.persist.logging._
import logging_demo.BuildInfo
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.concurrent.Await
import org.slf4j.LoggerFactory
case class Slf4jDemo() {
val slf4jlog = LoggerFactory.getLogger(classOf[Slf4jDemo])
def demo(): Unit = {
slf4jlog.warn("slf4j")
}
}
object AkkaActor {
def props() = Props(new AkkaActor())
}
class AkkaActor() extends Actor with akka.actor.ActorLogging {
def receive = {
case "foo" => log.warning("Saw foo")
case "done" => context.stop(self)
case x: Any => log.error(s"Unexpected actor message: ${x}")
}
}
case class AkkaDemo(system: ActorSystem) {
def demo(): Unit = {
val a = system.actorOf(AkkaActor.props(), name="Demo")
a ! "foo"
a ! "bar"
a ! "done"
}
}
object OtherApis {
def main(args: Array[String]) {
val system = ActorSystem("test")
val host = InetAddress.getLocalHost.getHostName
val loggingSystem = LoggingSystem(system, BuildInfo.name, BuildInfo.version, host)
val slf = Slf4jDemo()
slf.demo()
val act = AkkaDemo(system)
act.demo()
Await.result(loggingSystem.stop, 30 seconds)
Await.result(system.terminate(), 20 seconds)
}
}
Example 104
| Source File: Appender.scala From logging with Apache License 2.0 | 5 votes |
|
package demo.test
import java.net.InetAddress
import akka.actor.{ActorRefFactory, ActorSystem}
import com.persist.logging._
import logging_demo.BuildInfo
import scala.concurrent.Await
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.concurrent.Future
case class AppenderClass() extends ClassLogging {
def demo(): Unit = {
log.info("Test")
log.error("Foo failed")
log.warn(Map("@msg" -> "fail", "value" -> 23))
}
}
object FlatAppender extends LogAppenderBuilder {
def apply(factory: ActorRefFactory, stdHeaders: Map[String, RichMsg])
= new FlatAppender(factory, stdHeaders)
}
class FlatAppender(factory: ActorRefFactory, stdHeaders: Map[String, RichMsg]) extends LogAppender {
def append(msg: Map[String, RichMsg], category: String) {
if (category == "common") {
val level = msg.get("@severity") match {
case Some(s: String) => s
case _ => "???"
}
val time = msg.get("@timestamp") match {
case Some(s: String) => s
case _ => "???"
}
val message = richToString(msg.getOrElse("msg","???"))
println(s"$time\t$level\t$message")
}
}
def finish(): Future[Unit] = Future.successful(())
def stop(): Future[Unit] = Future.successful(())
}
object Appender {
def main(args: Array[String]) {
val system = ActorSystem("test")
val host = InetAddress.getLocalHost.getHostName
val loggingSystem = LoggingSystem(system, BuildInfo.name,
BuildInfo.version, host, appenderBuilders = Seq(FileAppender, FlatAppender))
val sc = new SimpleClass()
sc.demo()
Await.result(loggingSystem.stop, 30 seconds)
Await.result(system.terminate(), 20 seconds)
}
}
Example 105
| Source File: Alternative.scala From logging with Apache License 2.0 | 5 votes |
|
package demo.test
import java.net.InetAddress
import akka.actor.ActorSystem
import com.persist.logging._
import logging_demo.BuildInfo
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.concurrent.Await
case class AltClass() extends ClassLogging {
def demo(): Unit = {
log.alternative("foo", Map("message"->"test"))
log.alternative("foo", Map("a" -> "x", "b" -> false, "c" -> 65))
log.alternative("bar", Map("message"->"btest"))
}
}
object Alternative {
def main(args: Array[String]) {
val system = ActorSystem("test")
val host = InetAddress.getLocalHost.getHostName
val loggingSystem = LoggingSystem(system, BuildInfo.name, BuildInfo.version, host)
val alt = new AltClass()
alt.demo()
Await.result(loggingSystem.stop, 30 seconds)
Await.result(system.terminate(), 20 seconds)
}
}
Example 106
| Source File: RequestIds.scala From logging with Apache License 2.0 | 5 votes |
|
package demo.test
import java.net.InetAddress
import akka.actor.ActorSystem
import com.persist.logging._
import logging_demo.BuildInfo
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.concurrent.Await
case class RequestB() extends ClassLogging {
def demo(id: AnyId): Unit = {
log.trace("In B", id = id)
log info("BBB", id = id)
}
}
case class RequestC() extends ClassLogging {
def demo(id: AnyId): Unit = {
log.trace("In C", id = id)
log.info("CCC", id = id)
}
}
case class RequestA() extends ClassLogging {
val b = RequestB()
val c = RequestC()
def demo(id: AnyId): Unit = {
log.trace("Enter A", id = id)
b.demo(id)
log.info("AAA", id = id)
c.demo(id)
log.trace("Exit A", id = id)
}
}
object RequestIds {
def main(args: Array[String]) {
val system = ActorSystem("test")
val host = InetAddress.getLocalHost.getHostName
val loggingSystem = LoggingSystem(system, BuildInfo.name, BuildInfo.version, host)
val a = new RequestA()
a.demo(noId)
a.demo(RequestId())
a.demo(RequestId(level = Some(LoggingLevels.TRACE)))
Await.result(loggingSystem.stop, 30 seconds)
Await.result(system.terminate(), 20 seconds)
}
}
Example 107
| Source File: Simple.scala From logging with Apache License 2.0 | 5 votes |
|
package demo.test
import java.net.InetAddress
import akka.actor.ActorSystem
import com.persist.logging._
import logging_demo.BuildInfo
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.concurrent.Await
case class SimpleClass() extends ClassLogging {
def demo(): Unit = {
log.info("Test")
log.error("Foo failed")
log.warn(Map("@msg" -> "fail", "value" -> 23))
}
}
object Simple {
def main(args: Array[String]) {
val system = ActorSystem("test")
val host = InetAddress.getLocalHost.getHostName
val loggingSystem = LoggingSystem(system, BuildInfo.name, BuildInfo.version, host)
val sc = new SimpleClass()
sc.demo()
Await.result(loggingSystem.stop, 30 seconds)
Await.result(system.terminate(), 20 seconds)
}
}
Example 108
| Source File: Filter.scala From logging with Apache License 2.0 | 5 votes |
|
package demo.test
import java.net.InetAddress
import akka.actor.ActorSystem
import com.persist.logging._
import logging_demo.BuildInfo
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.concurrent.Await
case class Class1() extends ClassLogging {
def demo(name: String): Unit = {
log.debug(name)
log.warn(name)
}
}
case class Class2() extends ClassLogging {
def demo(name: String): Unit = {
log.debug(name)
log.warn(name)
}
}
object Filter {
import LoggingLevels._
def filter(fields: Map[String, RichMsg], level:Level): Boolean = {
val cls = fields.get("class") match {
case Some(s: String) => s
case _ => ""
}
if (cls == "demo.test.Class1") {
level >= DEBUG
} else {
level >= WARN
}
}
def main(args: Array[String]) {
val system = ActorSystem("test")
val host = InetAddress.getLocalHost.getHostName
val loggingSystem = LoggingSystem(system, BuildInfo.name, BuildInfo.version, host)
val c1 = new Class1()
val c2 = new Class2()
c1.demo("no filter")
c2.demo("no filter")
// Add filter and change level
val oldLevel = loggingSystem.getLevel.current
loggingSystem.setFilter(Some(filter))
loggingSystem.setLevel(DEBUG)
c1.demo("filter")
c2.demo("filter")
// Reset it back
loggingSystem.setLevel(oldLevel)
loggingSystem.setFilter(None)
Await.result(loggingSystem.stop, 30 seconds)
Await.result(system.terminate(), 20 seconds)
}
}
Example 109
| Source File: TestKafka.scala From logging with Apache License 2.0 | 5 votes |
|
package com.persist.logging.test
import java.net.InetAddress
import akka.actor.ActorSystem
import com.persist.logging.kafka.KafkaAppender
import com.persist.logging._
import kafka_logging_demo.BuildInfo
import scala.language.postfixOps
import scala.concurrent.duration._
import scala.concurrent.Await
case class TestKafka() extends ClassLogging {
def send: Unit = {
for (i <- 1 to 5) {
log.warn(Map("msg" -> "test", "i" -> i))
Thread.sleep(500)
}
}
}
object TestKafka {
def main(args: Array[String]): Unit = {
val system = ActorSystem("test")
val host = InetAddress.getLocalHost.getHostName
val loggingSystem = LoggingSystem(system, BuildInfo.name,
BuildInfo.version, host, appenderBuilders = Seq(StdOutAppender, KafkaAppender))
val tc = TestKafka()
tc.send
//Thread.sleep(60000)
Await.result(loggingSystem.stop, 30 seconds)
Await.result(system.terminate(), 20 seconds)
}
}
Example 110
| Source File: RepositoryTest.scala From akka-http-slick-sample with MIT License | 5 votes |
|
package net.softler.data
import java.util.UUID
import akka.actor.ActorSystem
import akka.testkit.TestKit
import net.softler.data.model.User
import net.softler.data.persistence.{H2, UserComponent}
import org.scalatest._
import org.scalatest.concurrent.ScalaFutures
import scala.concurrent.ExecutionContext
import scala.concurrent.duration._
import scala.language.postfixOps
class RepositoryTest
extends TestKit(ActorSystem("test-system"))
with FlatSpecLike
with Matchers
with ScalaFutures
with UserComponent
with H2 {
implicit val ec: ExecutionContext = system.dispatcher
val repo: UserRepository = new UserRepository
private val testId = UUID.fromString("00000000-0000-0000-0000-000000000000")
override implicit def patienceConfig: PatienceConfig = PatienceConfig(5 seconds)
"The generic repository" should "handle a in memory database" in {
whenReady(repo.all)(_.size shouldBe 3)
}
it should "retrieve a user by id" in {
whenReady(repo.byId(testId)) { user =>
user.get.login shouldBe "tom"
}
}
it should "update a single entity" in {
val testEntity: User = repo.byId(testId).futureValue.get
val result = repo.update(testEntity).futureValue
result shouldBe 1
}
it should "delete a single user by id" in {
whenReady(repo.delete(testId)) { result =>
result shouldBe true
}
whenReady(repo.byId(testId)) { user =>
user shouldBe None
}
whenReady(repo.all)(_.size shouldBe 2)
}
}
Example 111
| Source File: Search.scala From Principles-of-Reactive-Programming with GNU General Public License v3.0 | 5 votes |
|
package suggestions
package search
import org.json4s._
import scala.concurrent.{ ExecutionContext, Future, Promise }
import ExecutionContext.Implicits.global
import scala.language.postfixOps
import scala.collection._
import scala.collection.JavaConverters._
import scala.util.Try
import scala.async.Async._
import rx.lang.scala.Observable
import observablex.{SchedulerEx, ObservableEx}
import ObservableEx._
import dispatch._
import org.json4s.native._
import retrofit.http.{GET, Query}
import retrofit.Callback
import retrofit.client.Response
import retrofit.{RetrofitError, Callback, RestAdapter}
import com.google.gson.annotations.SerializedName
object Search {
trait WikipediaService {
@GET("/w/api.php?action=opensearch&format=json&limit=15")
def suggestions(@Query("search") term: String, callback: Callback[Array[AnyRef]]): Unit
@GET("/w/api.php?action=parse&format=json&prop=text§ion=0")
def page(@Query("page") term: String, callback: Callback[Page]): Unit
}
val restAdapter = new RestAdapter.Builder().setEndpoint("https://en.wikipedia.org").build()
val service = restAdapter.create(classOf[WikipediaService])
def callbackFuture[T]: (Callback[T], Future[T]) = {
val p = Promise[T]()
val cb = new Callback[T] {
def success(t: T, response: Response) = {
p success t
}
def failure(error: RetrofitError) = {
p failure error
}
}
(cb, p.future)
}
def wikipediaSuggestionRetrofit(term: String): Future[List[String]] = {
async {
val (cb, f) = callbackFuture[Array[AnyRef]]
service.suggestions(term, cb)
val result = await { f }
val arraylist = result(1).asInstanceOf[java.util.List[String]]
arraylist.asScala.toList
}
}
def wikipediaPageRetrofit(term: String): Future[String] = {
async {
val (cb, f) = callbackFuture[Page]
service.page(term, cb)
val result = await { f }
result.parse.text.all
}
}
def wikipediaSuggestion(term: String): Future[List[String]] = wikipediaSuggestionRetrofit(term)
def wikipediaPage(term: String): Future[String] = wikipediaPageRetrofit(term)
}
Example 112
| Source File: package.scala From Principles-of-Reactive-Programming with GNU General Public License v3.0 | 5 votes |
|
import scala.language.postfixOps
import scala.io.StdIn
import scala.util._
import scala.util.control.NonFatal
import scala.concurrent._
import scala.concurrent.duration._
import ExecutionContext.Implicits.global
import scala.async.Async.{async, await}
def apply() = new CancellationTokenSource {
val p = Promise[Unit]()
val cancellationToken = new CancellationToken {
def isCancelled = p.future.value != None
}
def unsubscribe() {
p.trySuccess(())
}
}
}
}
Example 113
| Source File: JacksonSupport.scala From Sidechains-SDK with MIT License | 5 votes |
|
package com.horizen.api.http
import akka.http.scaladsl.model.HttpRequest
import akka.http.scaladsl.unmarshalling._
import akka.stream.Materializer
import com.fasterxml.jackson.databind._
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import scala.concurrent.{ExecutionContext, Future}
import scala.reflect.ClassTag
import scala.language.postfixOps
object JacksonSupport {
private val mapper = new ObjectMapper().registerModule(DefaultScalaModule)
implicit def JacksonRequestUnmarshaller[T <: AnyRef](implicit c: ClassTag[T]): FromRequestUnmarshaller[T] = {
new FromRequestUnmarshaller[T] {
override def apply(request: HttpRequest)(implicit ec: ExecutionContext, materializer: Materializer): Future[T] = {
Unmarshal(request.entity).to[String].map(str => {
if (str.isEmpty) mapper.readValue("{}", c.runtimeClass).asInstanceOf[T]
else mapper.readValue(str, c.runtimeClass).asInstanceOf[T]
})
}
}
}
}
Example 114
| Source File: FeatureUtils.scala From automl with Apache License 2.0 | 5 votes |
|
package com.tencent.angel.spark.automl.feature
import org.apache.spark.ml.linalg.{DenseVector, SparseVector, Vector}
import org.apache.spark.sql.{Dataset, Row}
import scala.language.postfixOps
object FeatureUtils {
def maxDim(dataset: Dataset[Row], col: String = "features"): Int = {
dataset.select(col).rdd.mapPartitions { rows: Iterator[Row] =>
val dim = rows.map { case Row(v: Vector) =>
v match {
case sv: SparseVector => sv.indices.last
case dv: DenseVector => dv.size
}
}.max
Iterator(dim)
}.max + 1
}
def countNonZero(dataset: Dataset[Row], col: String = "features"): Array[Int] = {
dataset.select(col).rdd.mapPartitions { rows: Iterator[Row] =>
val mergeIndices = rows.map { case Row(v: Vector) =>
v match {
case sv: SparseVector =>
sv.indices.toList
}
}.reduce(_ union _ distinct)
Iterator(mergeIndices)
}.reduce((a, b) => (a union b).distinct).toArray
}
}
Example 115
| Source File: DefaultIngestionHandler.scala From hydra with Apache License 2.0 | 5 votes |
|
package hydra.ingest.services
import akka.actor._
import hydra.common.config.ConfigSupport
import hydra.core.ingest.{HydraRequest, IngestionReport}
import hydra.core.protocol.HydraApplicationError
import scala.concurrent.duration._
import scala.language.postfixOps
class DefaultIngestionHandler(
val request: HydraRequest,
val registry: ActorRef,
val requestor: ActorRef,
val timeout: FiniteDuration
) extends Actor
with IngestionHandler
with ActorLogging {
private implicit val ec = context.dispatcher
// private val mediator = DistributedPubSub(context.system).mediator
//mediator ! Subscribe(HydraRequestPublisher.TopicName, Some(HydraRequestPublisher.GroupName), self)
override def complete(report: IngestionReport): Unit = {
requestor ! report
context.stop(self)
}
override def fail(e: Throwable): Unit = {
requestor ! HydraApplicationError(e)
context.stop(self)
}
}
object DefaultIngestionHandler extends ConfigSupport {
def props(
request: HydraRequest,
registry: ActorRef,
requestor: ActorRef
): Props = {
import ConfigSupport._
val timeout = applicationConfig
.getDurationOpt("ingestion.timeout")
.getOrElse(3.seconds)
props(request, registry, requestor, timeout)
}
def props(
request: HydraRequest,
registry: ActorRef,
requestor: ActorRef,
timeout: FiniteDuration
): Props = {
Props(new DefaultIngestionHandler(request, registry, requestor, timeout))
}
}
Example 116
| Source File: KinesisProducerIntegrationSpec.scala From reactive-kinesis with Apache License 2.0 | 5 votes |
|
package com.weightwatchers.reactive.kinesis
import java.io.File
import com.amazonaws.services.kinesis.producer.{KinesisProducer => AWSKinesisProducer}
import com.typesafe.config.ConfigFactory
import com.weightwatchers.reactive.kinesis.common.{
KinesisSuite,
KinesisTestConsumer,
TestCredentials
}
import com.weightwatchers.reactive.kinesis.consumer.KinesisConsumer.ConsumerConf
import com.weightwatchers.reactive.kinesis.models.ProducerEvent
import com.weightwatchers.reactive.kinesis.producer.{KinesisProducer, ProducerConf}
import org.scalatest.concurrent.Eventually
import org.scalatest.mockito.MockitoSugar
import org.scalatest.time.{Millis, Seconds, Span}
import org.scalatest.{BeforeAndAfterAll, FreeSpec, Matchers}
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.util.Random
//scalastyle:off magic.number
class KinesisProducerIntegrationSpec
extends FreeSpec
with Matchers
with MockitoSugar
with BeforeAndAfterAll
with Eventually
with KinesisSuite {
implicit val ece = scala.concurrent.ExecutionContext.global
val TestStreamNrOfMessagesPerShard: Long = 0
implicit override val patienceConfig: PatienceConfig =
PatienceConfig(timeout = Span(5, Seconds), interval = Span(100, Millis))
"The KinesisProducer" - {
"Should publish a message to a stream" in new withKinesisConfForApp(
"int-test-stream-producer-1"
) {
val conf = producerConf()
val producer = KinesisProducer(conf)
val existingRecordCount = testConsumer.retrieveRecords(conf.streamName, 10).size
val event = ProducerEvent("1234", Random.alphanumeric.take(10).mkString)
producer.addUserRecord(event)
eventually {
val records: Seq[String] = testConsumer.retrieveRecords(conf.streamName, 10)
records.size shouldBe (existingRecordCount + 1)
records should contain(
new String(event.payload.array(), java.nio.charset.StandardCharsets.UTF_8)
)
}
}
}
}
//scalastyle:on
Example 117
| Source File: StringHelperTrees.scala From sbt-actor-api with MIT License | 5 votes |
|
package im.actor.api
import im.actor.api.Categories.Danger
import treehugger.forest._, definitions._
import treehuggerDSL._
import scala.language.postfixOps
trait StringHelperTrees extends TreeHelpers with Hacks {
private val DANGER_NOTE = DocTag.Note("Contains sensitive data!!!")
protected def generateDoc(doc: Doc): Vector[DocElement] = {
val attrDocs = doc.attributeDocs map { aDoc ⇒
val nameDesc = List(hackAttributeName(aDoc.argument), aDoc.description)
val args = if (aDoc.category == Danger) nameDesc :+ DANGER_NOTE else nameDesc
DocTag.Param(args: _*)
}
DocText(doc.generalDoc) +: attrDocs
}
protected val stringHelpersTree: Tree = OBJECTDEF("StringHelpers") withFlags PRIVATEWITHIN("api") := BLOCK(
DEF("escapeSpecial", StringClass).withParams(PARAM("source", StringClass)) :=
REF("source") DOT "flatMap" APPLY BLOCK(
CASE(
ID("c"),
IF((REF("c") ANY_== LIT('\n')) OR (REF("c") ANY_== LIT('\r')) OR (REF("c") ANY_== LIT('\t')) OR (REF("c") ANY_== LIT('\f')))
) ==> INTERP("s", LIT("\\"), REF("c")),
CASE(
ID("c"),
IF(REF("c") ANY_== LIT('"'))
) ==> LIT("\\\""),
CASE(ID("c")) ==> REF("c") TOSTRING
)
)
protected def generateToString(name: String, attributes: Vector[Attribute], attribureDocs: Vector[AttributeDoc]): DefDef = {
DEF("toString", StringClass) withFlags Flags.OVERRIDE := IF(REF("Debug") DOT "isDebug") THEN
INFIX_CHAIN("+", debug(name, attributes)) ELSE INFIX_CHAIN("+", prod(name, attributes, attribureDocs))
}
private def prod(name: String, attributes: Vector[Attribute], attribureDocs: Vector[AttributeDoc]) = {
val docMapped = attribureDocs.map(e ⇒ e.argument → e).toMap
val body = className(name) +: (attributes foldLeft Vector.empty[Vector[Tree]]) { (acc, elem) ⇒
val show = docMapped.get(elem.name) map (ad ⇒ Categories.Category.isVisible(ad.category)) getOrElse true
if (show) acc :+ ((wrapQoutes(LIT(elem.name)) :+ LIT(": ")) ++ wrapAttr(elem.typ, elem.name)) else acc
}
jsonTree(flattenTree(body))
}
private def debug(name: String, attributes: Vector[Attribute]) = {
val body = className(name) +: (attributes foldLeft Vector.empty[Vector[Tree]]) { (acc, elem) ⇒
acc :+ ((wrapQoutes(LIT(elem.name)) :+ LIT(": ")) ++ wrapAttr(elem.typ, elem.name))
}
jsonTree(flattenTree(body))
}
private def jsonTree(body: Vector[Tree]): Vector[Tree] = (LIT("{ ") +: body) :+ LIT(" }")
private def flattenTree(elements: Vector[Vector[Tree]]): Vector[Tree] = elements match {
case Vector() ⇒ Vector.empty[Tree]
case v ⇒ v reduce ((a, b) ⇒ (a :+ LIT(",")) ++ b)
}
private def className(name: String): Vector[Tree] =
(wrapQoutes(LIT("_name")) :+ LIT(": ")) ++ wrapQoutes(LIT(name))
private def wrapQoutes(e: Tree): Vector[Tree] = Vector(LIT("\""), e, LIT("\""))
private def wrapAttr(attrType: Types.AttributeType, attrName: String): Vector[Tree] = {
attrType match {
case Types.Int32 | Types.Int64 | Types.Bool | Types.Double | Types.Bytes ⇒ Vector(REF(attrName))
case Types.String ⇒ wrapQoutes(REF("StringHelpers") DOT "escapeSpecial" APPLY REF(attrName))
case Types.Enum(_) ⇒ Vector(REF(attrName) DOT "id")
case Types.Opt(typ) ⇒
val optMapToString = REF(attrName) MAP LAMBDA(PARAM("e")) ==> INFIX_CHAIN("+", wrapAttr(typ, "e"))
Vector(optMapToString DOT "getOrElse" APPLY NULL)
case Types.List(typ) ⇒
val listMapToString =
REF(attrName) MAP LAMBDA(PARAM("e")) ==> INFIX_CHAIN("+", wrapAttr(typ, "e")) DOT "mkString" APPLY LIT(", ")
Vector(LIT("[ "), listMapToString, LIT(" ]"))
case Types.Struct(_) | Types.Trait(_) ⇒ Vector(REF(attrName) TOSTRING)
case Types.Alias(aliasName) ⇒ wrapAttr(aliasesPrim.get(aliasName).get, attrName)
case _ ⇒ Vector(REF(attrName))
}
}
}
Example 118
| Source File: RichScheduledExecutorService.scala From mango with Apache License 2.0 | 5 votes |
|
package com.kakao.mango.concurrent
import java.util.concurrent.{Callable, ScheduledFuture, TimeUnit, ScheduledExecutorService}
import scala.concurrent.duration.Duration
import scala.concurrent.duration._
import scala.language.postfixOps
class RichScheduledExecutorService(underlying: ScheduledExecutorService) extends RichExecutorService(underlying) with ScheduledExecutorService {
def scheduleIn[T](delay: Duration)(command: => T): ScheduledFuture[T] = schedule(new Callable[T] {
override def call(): T = command
}, delay.toMillis, TimeUnit.MILLISECONDS)
def withFixedRate[T](rate: Duration, initialDelay: Duration = 0.second)(command: => Unit) = scheduleAtFixedRate(new Runnable {
override def run(): Unit = command
}, initialDelay.toMillis, rate.toMillis, TimeUnit.MILLISECONDS)
def withFixedDelay[T](delay: Duration, initialDelay: Duration = 0.second)(command: => Unit) = scheduleWithFixedDelay(new Runnable {
override def run(): Unit = command
}, initialDelay.toMillis, delay.toMicros, TimeUnit.MILLISECONDS)
// delegating to underlying
override def schedule(command: Runnable, delay: Long, unit: TimeUnit): ScheduledFuture[_] = underlying.schedule(wrap(command), delay, unit)
override def scheduleAtFixedRate(command: Runnable, initialDelay: Long, period: Long, unit: TimeUnit): ScheduledFuture[_] = underlying.scheduleAtFixedRate(wrap(command), initialDelay, period, unit)
override def schedule[V](callable: Callable[V], delay: Long, unit: TimeUnit): ScheduledFuture[V] = underlying.schedule(wrap(callable), delay, unit)
override def scheduleWithFixedDelay(command: Runnable, initialDelay: Long, delay: Long, unit: TimeUnit): ScheduledFuture[_] = underlying.scheduleWithFixedDelay(wrap(command), initialDelay, delay, unit)
}
Example 119
| Source File: TransactionDBSpec.scala From zio-rocksdb with Apache License 2.0 | 5 votes |
|
package zio.rocksdb
import java.nio.charset.StandardCharsets.UTF_8
import org.{ rocksdb => jrocks }
import zio.duration._
import zio.rocksdb.internal.internal.ManagedPath
import zio.test.Assertion._
import zio.test.TestAspect._
import zio.test._
import zio.{ console, RIO, ZIO }
import scala.language.postfixOps
object TransactionDBSpec extends DefaultRunnableSpec {
override def spec = {
val rocksSuite = suite("TransactionDB")(
testM("get/put") {
val key = "key".getBytes(UTF_8)
val value = "value".getBytes(UTF_8)
for {
_ <- TransactionDB.put(key, value)
result <- TransactionDB.get(key)
} yield assert(result)(isSome(equalTo(value)))
},
testM("delete") {
val key = "key".getBytes(UTF_8)
val value = "value".getBytes(UTF_8)
for {
_ <- TransactionDB.put(key, value)
before <- TransactionDB.get(key)
_ <- TransactionDB.delete(key)
after <- TransactionDB.get(key)
} yield assert(before)(isSome(equalTo(value))) && assert(after)(isNone)
},
testM("newIterator") {
val data = (1 to 10).map(i => (s"key$i", s"value$i")).toList
for {
_ <- RIO.foreach_(data) { case (k, v) => TransactionDB.put(k.getBytes(UTF_8), v.getBytes(UTF_8)) }
results <- TransactionDB.newIterator.runCollect
resultsStr = results.map { case (k, v) => new String(k, UTF_8) -> new String(v, UTF_8) }
} yield assert(resultsStr)(hasSameElements(data))
},
testM("get/put with Console") {
val key = "key".getBytes(UTF_8)
val value = "value".getBytes(UTF_8)
for {
result <- TransactionDB.atomically(for {
_ <- Transaction.put(key, value)
result <- Transaction.get(key)
_ <- console.putStrLn(result.toString)
} yield result)
} yield assert(result)(isSome(equalTo(value)))
},
testM("concurrent updates to the same key") {
val count = 10
val key = "COUNT".getBytes(UTF_8)
val expected = isSome(equalTo(count))
for {
_ <- TransactionDB.put(key, 0.toString.getBytes(UTF_8))
_ <- concurrent(count) {
TransactionDB.atomically {
Transaction.getForUpdate(key, exclusive = true) >>= { iCount =>
Transaction.put(key, iCount.map(bytesToInt).map(_ + 1).getOrElse(-1).toString.getBytes(UTF_8))
}
}
}
actual <- TransactionDB.get(key)
} yield assert(actual.map(bytesToInt))(expected)
},
testM("thread safety inside transaction") {
checkM(byteArray, byteArray) { (b1, b2) =>
for {
_ <- TransactionDB.atomically {
Transaction.put(b1, b2) <&> Transaction.put(b2, b1)
}
} yield assertCompletes
}
} @@ nonFlaky(10)
) @@ timeout(5 second)
rocksSuite.provideCustomLayerShared(database)
}
private def bytesToInt(bytes: Array[Byte]): Int = new String(bytes, UTF_8).toInt
private def concurrent[R, E, A](n: Int)(zio: ZIO[R, E, A]): ZIO[R, E, List[A]] = ZIO.foreachPar(0 until n)(_ => zio)
private val database = (for {
dir <- ManagedPath()
db <- {
val opts = new jrocks.Options().setCreateIfMissing(true)
TransactionDB.Live.open(opts, dir.toAbsolutePath.toString)
}
} yield db).toLayer.mapError(TestFailure.die)
private def byteArray = Gen.listOf(Gen.anyByte).map(_.toArray)
}
Example 120
| Source File: ParameterValues.scala From rug with GNU General Public License v3.0 | 5 votes |
|
package com.atomist.param
import scala.language.postfixOps
trait ParameterValues {
@throws[IllegalArgumentException]
def stringParamValue(name: String): String = paramValue(name) match {
case null => null
case s: String => s.toString
case _ => throw new IllegalArgumentException(s"'$name' is not a String")
}
override def toString: String = {
val parms = new StringBuilder()
parameterValues.foreach(p => parms.append(s"${p.getName} -> ${p.getValue}\n"))
parms.toString()
}
}
Example 121
| Source File: TestApplication.scala From vat-api with Apache License 2.0 | 5 votes |
|
package uk.gov.hmrc.vatapi
import com.github.tomakehurst.wiremock.WireMockServer
import com.github.tomakehurst.wiremock.client.WireMock
import com.github.tomakehurst.wiremock.client.WireMock._
import com.github.tomakehurst.wiremock.core.WireMockConfiguration._
import com.github.tomakehurst.wiremock.stubbing.StubMapping
import org.scalamock.scalatest.MockFactory
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import play.api.http.Status._
import scala.concurrent.duration._
import scala.language.postfixOps
trait TestApplication
extends UnitSpec
with BeforeAndAfterEach
with BeforeAndAfterAll
with MockFactory {
override implicit val timeout: FiniteDuration = 100 seconds
val mockPort = 22222
val mockHost = "localhost"
protected val wiremockBaseUrl: String = s"http://$mockHost:$mockHost"
private val wireMockServer = new WireMockServer(wireMockConfig().port(mockPort))
protected def baseBeforeAll(): StubMapping = {
wireMockServer.stop()
wireMockServer.start()
WireMock.configureFor(mockHost, mockPort)
// the below stub is here so that the application finds the registration endpoint which is called on startup
stubFor(post(urlPathEqualTo("/registration")).willReturn(aResponse().withStatus(OK)))
}
override def beforeAll(): Unit = {
super.beforeAll()
baseBeforeAll()
}
override def afterAll(): Unit = {
super.afterAll()
wireMockServer.stop()
}
override def beforeEach(): Unit = {
super.beforeEach()
WireMock.reset()
}
}
Example 122
| Source File: UnitSpec.scala From vat-api with Apache License 2.0 | 5 votes |
|
package uk.gov.hmrc.vatapi
import org.joda.time.{DateTime, DateTimeZone, LocalDate}
import org.scalatest.OptionValues
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.{AnyWordSpec, AsyncWordSpec}
import play.api.test.{DefaultAwaitTimeout, FutureAwaits}
import scala.concurrent.duration._
import scala.concurrent.{Await, Future}
import scala.language.postfixOps
trait BaseUnitSpec extends Matchers with OptionValues with TestUtils with FutureAwaits
with DefaultAwaitTimeout {
implicit val timeout: FiniteDuration = 5 seconds
def await[T](f: Future[T])(implicit duration: FiniteDuration = timeout): T =
Await.result(f, duration)
}
trait UnitSpec extends AnyWordSpec with BaseUnitSpec{
implicit def extractAwait[A](future: Future[A]): A = await[A](future)
def await[A](future: Future[A])(implicit timeout: Duration): A = Await.result(future, timeout)
}
trait AsyncUnitSpec extends AsyncWordSpec with BaseUnitSpec
trait TestUtils {
private val vrnGenerator = VrnGenerator()
def now: DateTime = DateTime.now(DateTimeZone.UTC)
def generateVrn = vrnGenerator.nextVrn()
implicit def toLocalDate(d: DateTime): LocalDate = d.toLocalDate
}
object TestUtils extends TestUtils
Example 123
| Source File: GuideDefaultStyles.scala From udash-core with Apache License 2.0 | 5 votes |
|
package io.udash.web.guide.styles
import io.udash.css.CssBase
import io.udash.web.commons.styles.DefaultStyles
import io.udash.web.commons.styles.utils.{FontWeight, MediaQueries, UdashFonts}
import scala.language.postfixOps
object GuideDefaultStyles extends CssBase with DefaultStyles {
import dsl._
style(
unsafeRoot("body") (
fontSize(1 rem)
),
unsafeRoot("pre") (
backgroundColor(c"#f5f5f5"),
overflow.auto
),
unsafeRoot("p")(
marginTop(1.5625 rem),
fontSize(1 rem),
lineHeight(1.3),
&.firstChild (
marginTop(`0`)
)
),
unsafeRoot("h3") (
UdashFonts.roboto(FontWeight.Thin),
marginTop(2.8125 rem),
marginBottom(.9375 rem),
lineHeight(1.2),
fontSize(2 rem),
MediaQueries.phone(
fontSize(1.625 rem)
)
),
unsafeRoot("h4") (
UdashFonts.roboto(FontWeight.Thin),
marginTop(2.1875 rem),
marginBottom(.9375 rem),
lineHeight(1.2),
fontSize(1.5 rem),
MediaQueries.phone(
fontSize(1.25 rem)
)
)
)
}
Example 124
| Source File: MediaQueries.scala From udash-core with Apache License 2.0 | 5 votes |
|
package io.udash.web.guide.styles.utils
import io.udash.css.{CssBase, CssStyle}
import io.udash.web.commons.styles.utils.StyleConstants
import scalacss.internal.DslBase.ToStyle
import scala.language.postfixOps
object MediaQueries extends CssBase {
import dsl._
def desktop(properties: ToStyle*): CssStyle = mixin(
media.screen.minWidth(StyleConstants.MediaQueriesBounds.TabletLandscapeMax + 1 px)(
properties: _*
)
)
def tabletLandscape(properties: ToStyle*): CssStyle = mixin(
media.screen.minWidth(1 px).maxWidth(StyleConstants.MediaQueriesBounds.TabletLandscapeMax px) (
properties:_*
)
)
def tabletPortrait(properties: ToStyle*): CssStyle = mixin(
media.screen.minWidth(1 px).maxWidth(StyleConstants.MediaQueriesBounds.TabletMax px) (
properties:_*
)
)
def phone(properties: ToStyle*): CssStyle = mixin(
media.screen.minWidth(1 px).maxWidth(StyleConstants.MediaQueriesBounds.PhoneMax px) (
properties:_*
)
)
}
Example 125
| Source File: GuideStyleUtils.scala From udash-core with Apache License 2.0 | 5 votes |
|
package io.udash.web.guide.styles.utils
import io.udash.css.{CssBase, CssStyle}
import io.udash.web.commons.styles.utils.StyleConstants
import scalacss.internal.Macros.Color
import scalacss.internal.{AV, Attr, Length}
import scala.concurrent.duration.{DurationInt, FiniteDuration}
import scala.language.postfixOps
object GuideStyleUtils extends CssBase {
import dsl._
val relativeMiddle: CssStyle = mixin(
top(50 %%),
transform := "translateY(-50%)",
position.relative
)
def transition(property: Attr = all, duration: FiniteDuration = 250 milliseconds): CssStyle = mixin(
transitionProperty := property.toString(),
transitionDuration(duration),
transitionTimingFunction.easeInOut
)
def border(bColor: Color = StyleConstants.Colors.GreyExtra, bWidth: Length[Double] = 1.0 px, bStyle: AV = borderStyle.solid): CssStyle = mixin(
borderWidth(bWidth),
bStyle,
borderColor(bColor)
)
}
Example 126
| Source File: HeaderStyles.scala From udash-core with Apache License 2.0 | 5 votes |
|
package io.udash.web.guide.styles.partials
import io.udash.css.{CssBase, CssStyle}
import io.udash.web.commons.styles.components.{HeaderButtonsStyles, HeaderNavStyles}
import io.udash.web.commons.styles.utils.StyleConstants
import io.udash.web.guide.styles.utils.{GuideStyleUtils, MediaQueries}
import scala.language.postfixOps
object HeaderStyles extends CssBase with HeaderButtonsStyles with HeaderNavStyles {
import dsl._
val header: CssStyle = style(
position.relative,
backgroundColor.black,
height(StyleConstants.Sizes.HeaderHeight px),
fontSize(1 rem),
zIndex(99),
position.fixed,
top(`0`),
width(100 %%),
MediaQueries.tabletPortrait(
height(StyleConstants.Sizes.GuideHeaderHeightMobile px)
)
)
val headerLeft: CssStyle = style(
position.relative,
float.left,
height(100 %%)
)
val headerLogo: CssStyle = style(
GuideStyleUtils.relativeMiddle,
display.inlineBlock,
verticalAlign.top,
width(130 px),
marginRight(25 px),
MediaQueries.tabletLandscape(
marginLeft(StyleConstants.Sizes.GuideHeaderHeightMobile px)
),
MediaQueries.tabletPortrait(
width(130 * .8 px)
)
)
}
Example 127
| Source File: MarkdownStyles.scala From udash-core with Apache License 2.0 | 5 votes |
|
package io.udash.web.guide.styles
import io.udash.css.{CssBase, CssStyle}
import scala.language.postfixOps
object MarkdownStyles extends CssBase {
import dsl._
val markdownPage: CssStyle = style(
unsafeChild("li") (
position.relative,
paddingLeft(2 rem),
margin(.5 rem, `0`, .5 rem, 4.5 rem),
&.before(
position.absolute,
left(`0`),
top(`0`),
content := "\"•\""
)
),
unsafeChild("iframe")(
marginTop(2.5 rem),
&.firstChild (
marginTop(`0`)
)
),
unsafeChild("pre")(
marginTop(2.5 rem),
&.firstChild (
marginTop(`0`)
)
),
unsafeChild("code") (
backgroundColor(c"#f5f2f0"),
fontFamily :=! "Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace"
),
)
}
Example 128
| Source File: MediaQueries.scala From udash-core with Apache License 2.0 | 5 votes |
|
package io.udash.web.commons.styles.utils
import io.udash.css.{CssBase, CssStyle}
import scalacss.internal.DslBase.ToStyle
import scala.language.postfixOps
object MediaQueries extends CssBase {
import dsl._
def desktop(properties: ToStyle*): CssStyle = mixin(
media.screen.minWidth(StyleConstants.MediaQueriesBounds.TabletLandscapeMax + 1 px) (
properties:_*
)
)
def tabletLandscape(properties: ToStyle*): CssStyle = mixin(
media.screen.minWidth(1 px).maxWidth(StyleConstants.MediaQueriesBounds.TabletLandscapeMax px) (
properties:_*
)
)
def tabletPortrait(properties: ToStyle*): CssStyle = mixin(
media.screen.minWidth(1 px).maxWidth(StyleConstants.MediaQueriesBounds.TabletMax px) (
properties:_*
)
)
def phone(properties: ToStyle*): CssStyle = mixin(
media.screen.minWidth(1 px).maxWidth(StyleConstants.MediaQueriesBounds.PhoneMax px) (
properties:_*
)
)
}
Example 129
| Source File: CommonStyleUtils.scala From udash-core with Apache License 2.0 | 5 votes |
|
package io.udash.web.commons.styles.utils
import io.udash.css.{CssBase, CssStyle}
import scalacss.internal.Macros.Color
import scalacss.internal.{AV, Attr, Length}
import scala.concurrent.duration.{DurationInt, FiniteDuration}
import scala.language.postfixOps
object CommonStyleUtils extends CssBase {
import dsl._
val middle: CssStyle = mixin(
top(50 %%),
transform := "translateY(-50%)"
)
val center: CssStyle = mixin(
top(50 %%),
left(50 %%),
transform := "translateY(-50%) translateX(-50%)"
)
val relativeMiddle: CssStyle = mixin(
middle,
position.relative
)
val absoluteMiddle: CssStyle = mixin(
middle,
position.absolute
)
val absoluteCenter: CssStyle = mixin(
center,
position.absolute
)
def transition(property: Attr = all, duration: FiniteDuration = 250 milliseconds): CssStyle = style(
transitionProperty := property.toString(),
transitionDuration(duration),
transitionTimingFunction.easeInOut
)
def border(bColor: Color = StyleConstants.Colors.GreyExtra, bWidth: Length[Double] = 1.0 px, bStyle: AV = borderStyle.solid): CssStyle = style(
borderWidth(bWidth),
bStyle,
borderColor(bColor)
)
}
Example 130
| Source File: MobileMenuStyles.scala From udash-core with Apache License 2.0 | 5 votes |
|
package io.udash.web.commons.styles.components
import io.udash.css.{CssBase, CssStyle}
import io.udash.web.commons.styles.attributes.Attributes
import io.udash.web.commons.styles.utils.{CommonStyleUtils, MediaQueries, StyleConstants}
import scala.language.postfixOps
object MobileMenuStyles extends CssBase {
import dsl._
private val lineHeight = 4
val btnMobileLines: CssStyle = style(
CommonStyleUtils.absoluteCenter,
width(60 %%)
)
private val btnMobileLine: CssStyle = mixin(
CommonStyleUtils.transition(),
position.relative,
display.block,
height(lineHeight px),
width(100 %%),
marginTop(lineHeight px),
marginBottom(lineHeight px),
backgroundColor(StyleConstants.Colors.Red)
)
val btnMobileLineTop: CssStyle = style(
btnMobileLine,
transformOrigin := s"50% calc(50% + ${lineHeight * 2}px)"
)
val btnMobileLineMiddle: CssStyle = style(
btnMobileLine
)
val btnMobileLineBottom: CssStyle = style(
btnMobileLine,
transformOrigin := s"50% calc(50% - ${lineHeight * 2}px)"
)
private val btnMobileActive = mixin(
unsafeChild(s".${btnMobileLineTop.className}") (
transform := s"rotate(45deg) translateY(${lineHeight * 2}px)"
),
unsafeChild(s".${btnMobileLineBottom.className}") (
transform := s"rotate(-45deg) translateY(-${lineHeight * 2}px)"
),
unsafeChild(s".${btnMobileLineMiddle.className}") (
opacity(0)
)
)
val btnMobile: CssStyle = style(
display.none,
width(StyleConstants.Sizes.GuideHeaderHeightMobile + 1 px),
height(StyleConstants.Sizes.GuideHeaderHeightMobile px),
zIndex(9),
MediaQueries.tabletPortrait(
display.inlineBlock,
verticalAlign.middle
),
&.attr(Attributes.data(Attributes.Active), "true") (
btnMobileActive
)
)
}
Example 131
| Source File: HomepageDefaultStyles.scala From udash-core with Apache License 2.0 | 5 votes |
|
package io.udash.web.homepage.styles
import io.udash.css.CssBase
import io.udash.web.commons.styles.DefaultStyles
import io.udash.web.commons.styles.utils.MediaQueries
import scala.language.postfixOps
object HomepageDefaultStyles extends CssBase with DefaultStyles {
import dsl._
style(
unsafeRoot("body") (
fontSize(1.0625 rem)
),
unsafeRoot("p")(
fontSize(1 rem)
),
unsafeRoot("h1") (
marginBottom(3.125 rem),
MediaQueries.phone(
paddingTop(3.125 rem),
marginBottom(1.875 rem)
)
)
)
}
Example 132
| Source File: HeaderStyles.scala From udash-core with Apache License 2.0 | 5 votes |
|
package io.udash.web.homepage.styles.partials
import io.udash.css.{CssBase, CssStyle}
import io.udash.web.commons.styles.attributes.Attributes
import io.udash.web.commons.styles.components.{HeaderButtonsStyles, HeaderNavStyles}
import io.udash.web.commons.styles.utils.{CommonStyleUtils, MediaQueries, StyleConstants}
import scalacss.internal.Literal
import scala.concurrent.duration.DurationInt
import scala.language.postfixOps
object HeaderStyles extends CssBase with HeaderButtonsStyles with HeaderNavStyles {
import dsl._
val header: CssStyle = style(
position.absolute,
top(`0`),
left(`0`),
width(100 %%),
height(StyleConstants.Sizes.LandingPageHeaderHeight px),
fontSize(1 rem),
zIndex(999),
&.attr(Attributes.data(Attributes.Pinned), "true")(
position.fixed,
height(StyleConstants.Sizes.HeaderHeightPin px),
backgroundColor.black,
animationName(headerAnimation),
animationIterationCount.count(1),
animationDuration(300 milliseconds),
MediaQueries.tabletLandscape(
height(StyleConstants.Sizes.HeaderHeightPin * .85 px)
),
unsafeChild(s".${headerLogo.className}")(
width(48 px),
height(56 px),
backgroundImage := "url(/assets/images/udash_logo.png)",
MediaQueries.tabletPortrait(
display.none
)
),
unsafeChild(s".${btnMobile.className}")(
CommonStyleUtils.middle
)
),
MediaQueries.tabletPortrait(
height(StyleConstants.Sizes.HeaderHeight * .9 px)
)
)
private lazy val headerAnimation: CssStyle = keyframes(
0d -> keyframe(
transform := "translateY(-100%)"
),
100d -> keyframe(
transform := "translateY(0)"
)
)
val headerLeft: CssStyle = style(
position.relative,
float.left,
height(100 %%)
)
lazy val headerLogo: CssStyle = style(
CommonStyleUtils.relativeMiddle,
display.inlineBlock,
verticalAlign.middle,
width(65 px),
height(96 px),
marginRight(25 px),
backgroundImage := "url(/assets/images/udash_logo_l.png)",
backgroundRepeat.noRepeat,
backgroundSize := "100%",
MediaQueries.tabletPortrait(
display.block,
width(StyleConstants.Sizes.GuideHeaderHeightMobile px),
height(14 px),
backgroundPosition := Literal.bottom,
transform := none,
top.auto
)
)
lazy val btnMobile: CssStyle = style(
position.relative
)
}
Example 133
| Source File: DependencyGraphReadyCheckSpec.scala From docker-it-scala with MIT License | 5 votes |
|
package com.whisk.docker
import com.whisk.docker.impl.spotify.DockerKitSpotify
import org.scalatest.{FlatSpec, Matchers}
import org.slf4j.LoggerFactory
import scala.concurrent.Await
import scala.concurrent.duration._
import scala.language.postfixOps
class DependencyGraphReadyCheckSpec extends FlatSpec with Matchers with DockerKitSpotify {
override val StartContainersTimeout = 45 seconds
private lazy val log = LoggerFactory.getLogger(this.getClass)
val zookeeperContainer =
DockerContainer("confluentinc/cp-zookeeper:3.1.2", name = Some("zookeeper"))
.withEnv("ZOOKEEPER_TICK_TIME=2000", "ZOOKEEPER_CLIENT_PORT=2181")
.withReadyChecker(DockerReadyChecker.LogLineContains("binding to port"))
val kafkaContainer = DockerContainer("confluentinc/cp-kafka:3.1.2", name = Some("kafka"))
.withEnv("KAFKA_BROKER_ID=1",
"KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181",
"KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092")
.withLinks(ContainerLink(zookeeperContainer, "zookeeper"))
.withReadyChecker(DockerReadyChecker.LogLineContains("[Kafka Server 1], started"))
val schemaRegistryContainer = DockerContainer("confluentinc/cp-schema-registry:3.1.2",
name = Some("schema_registry"))
.withEnv("SCHEMA_REGISTRY_HOST_NAME=schema_registry",
"SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181")
.withLinks(ContainerLink(zookeeperContainer, "zookeeper"),
ContainerLink(kafkaContainer, "kafka"))
.withReadyChecker(DockerReadyChecker.LogLineContains("Server started, listening for requests"))
override def dockerContainers =
schemaRegistryContainer :: kafkaContainer :: zookeeperContainer :: super.dockerContainers
"all containers except the leaves of the dependency graph" should "be ready after initialization" in {
startAllOrFail()
try {
containerManager.isReady(zookeeperContainer).isCompleted shouldBe true
containerManager.isReady(kafkaContainer).isCompleted shouldBe true
containerManager.isReady(schemaRegistryContainer).isCompleted shouldBe false
Await.ready(containerManager.isReady(schemaRegistryContainer), 45 seconds)
containerManager.isReady(schemaRegistryContainer).isCompleted shouldBe true
} catch {
case e: RuntimeException => log.error("Test failed during readychecks", e)
} finally {
Await.ready(containerManager.stopRmAll(), StopContainersTimeout)
}
}
override def startAllOrFail(): Unit = {
Await.result(containerManager.pullImages(), PullImagesTimeout)
containerManager.initReadyAll(StartContainersTimeout).map(_.map(_._2).forall(identity))
sys.addShutdownHook(
containerManager.stopRmAll()
)
}
}
Example 134
| Source File: ActivationStoreBehaviorBase.scala From openwhisk with Apache License 2.0 | 5 votes |
|
package org.apache.openwhisk.core.database.test.behavior
import java.time.Instant
import akka.stream.ActorMaterializer
import common.{StreamLogging, WskActorSystem}
import org.apache.openwhisk.common.TransactionId
import org.apache.openwhisk.core.database.{ActivationStore, CacheChangeNotification, UserContext}
import org.apache.openwhisk.core.database.test.behavior.ArtifactStoreTestUtil.storeAvailable
import org.apache.openwhisk.core.entity._
import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures}
import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers, Outcome}
import scala.collection.mutable.ListBuffer
import scala.concurrent.Await
import scala.concurrent.duration.Duration
import scala.concurrent.duration.DurationInt
import scala.language.postfixOps
import scala.util.{Random, Try}
trait ActivationStoreBehaviorBase
extends FlatSpec
with ScalaFutures
with Matchers
with StreamLogging
with WskActorSystem
with IntegrationPatience
with BeforeAndAfterEach {
protected implicit val materializer: ActorMaterializer = ActorMaterializer()
protected implicit val notifier: Option[CacheChangeNotification] = None
def context: UserContext
def activationStore: ActivationStore
private val docsToDelete = ListBuffer[(UserContext, ActivationId)]()
def storeType: String
protected def transId() = TransactionId(Random.alphanumeric.take(32).mkString)
override def afterEach(): Unit = {
cleanup()
stream.reset()
}
override protected def withFixture(test: NoArgTest): Outcome = {
assume(storeAvailable(storeAvailableCheck), s"$storeType not configured or available")
val outcome = super.withFixture(test)
if (outcome.isFailed) {
println(logLines.mkString("\n"))
}
outcome
}
protected def storeAvailableCheck: Try[Any] = Try(true)
//~----------------------------------------< utility methods >
protected def store(activation: WhiskActivation, context: UserContext)(
implicit transid: TransactionId,
notifier: Option[CacheChangeNotification]): DocInfo = {
val doc = activationStore.store(activation, context).futureValue
docsToDelete.append((context, ActivationId(activation.docid.asString)))
doc
}
protected def newActivation(ns: String, actionName: String, start: Long): WhiskActivation = {
WhiskActivation(
EntityPath(ns),
EntityName(actionName),
Subject(),
ActivationId.generate(),
Instant.ofEpochMilli(start),
Instant.ofEpochMilli(start + 1000))
}
def cleanup()(implicit timeout: Duration = 10 seconds): Unit = {
implicit val tid: TransactionId = transId()
docsToDelete.map { e =>
Try {
Await.result(activationStore.delete(e._2, e._1), timeout)
}
}
docsToDelete.clear()
}
}
Example 135
| Source File: LogLimit.scala From openwhisk with Apache License 2.0 | 5 votes |
|
package org.apache.openwhisk.core.entity
import pureconfig._
import pureconfig.generic.auto._
import scala.language.postfixOps
import scala.util.Failure
import scala.util.Success
import scala.util.Try
import spray.json._
import org.apache.openwhisk.core.ConfigKeys
import org.apache.openwhisk.core.entity.size._
case class LogLimitConfig(min: ByteSize, max: ByteSize, std: ByteSize)
@throws[IllegalArgumentException]
protected[core] def apply(megabytes: ByteSize): LogLimit = {
require(megabytes >= MIN_LOGSIZE, s"log size $megabytes below allowed threshold of $MIN_LOGSIZE")
require(megabytes <= MAX_LOGSIZE, s"log size $megabytes exceeds allowed threshold of $MAX_LOGSIZE")
new LogLimit(megabytes.toMB.toInt)
}
override protected[core] implicit val serdes = new RootJsonFormat[LogLimit] {
def write(m: LogLimit) = JsNumber(m.megabytes)
def read(value: JsValue) =
Try {
val JsNumber(mb) = value
require(mb.isWhole, "log limit must be whole number")
LogLimit(mb.intValue MB)
} match {
case Success(limit) => limit
case Failure(e: IllegalArgumentException) => deserializationError(e.getMessage, e)
case Failure(e: Throwable) => deserializationError("log limit malformed", e)
}
}
}
Example 136
| Source File: MemoryLimit.scala From openwhisk with Apache License 2.0 | 5 votes |
|
package org.apache.openwhisk.core.entity
import scala.language.postfixOps
import scala.util.Failure
import scala.util.Success
import scala.util.Try
import spray.json._
import org.apache.openwhisk.core.entity.size._
import org.apache.openwhisk.core.ConfigKeys
import pureconfig._
import pureconfig.generic.auto._
case class MemoryLimitConfig(min: ByteSize, max: ByteSize, std: ByteSize)
@throws[IllegalArgumentException]
protected[core] def apply(megabytes: ByteSize): MemoryLimit = {
require(megabytes >= MIN_MEMORY, s"memory $megabytes below allowed threshold of $MIN_MEMORY")
require(megabytes <= MAX_MEMORY, s"memory $megabytes exceeds allowed threshold of $MAX_MEMORY")
new MemoryLimit(megabytes.toMB.toInt)
}
override protected[core] implicit val serdes = new RootJsonFormat[MemoryLimit] {
def write(m: MemoryLimit) = JsNumber(m.megabytes)
def read(value: JsValue) =
Try {
val JsNumber(mb) = value
require(mb.isWhole, "memory limit must be whole number")
MemoryLimit(mb.intValue MB)
} match {
case Success(limit) => limit
case Failure(e: IllegalArgumentException) => deserializationError(e.getMessage, e)
case Failure(e: Throwable) => deserializationError("memory limit malformed", e)
}
}
}
Example 137
| Source File: Entities.scala From openwhisk with Apache License 2.0 | 5 votes |
|
package org.apache.openwhisk.core.controller
import scala.concurrent.Future
import scala.language.postfixOps
import scala.util.Try
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
import akka.http.scaladsl.model.StatusCodes.PayloadTooLarge
import akka.http.scaladsl.model.headers.RawHeader
import akka.http.scaladsl.server.Directive0
import akka.http.scaladsl.server.Directives
import akka.http.scaladsl.server.RequestContext
import akka.http.scaladsl.server.Route
import akka.http.scaladsl.server.RouteResult
import spray.json.JsonPrinter
import org.apache.openwhisk.common.TransactionId
import org.apache.openwhisk.core.entitlement.Privilege._
import org.apache.openwhisk.core.entitlement.Privilege
import org.apache.openwhisk.core.entitlement.Privilege.READ
import org.apache.openwhisk.core.entitlement.Resource
import org.apache.openwhisk.core.entity._
import org.apache.openwhisk.core.entity.ActivationEntityLimit
import org.apache.openwhisk.core.entity.size._
import org.apache.openwhisk.http.ErrorResponse.terminate
import org.apache.openwhisk.http.Messages
protected[controller] trait ValidateRequestSize extends Directives {
protected def validateSize(check: => Option[SizeError])(implicit tid: TransactionId, jsonPrinter: JsonPrinter) =
new Directive0 {
override def tapply(f: Unit => Route) = {
check map {
case e: SizeError => terminate(PayloadTooLarge, Messages.entityTooBig(e))
} getOrElse f(None)
}
}
protected final def isEntity(n: String) = Try { EntityName(n) } isSuccess
}
Example 138
| Source File: MultiDataStreamer.scala From structured-streaming-application with Apache License 2.0 | 5 votes |
|
package knolx.kafka
import java.util.Properties
import akka.actor.ActorSystem
import knolx.Config.{bootstrapServer, topic}
import knolx.KnolXLogger
import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord}
import org.apache.kafka.common.serialization.StringSerializer
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.DurationInt
import scala.language.postfixOps
import scala.util.Random
object MultiDataStreamer extends App with KnolXLogger {
val system = ActorSystem("DataStreamer")
val props = new Properties()
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServer)
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer].getName)
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer].getName)
val producer = new KafkaProducer[String, String](props)
info("Streaming data into Kafka...")
system.scheduler.schedule(0 seconds, 3000 milliseconds) {
(1 to Random.nextInt(100)).foreach { id =>
producer.send(new ProducerRecord[String, String](topic,s"device$id", (Math.random * 2 + 1).toString))
}
}
}
Example 139
| Source File: UsageSpec.scala From dependency with MIT License | 5 votes |
|
package controllers
import io.flow.test.utils.FlowPlaySpec
import io.flow.usage.util.UsageUtil
import io.flow.usage.v0.Client
import io.flow.usage.v0.models.json._
import play.api.libs.json.Json
import scala.concurrent.Await
import scala.concurrent.duration._
import scala.language.postfixOps
class UsageSpec extends FlowPlaySpec {
private[this] def uu = app.injector.instanceOf[UsageUtil]
import scala.concurrent.ExecutionContext.Implicits.global
"Check usage" in {
val j = Json.toJson(uu.currentUsage)
println(s"Found API Usage: $j" )
val r = Json.toJson(
Await.result(
new Client(
wsClient,
s"http://localhost:$port"
).Usages.getUsage(), 3 seconds
)
)
j must be(r)
}
}
Example 140
| Source File: CollectionTypes.scala From xmlrpc with MIT License | 5 votes |
|
package xmlrpc.protocol
import xmlrpc.protocol.Deserializer.Deserialized
import scala.xml.{NodeSeq, Node}
import scala.language.postfixOps
import scalaz.Scalaz._
trait CollectionTypes extends Protocol {
import Deserializer.StringToError
// We only support array of the same type, if an array contains elements with different
// types, we deserialize it with case classes
implicit def ArrayXmlrpc[T: Datatype]: Datatype[Seq[T]] = new Datatype[Seq[T]] {
override def serialize(value: Seq[T]): Node =
<array><data>{for {elem <- value} yield toXmlrpc(elem)}</data></array>.inValue
override def deserialize(from: NodeSeq): Deserialized[Seq[T]] =
from \\ "array" headOption match {
case Some(<array><data>{array @ _*}</data></array>) =>
(for { value <- array}
yield fromXmlrpc[T](value)).toList.sequence[Deserialized, T]
case _ => "Expected array structure in $from".toError.failures
}
}
implicit def StructXmlrpc[T: Datatype]: Datatype[Map[String, T]] = new Datatype[Map[String, T]] {
override def serialize(map: Map[String, T]): Node = {
def inName(name: String): Node = <name>{name}</name>
def inMember(elems: NodeSeq): NodeSeq = <member>{elems}</member>
lazy val struct: NodeSeq = (for {
(key, value) <- map
} yield inMember(inName(key) ++ toXmlrpc(value))).reduce(_ ++ _)
<struct>{struct}</struct>.inValue
}
override def deserialize(from: NodeSeq): Deserialized[Map[String, T]] =
from \\ "struct" headOption match {
case Some(<struct>{members @ _*}</struct>) =>
(for { member <- members }
yield fromXmlrpc[T](member \ "value" head) map ((member \ "name" text) -> _))
.toList
.sequence[Deserialized, (String, T)]
.map(_.toMap[String, T])
case _ => s"Expected struct in:\n$from".toError.failures
}
}
}
Example 141
| Source File: ScalaTypes.scala From xmlrpc with MIT License | 5 votes |
|
package xmlrpc.protocol
import xmlrpc.protocol.Deserializer.Deserialized
import scala.language.{postfixOps, implicitConversions}
import scala.xml.NodeSeq
import scalaz.Scalaz._
trait ScalaTypes extends Protocol {
implicit def optionXmlrpc[T: Datatype]: Datatype[Option[T]] = new Datatype[Option[T]] {
override def serialize(value: Option[T]): NodeSeq = value match {
case Some(a) => toXmlrpc[T](a)
case None => NodeSeq.Empty
}
override def deserialize(from: NodeSeq): Deserialized[Option[T]] =
from \\ "value" headOption match {
case Some(a) => fromXmlrpc[T](a) map (Some(_))
case None => None.success
}
}
}
Example 142
| Source File: XmlrpcConnection.scala From xmlrpc with MIT License | 5 votes |
|
package xmlrpc
import akka.actor.ActorSystem
import akka.stream.ActorMaterializer
import akka.util.Timeout
import org.scalatest.FunSpec
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.time.{Millis, Seconds, Span}
import xmlrpc.protocol.XmlrpcProtocol
import scala.concurrent.duration._
import scala.language.postfixOps
import scalaz.{Success, Failure}
class XmlrpcConnection extends FunSpec with ScalaFutures {
// Xmlrpc imports
import Xmlrpc._
import XmlrpcProtocol._
// Scalatest setup
implicit val default: PatienceConfig = PatienceConfig(timeout = Span(5, Seconds), interval = Span(500, Millis))
// Xmrpc setup, server is up but it is not mine, found on Internet
implicit val testServer = XmlrpcServer("http://betty.userland.com/RPC2")
// Spray setup
implicit val system = ActorSystem()
implicit val ma = ActorMaterializer()
implicit val timeout = Timeout(5 seconds)
import system.dispatcher
describe("The connection with a XML-RPC server") {
it("should invoke the test method successfully in the server") {
val invocation = invokeMethod[Int, String]("examples.getStateName", 41).underlying
val responseMessage = "South Dakota"
whenReady(invocation) {
case Success(value) => assertResult(responseMessage) {value}
case Failure(errors) => fail("Errors when deserializing\n" + errors)
}
}
}
}
Example 143
| Source File: EventHubsRelation.scala From azure-event-hubs-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.eventhubs
import org.apache.spark.eventhubs.rdd.{ EventHubsRDD, OffsetRange }
import org.apache.spark.internal.Logging
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{ Row, SQLContext }
import org.apache.spark.sql.sources.{ BaseRelation, TableScan }
import org.apache.spark.sql.types.StructType
import scala.language.postfixOps
private[eventhubs] class EventHubsRelation(override val sqlContext: SQLContext,
parameters: Map[String, String])
extends BaseRelation
with TableScan
with Logging {
import org.apache.spark.eventhubs._
private val ehConf = EventHubsConf.toConf(parameters)
private val eventHubClient = EventHubsSourceProvider.clientFactory(parameters)(ehConf)
override def schema: StructType = EventHubsSourceProvider.eventHubsSchema
override def buildScan(): RDD[Row] = {
val partitionCount: Int = eventHubClient.partitionCount
val fromSeqNos = eventHubClient.translate(ehConf, partitionCount)
val untilSeqNos = eventHubClient.translate(ehConf, partitionCount, useStart = false)
require(fromSeqNos.forall(f => f._2 >= 0L),
"Currently only sequence numbers can be passed in your starting positions.")
require(untilSeqNos.forall(u => u._2 >= 0L),
"Currently only sequence numbers can be passed in your ending positions.")
val offsetRanges = untilSeqNos.keySet.map { p =>
val fromSeqNo = fromSeqNos
.getOrElse(p, throw new IllegalStateException(s"$p doesn't have a fromSeqNo"))
val untilSeqNo = untilSeqNos(p)
OffsetRange(ehConf.name, p, fromSeqNo, untilSeqNo, None)
}.toArray
eventHubClient.close()
logInfo(
"GetBatch generating RDD of with offsetRanges: " +
offsetRanges.sortBy(_.nameAndPartition.toString).mkString(", "))
val rdd = EventHubsSourceProvider.toInternalRow(
new EventHubsRDD(sqlContext.sparkContext, ehConf.trimmed, offsetRanges))
sqlContext.internalCreateDataFrame(rdd, schema, isStreaming = false).rdd
}
}
Example 144
| Source File: FutureResultSupport.scala From sangria-slowlog with Apache License 2.0 | 5 votes |
|
package sangria.slowlog.util
import sangria.execution.{ErrorWithResolver, QueryAnalysisError}
import sangria.marshalling.ResultMarshallerForType
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
import scala.concurrent.{Await, ExecutionContext, Future}
import scala.language.postfixOps
trait FutureResultSupport {
implicit class FutureResult[T](f: Future[T]) {
def await = Await.result(f, 10 seconds)
def await(duration: Duration) = Await.result(f, duration)
def awaitAndRecoverQueryAnalysis(implicit m: ResultMarshallerForType[T]): T = Await.result(recoverQueryAnalysis, 10 seconds)
def recoverQueryAnalysis(implicit m: ResultMarshallerForType[T]): Future[T] = f.recover {
case analysisError: QueryAnalysisError => analysisError.resolveError(m.marshaller).asInstanceOf[T]
}
def awaitAndRecoverQueryAnalysisScala(implicit ev: T =:= Any) = Await.result(recoverQueryAnalysisScala, 10 seconds)
def recoverQueryAnalysisScala(implicit ev: T =:= Any) = f.recover {
case analysisError: ErrorWithResolver => analysisError.resolveError
}
}
object sync {
val executionContext = ExecutionContext.fromExecutor(new java.util.concurrent.Executor {
def execute(command: Runnable) = command.run()
})
}
}
Example 145
| Source File: DiscoveryClient.scala From temperature-machine with Apache License 2.0 | 5 votes |
|
package bad.robot.temperature.client
import java.net.{DatagramPacket, DatagramSocket, InetAddress, NetworkInterface, Socket => _}
import bad.robot.logging._
import bad.robot.temperature.server.DatagramPacketOps
import bad.robot.temperature.server.DiscoveryServer._
import bad.robot.temperature.server.Socket._
import scala.collection.JavaConverters._
import scala.concurrent.duration._
import scala.language.postfixOps
object DiscoveryClient {
def discover: InetAddress = {
val socket = new DatagramSocket()
socket.setBroadcast(true)
allBroadcastAddresses.foreach(ping(_, socket))
Log.info("Awaiting discovery server...")
socket.await(30 seconds).fold(error => {
Log.error(error.message)
retry
}, sender => {
sender.payload match {
case ServerAddressResponseMessage => sender.getAddress
case _ => retry
}
})
}
private def retry = {
Thread.sleep((30 seconds).toMillis)
discover
}
private def allNetworkInterfaces: List[NetworkInterface] = {
NetworkInterface.getNetworkInterfaces
.asScala
.toList
.filter(_.isUp)
.filterNot(_.isLoopback)
}
private val broadcastAddresses: (NetworkInterface) => List[InetAddress] = (interface) => {
interface.getInterfaceAddresses.asScala.toList.map(_.getBroadcast).filter(_ != null)
}
def allBroadcastAddresses: List[InetAddress] = {
InetAddress.getByName(LocalNetworkBroadcastAddress) :: allNetworkInterfaces.flatMap(broadcastAddresses)
}
def ping: (InetAddress, DatagramSocket) => Unit = (address, socket) => {
try {
val data = ServerAddressRequestMessage.getBytes
Log.info(s"Sending ping request to $address")
socket.send(new DatagramPacket(data, data.length, address, ServerPort))
} catch {
case e: Throwable => Log.error(s"An error occurred pinging server. ${e.getMessage}")
}
}
}
Example 146
| Source File: TemperatureEndpoint.scala From temperature-machine with Apache License 2.0 | 5 votes |
|
package bad.robot.temperature.server
import java.time.Clock
import bad.robot.logging.Log
import bad.robot.temperature.rrd.Host
import bad.robot.temperature.{jsonEncoder, _}
import cats.effect.IO
import io.circe._
import fs2.{Sink, _}
import org.http4s.HttpService
import org.http4s.dsl.io._
import org.http4s.headers.`X-Forwarded-For`
import org.http4s.server.websocket._
import org.http4s.websocket.WebsocketBits._
import scala.concurrent.duration._
import scala.language.postfixOps
object TemperatureEndpoint {
private implicit val encoder = jsonEncoder[Json]
private implicit val decoder = jsonDecoder[Measurement]
implicit def jsonMapEncoder: Encoder[Map[Host, Measurement]] = new Encoder[Map[Host, Measurement]] {
def apply(measurements: Map[Host, Measurement]): Json = Json.obj(
("measurements", Encoder[List[Measurement]].apply(measurements.values.toList))
)
}
private val latestTemperatures = CurrentTemperatures(Clock.systemDefaultZone)
private val sink: Sink[IO, WebSocketFrame] = _.evalMap { (ws: WebSocketFrame) => ws match {
case Text(fromClient, _) => IO(Log.debug(s"Client sent ws data: $fromClient"))
case frame => IO(Log.debug(s"Unknown type sent from ws client: $frame"))
}}
import scala.concurrent.ExecutionContext.Implicits.global // todo replace with explicit one
def apply(scheduler: Scheduler, sensors: TemperatureReader, allTemperatures: AllTemperatures, connections: Connections) = HttpService[IO] {
case GET -> Root / "temperatures" / "average" => {
Ok(encode(latestTemperatures.average))
}
case GET -> Root / "temperatures" / "live" / "average" => {
val source: Stream[IO, WebSocketFrame] = scheduler.awakeEvery[IO](1 second).map { _ =>
Text(encode(latestTemperatures.average).spaces2ps)
}
WebSocketBuilder[IO].build(source, sink)
}
case GET -> Root / "temperatures" => {
Ok(encode(latestTemperatures.all))
}
case GET -> Root / "temperatures" / "live" => {
val source: Stream[IO, WebSocketFrame] = scheduler.awakeEvery[IO](1 second).map { _ =>
Text(encode(latestTemperatures.all).spaces2ps)
}
WebSocketBuilder[IO].build(source, sink)
}
case DELETE -> Root / "temperatures" => {
latestTemperatures.clear()
NoContent()
}
case request @ PUT -> Root / "temperature" => {
request.decode[Measurement](measurement => {
val result = connections.update(measurement.host, request.headers.get(`X-Forwarded-For`))
result.toHttpResponse(_ => {
latestTemperatures.updateWith(measurement)
allTemperatures.put(measurement)
NoContent()
})
})
}
}
}
Example 147
| Source File: IOs.scala From temperature-machine with Apache License 2.0 | 5 votes |
|
package bad.robot.temperature.task
import java.util.concurrent.Executors._
import bad.robot.logging._
import bad.robot.temperature.ds18b20.{SensorFile, SensorReader}
import bad.robot.temperature.rrd.RrdFile.MaxSensors
import bad.robot.temperature.rrd.{Host, RrdFile}
import bad.robot.temperature.server.AllTemperatures
import bad.robot.temperature.task.Scheduler.ScheduledExecutorServiceOps
import bad.robot.temperature.{FixedTimeMeasurementWriter, JsonExport, TemperatureWriter, XmlExport}
import cats.effect.IO
import cats.implicits._
import scala.concurrent.duration._
import scala.language.postfixOps
object IOs {
def init(hosts: List[Host]) = {
for {
exists <- RrdFile.exists
_ <- info(s"Creating RRD for ${hosts.map(_.name).mkString("'", "', '", "'")} (with up to $MaxSensors sensors each)...").unlessA(exists)
_ <- IO(RrdFile(hosts, 30 seconds).create()).unlessA(exists)
_ <- info("RRD created Ok").unlessA(exists)
} yield ()
}
def gather(temperatures: AllTemperatures, destination: FixedTimeMeasurementWriter) = {
val frequency = 30 seconds
val executor = newSingleThreadScheduledExecutor(TemperatureMachineThreadFactory("rrd-writing-thread"))
for {
_ <- info(s"Writing to the RRD every $frequency (sample times may be off by +/- $frequency, maybe a little more)")
tasks <- IO(executor.schedule(frequency, RecordTemperatures(temperatures, destination, Log)))
} yield tasks
}
def record(host: Host, sensors: List[SensorFile], destination: TemperatureWriter) = {
val executor = newSingleThreadScheduledExecutor(TemperatureMachineThreadFactory("reading-thread"))
for {
_ <- info(s"Monitoring sensor file(s) on '${host.name}' ${sensors.mkString("\n\t", "\n\t", "\n")}")
tasks <- IO(executor.schedule(1 second, RecordTemperature(SensorReader(host, sensors), destination, Log)))
} yield tasks
}
def graphing(implicit hosts: List[Host]) = {
val executor = newScheduledThreadPool(3, TemperatureMachineThreadFactory("graphing-thread"))
for {
_ <- IO(executor.schedule(90 seconds, GenerateGraph(24 hours)))
_ <- IO(executor.schedule(12 hours, GenerateGraph(7 days)))
_ <- IO(executor.schedule(24 hours, GenerateGraph(30 days)))
} yield ()
}
def exportXml(implicit hosts: List[Host]) = {
val executor = newSingleThreadScheduledExecutor(TemperatureMachineThreadFactory("xml-export-thread"))
IO(executor.schedule(100 seconds, XmlExport(24 hours)))
}
def exportJson(implicit hosts: List[Host]) = {
val executor = newSingleThreadScheduledExecutor(TemperatureMachineThreadFactory("json-export-thread"))
IO(executor.schedule(100 seconds, JsonExport(24 hours)))
}
}
Example 148
package flaky
import java.io._
import java.net.{HttpURLConnection, URL}
import java.util.Scanner
import sbt.{File, Logger}
import scala.language.postfixOps
import scala.util.{Failure, Success, Try}
object Io {
def writeToFile(file: File, content: String): Unit = {
new PrintWriter(file) {
write(content)
close()
}
}
def writeToFile(file: File, content: Array[Byte]): Unit = {
new FileOutputStream(file) {
write(content)
close()
}
}
def writeToFile(file: File, is: InputStream): Unit = {
val array: Array[Byte] = Stream.continually(is.read).takeWhile(_ != -1).map(_.toByte).toArray
writeToFile(file, array)
}
def sendToSlack(webHook: String, jsonMsg: String, log: Logger, backupFile: File): Unit = {
log.info("Sending report to slack")
log.debug("Dumping slack msg to file")
new PrintWriter(backupFile) {
write(jsonMsg)
close()
}
val send: Try[Unit] = Try {
val url = new URL(webHook)
val urlConnection = url.openConnection().asInstanceOf[HttpURLConnection]
// Indicate that we want to write to the HTTP request body
urlConnection.setDoOutput(true)
urlConnection.setRequestMethod("POST")
// Writing the post data to the HTTP request body
log.debug(jsonMsg)
val httpRequestBodyWriter = new BufferedWriter(new OutputStreamWriter(urlConnection.getOutputStream))
httpRequestBodyWriter.write(jsonMsg)
httpRequestBodyWriter.close()
val scanner = new Scanner(urlConnection.getInputStream)
log.debug("Response from SLACK:")
while (scanner.hasNextLine) {
log.debug(s"Response from SLACK: ${scanner.nextLine()}")
}
scanner.close()
}
send match {
case Success(_) => log.info("Notification successfully send to Slack")
case Failure(e) => log.error(s"Can't send message to slack: ${e.getMessage}")
}
}
}
Example 149
| Source File: ChangeStream.scala From changestream with MIT License | 5 votes |
|
package changestream
import java.io.IOException
import java.util.concurrent.TimeoutException
import com.github.shyiko.mysql.binlog.BinaryLogClient
import com.typesafe.config.ConfigFactory
import org.slf4j.LoggerFactory
import scala.concurrent.Await
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.concurrent.ExecutionContext.Implicits.global
object ChangeStream extends App {
protected val log = LoggerFactory.getLogger(getClass)
protected val config = ConfigFactory.load().getConfig("changestream")
protected val mysqlHost = config.getString("mysql.host")
protected val mysqlPort = config.getInt("mysql.port")
protected val overridePosition = System.getenv("OVERRIDE_POSITION") match {
case position:String if (position != null && position.length > 0) => Some(position) //scalastyle:ignore
case _ => None
}
protected val client = new BinaryLogClient(
mysqlHost,
mysqlPort,
config.getString("mysql.user"),
config.getString("mysql.password")
)
client.setKeepAliveInterval(config.getLong("mysql.keepalive"))
ChangeStreamEventListener.setConfig(config)
ChangestreamEventDeserializerConfig.setConfig(config)
ChangeStreamEventListener.startControlServer(config)
client.registerEventListener(ChangeStreamEventListener)
client.setEventDeserializer(ChangeStreamEventDeserializer)
client.registerLifecycleListener(ChangeStreamLifecycleListener)
getConnected(overridePosition)
def serverName = s"${mysqlHost}:${mysqlPort}"
def clientId = client.getServerId
def isConnected = client.isConnected
def getConnectedAndWait(startingPosition: Option[String]) = Await.result(getConnected(startingPosition), 60.seconds)
def disconnectClient = client.disconnect()
def getConnected(startingPosition: Option[String]) = {
log.info("Starting changestream...")
val getPositionFuture = startingPosition match {
case Some(_) =>
log.info("Overriding starting binlog position with OVERRIDE_POSITION={}", overridePosition)
ChangeStreamEventListener.setPosition(startingPosition)
case _ =>
ChangeStreamEventListener.getStoredPosition
}
getPositionFuture.map { position =>
setBinlogClientPosition(position)
getInternalClientConnected
}
}
protected def setBinlogClientPosition(position: Option[String]) = position match {
case Some(position) =>
log.info("Setting starting binlog position at {}.", position)
val Array(fileName, posLong) = position.split(":")
client.setBinlogFilename(fileName)
client.setBinlogPosition(java.lang.Long.valueOf(posLong))
case None =>
log.info("Starting binlog position in real time")
client.setBinlogFilename(null) //scalastyle:ignore
client.setBinlogPosition(4L)
}
protected def getInternalClientConnected = {
while(!client.isConnected) {
try {
client.connect(5000)
}
catch {
case e: IOException =>
log.error("Failed to connect to MySQL to stream the binlog, retrying in 5 seconds...", e)
Thread.sleep(5000)
case e: TimeoutException =>
log.error("Timed out connecting to MySQL to stream the binlog, retrying in 5 seconds...", e)
Thread.sleep(5000)
case e: Exception =>
log.error("Failed to connect, exiting...", e)
Await.result(ChangeStreamEventListener.shutdownAndExit(1), 60.seconds)
}
}
}
}
Example 150
| Source File: ControlInterface.scala From changestream with MIT License | 5 votes |
|
package changestream.actors
import akka.util.Timeout
import spray.httpx.SprayJsonSupport._
import spray.routing._
import akka.actor._
import ch.qos.logback.classic.Level
import changestream.{ChangeStream, ChangeStreamEventDeserializer, ChangeStreamEventListener}
import org.slf4j.LoggerFactory
import ch.qos.logback.classic.Logger
import spray.http.StatusCodes
import spray.routing.HttpService
import spray.json.DefaultJsonProtocol
import scala.concurrent.Await
import scala.concurrent.duration._
import scala.language.postfixOps
class ControlInterfaceActor extends Actor with ControlInterface {
def actorRefFactory = context
def receive = runRoute(controlRoutes)
}
trait ControlInterface extends HttpService with DefaultJsonProtocol {
import ControlActor._
protected val log = LoggerFactory.getLogger(getClass)
// yes this is backward on purpose
implicit val memoryInfoFormat = jsonFormat3(MemoryInfo)
implicit val statusFormat = jsonFormat7(Status)
implicit def executionContext = actorRefFactory.dispatcher
implicit val timeout = Timeout(10 seconds)
def controlRoutes: Route = {
get {
pathSingleSlash {
detach() {
complete(getStatus)
}
} ~
path("status") {
detach() {
complete(getStatus)
}
} ~
path("logs") {
parameter('level) { level => setLogLevel(level) }
}
}
}
def setLogLevel(level: String) = {
level.toLowerCase match {
case "all" | "trace" | "debug" | "info" | "warn" | "error" | "off" =>
val rootLogger = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[Logger]
rootLogger.setLevel(Level.toLevel(level))
complete("ChangeStream logging level has been set to {}.", level)
case _ =>
log.error("ControlActor received invalid log level {}.", level)
complete(StatusCodes.BadRequest, s"Invalid log level: ${level}")
}
}
def getStatus = {
val storedPosition = Await.result(ChangeStreamEventListener.getStoredPosition, 60 seconds)
Status(
server = ChangeStream.serverName,
clientId = ChangeStream.clientId,
isConnected = ChangeStream.isConnected,
binlogClientPosition = ChangeStreamEventListener.getCurrentPosition,
lastStoredPosition = storedPosition.getOrElse(""),
binlogClientSequenceNumber = ChangeStreamEventDeserializer.getCurrentSequenceNumber,
memoryInfo = MemoryInfo(
Runtime.getRuntime().totalMemory(),
Runtime.getRuntime().maxMemory(),
Runtime.getRuntime().freeMemory()
)
)
}
}
object ControlActor {
case class Status(
server: String,
clientId: Long,
isConnected: Boolean,
binlogClientPosition: String,
lastStoredPosition: String,
binlogClientSequenceNumber: Long,
memoryInfo: MemoryInfo
)
case class MemoryInfo(heapSize: Long, maxHeap: Long, freeHeap: Long)
}
Example 151
| Source File: BenchBase.scala From changestream with MIT License | 5 votes |
|
package changestream.helpers
import akka.actor.{ActorRefFactory, ActorSystem, Props}
import akka.testkit.{TestActorRef, TestProbe}
import com.github.mauricio.async.db.Configuration
import com.github.mauricio.async.db.mysql.MySQLConnection
import com.typesafe.config.ConfigFactory
import org.scalameter.api._
import org.scalameter.picklers.Implicits._
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.concurrent.Await
class BenchBase extends Bench[Double] {
lazy val executor = LocalExecutor(
new Executor.Warmer.Default,
Aggregator.min[Double],
measurer)
lazy val measurer = new Measurer.Default
lazy val reporter = new LoggingReporter[Double]
lazy val persistor = Persistor.None
implicit val system = ActorSystem("changestream", ConfigFactory.load("test.conf"))
implicit val ec = system.dispatcher
val probe = TestProbe()
val maker = (_: ActorRefFactory) => probe.ref
val testConfig = ConfigFactory.load("test.conf")
def getProbedActorOf[K](klass: Predef.Class[K], configPath: String = "changestream") =
TestActorRef(Props(klass, maker, testConfig.getConfig(configPath)))
protected val config = testConfig.getConfig("changestream.mysql")
protected val mysqlConfig = new Configuration(
config.getString("user"),
config.getString("host"),
config.getInt("port"),
Some(config.getString("password"))
)
protected val connectionTimeout = config.getLong("timeout")
protected val connection = new MySQLConnection(mysqlConfig)
Await.result(connection.connect, connectionTimeout milliseconds)
val result = connection.sendQuery("drop database if exists changestream_test")
.flatMap(_ => connection.sendQuery("create database changestream_test"))
.flatMap(_ => connection.sendQuery(s"""
| CREATE TABLE changestream_test.users (
| `id` int(11) NOT NULL AUTO_INCREMENT,
| `username` varchar(32) DEFAULT NULL,
| `password` varchar(32) DEFAULT NULL,
| `login_count` int(11) NOT NULL DEFAULT '0',
| `bio` text DEFAULT NULL,
| PRIMARY KEY (`id`)
| ) ENGINE=InnoDB
""".stripMargin))
Await.result(result, (connectionTimeout * 3) milliseconds)
}
Example 152
| Source File: Database.scala From changestream with MIT License | 5 votes |
|
package changestream.helpers
import com.github.mauricio.async.db.Configuration
import com.github.mauricio.async.db.mysql.MySQLConnection
import org.scalatest.BeforeAndAfterAll
import scala.concurrent.Await
import scala.concurrent.duration._
import scala.language.postfixOps
trait Database extends Base with Config with BeforeAndAfterAll {
val INSERT = s"""INSERT INTO changestream_test.users VALUES (
| NULL, "peter", "hello", 10, "I Am Peter", 0,
| 0.18289882, 0.23351681013224323546495497794239781796932220458984375, NOW(), CURDATE(), CURTIME(), NOW(), null)
""".stripMargin.trim
val INSERT_MULTI = s"""${INSERT},
| (NULL, "multiguy", "stupid", 1000000, "Leeloo Dallas Multiguy", 0, 0.18289882,
| 0.23351681013224323546495497794239781796932220458984375, NOW(), CURDATE(), CURTIME(), NOW(), null)""".stripMargin
val UPDATE = s"""UPDATE changestream_test.users set username = "username2", password = "password2", login_count = login_count + 1, bio = "bio2""""
val UPDATE_ALL = "UPDATE changestream_test.users set login_count = login_count + 1"
val DELETE = "DELETE from changestream_test.users LIMIT 1"
val DELETE_ALL = "DELETE from changestream_test.users"
protected val config = testConfig.getConfig("changestream.mysql")
protected val mysqlConfig = new Configuration(
config.getString("user"),
config.getString("host"),
config.getInt("port"),
Some(config.getString("password"))
)
protected val connectionTimeout = config.getLong("timeout")
protected val connection = new MySQLConnection(mysqlConfig)
def queryAndWait(sql: String): Unit = Await.result(connection.sendQuery(sql), connectionTimeout milliseconds)
override def beforeAll(): Unit = {
try {
Await.result(connection.connect, connectionTimeout milliseconds)
} catch {
case e: Exception =>
println(s"Could not connect to MySQL server for Metadata: ${e.getMessage}\n${e.getStackTrace.mkString("\n")}")
}
}
override def afterAll(): Unit = {
Await.result(connection.disconnect, connectionTimeout milliseconds)
}
before {
try {
val result = connection.sendQuery("drop database if exists changestream_test")
.flatMap(_ => connection.sendQuery("create database changestream_test"))
.flatMap(_ => connection.sendQuery(s"""
| CREATE TABLE changestream_test.users (
| `id` int(11) NOT NULL AUTO_INCREMENT,
| `username` varchar(32) DEFAULT NULL,
| `password` varchar(32) DEFAULT NULL,
| `login_count` int(11) NOT NULL DEFAULT '0',
| `bio` text DEFAULT NULL,
| `two_bit_field` bit DEFAULT NULL,
| `float_field` float DEFAULT NULL,
| `big_decimal_field` decimal DEFAULT NULL,
| `java_util_date` datetime DEFAULT NULL,
| `java_sql_date` date DEFAULT NULL,
| `java_sql_time` time DEFAULT NULL,
| `java_sql_timestamp` timestamp DEFAULT NOW(),
| `blob_that_should_be_ignored` blob DEFAULT NULL,
| PRIMARY KEY (`id`)
| ) ENGINE=InnoDB;
""".stripMargin))
Await.result(result, (connectionTimeout * 3) milliseconds)
} catch {
case e: Exception =>
println(s"Could not initialize database for tests ${e.getMessage}\n${e.getStackTrace.mkString("\n")}")
}
}
after {
try {
val result = connection.sendQuery("drop database if exists changestream_test")
Await.result(result, connectionTimeout milliseconds)
} catch {
case e: Exception =>
println(s"Could not tear down database for tests ${e.getMessage}\n${e.getStackTrace.mkString("\n")}")
}
}
}
Example 153
| Source File: StdoutActorSpec.scala From changestream with MIT License | 5 votes |
|
package changestream.actors
import akka.actor.{ActorRefFactory, Props}
import akka.testkit.{TestActorRef, TestProbe}
import changestream.actors.PositionSaver.EmitterResult
import changestream.helpers.{Config, Emitter}
import scala.concurrent.duration._
import scala.language.postfixOps
class StdoutActorSpec extends Emitter with Config {
val probe = TestProbe()
val maker = (_: ActorRefFactory) => probe.ref
val actorRef = TestActorRef(Props(classOf[StdoutActor], maker, awsConfig))
"When StdoutActor receives a single valid message" should {
"Print to stdout and forward result" in {
actorRef ! message
val result = probe.expectMsgType[EmitterResult](5000 milliseconds)
result.position should be(message.nextPosition)
}
}
}
Example 154
| Source File: SqsActorSpec.scala From changestream with MIT License | 5 votes |
|
package changestream.actors
import akka.actor.{ActorRefFactory, Props}
import akka.testkit.{TestActorRef, TestProbe}
import changestream.actors.PositionSaver.EmitterResult
import changestream.actors.SqsActor.BatchResult
import changestream.helpers.{Config, Emitter}
import scala.concurrent.duration._
import scala.language.postfixOps
class SqsActorSpec extends Emitter with Config {
val probe = TestProbe()
val maker = (_: ActorRefFactory) => probe.ref
val actorRef = TestActorRef(Props(classOf[SqsActor], maker, awsConfig))
"When SqsActor receives a single valid message" should {
"Add the message to the SQS queue in a batch of one" in {
actorRef ! message
val result = probe.expectMsgType[EmitterResult](5000 milliseconds)
result.position should be(message.nextPosition)
result.meta.get shouldBe a[BatchResult]
result.meta.get.asInstanceOf[BatchResult].failed shouldBe empty
result.meta.get.asInstanceOf[BatchResult].queued should have length 1
}
}
"When SqsActor receives multiple valid messages in quick succession" should {
"Add the messages to the SQS queue in a batch of multiple" in {
actorRef ! message
actorRef ! message.copy(nextPosition = "FOOBAZ")
val result = probe.expectMsgType[EmitterResult](5000 milliseconds)
result.position should be("FOOBAZ")
result.meta.get shouldBe a[BatchResult]
result.meta.get.asInstanceOf[BatchResult].failed shouldBe empty
result.meta.get.asInstanceOf[BatchResult].queued should have length 2
}
}
"When SqsActor receives multiple valid messages in slow succession" should {
"Add the messages to the SQS queue in multiple batches of one message" in {
actorRef ! message
Thread.sleep(500)
actorRef ! message.copy(nextPosition = "FOOBAZ")
val result1 = probe.expectMsgType[EmitterResult](5000 milliseconds)
val result2 = probe.expectMsgType[EmitterResult](5000 milliseconds)
result1.position should be(message.nextPosition)
result1.meta.get shouldBe a[BatchResult]
result1.meta.get.asInstanceOf[BatchResult].failed shouldBe empty
result1.meta.get.asInstanceOf[BatchResult].queued should have length 1
result2.position should be("FOOBAZ")
result2.meta.get shouldBe a[BatchResult]
result2.meta.get.asInstanceOf[BatchResult].failed shouldBe empty
result2.meta.get.asInstanceOf[BatchResult].queued should have length 1
}
}
}
Example 155
| Source File: SnsActorSpec.scala From changestream with MIT License | 5 votes |
|
package changestream.actors
import akka.actor.{ActorRefFactory, Props}
import akka.testkit.{TestActorRef, TestProbe}
import changestream.actors.PositionSaver.EmitterResult
import changestream.helpers.{Config, Emitter}
import com.typesafe.config.ConfigFactory
import scala.concurrent.duration._
import scala.language.postfixOps
class SnsActorSpec extends Emitter with Config {
val probe = TestProbe()
val maker = (_: ActorRefFactory) => probe.ref
val actorRef = TestActorRef(Props(classOf[SnsActor], maker, awsConfig))
val configWithInterpolation = ConfigFactory.
parseString("aws.sns.topic = \"__integration_tests-{database}-{tableName}\"").
withFallback(awsConfig)
val snsWithInterpolation = TestActorRef(Props(classOf[SnsActor], maker, configWithInterpolation))
"When SnsActor receives a single valid message" should {
"Immediately publish the message to SNS" in {
actorRef ! message
val result = probe.expectMsgType[EmitterResult](5000 milliseconds)
result.position should be(message.nextPosition)
}
}
"When SnsActor receives a message" should {
"Should correctly publish the message when the topic contains interpolated database and/or tableName" in {
snsWithInterpolation ! message
val result = probe.expectMsgType[EmitterResult](5000 milliseconds)
result.position should be(message.nextPosition)
}
}
}
Example 156
| Source File: S3ActorSpec.scala From changestream with MIT License | 5 votes |
|
package changestream.actors
import akka.actor.{ActorRefFactory, Props}
import akka.testkit.{TestActorRef, TestProbe}
import changestream.actors.PositionSaver.EmitterResult
import changestream.helpers.{Config, Emitter}
import scala.concurrent.duration._
import com.typesafe.config.ConfigFactory
import scala.language.postfixOps
class S3ActorSpec extends Emitter with Config {
val probe = TestProbe()
val maker = (_: ActorRefFactory) => probe.ref
val s3Config = ConfigFactory.
parseString("aws.s3.batch-size = 2, aws.s3.flush-timeout = 1000").
withFallback(awsConfig)
val actorRef = TestActorRef(Props(classOf[S3Actor], maker, s3Config))
"When S3Actor receives a single valid message" should {
"Add the message to S3 in a batch of one" in {
actorRef ! message
val result = probe.expectMsgType[EmitterResult](5000 milliseconds)
result.position should be(message.nextPosition)
result.meta.get.asInstanceOf[String] should endWith ("-1.json")
}
}
"When S3Actor receives multiple valid messages in quick succession" should {
"Add the messages to S3 in a batch of many" in {
actorRef ! message
actorRef ! message.copy(nextPosition = "FOOBAZ")
val result = probe.expectMsgType[EmitterResult](5000 milliseconds)
result.position should be("FOOBAZ")
result.meta.get.asInstanceOf[String] should endWith ("-2.json")
}
}
"When S3Actor receives multiple valid messages in slow succession" should {
"Add the messages to the S3 queue in multiple batches of one message" in {
actorRef ! message
Thread.sleep(2000)
actorRef ! message.copy(nextPosition = "FOOBAZ")
val result1 = probe.expectMsgType[EmitterResult](5000 milliseconds)
val result2 = probe.expectMsgType[EmitterResult](5000 milliseconds)
result1.position should be(message.nextPosition)
result1.meta.get.asInstanceOf[String] should endWith ("-1.json")
result2.position should be("FOOBAZ")
result2.meta.get.asInstanceOf[String] should endWith ("-1.json")
}
}
"When S3Actor receives multiple valid messages that exceed the flush size" should {
"Add the messages to the S3 queue in multiple batches" in {
actorRef ! message
actorRef ! message.copy(nextPosition = "FOOBAZ")
actorRef ! message.copy(nextPosition = "BIPBOP")
val result1 = probe.expectMsgType[EmitterResult](5000 milliseconds)
val result2 = probe.expectMsgType[EmitterResult](5000 milliseconds)
result1.position should be("FOOBAZ")
result1.meta.get.asInstanceOf[String] should endWith ("-2.json")
result2.position should be("BIPBOP")
result2.meta.get.asInstanceOf[String] should endWith ("-1.json")
}
}
}
Example 157
| Source File: RowsQueryDeserializerSpec.scala From changestream with MIT License | 5 votes |
|
package changestream
import changestream.helpers.{Base, Config}
import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream
import com.typesafe.config.ConfigFactory
import scala.language.postfixOps
class RowsQueryDeserializerSpec extends Base with Config {
val sqlString = "select 'foo';"
val bytes = Array[Byte](1) ++ sqlString.getBytes
var inputStream = new ByteArrayInputStream(bytes)
before {
inputStream = new ByteArrayInputStream(bytes)
inputStream.enterBlock(bytes.length)
}
"When not configured to truncate the SQL" should {
"Capture all of the SQL" in {
RowsQueryDeserializer.deserialize(inputStream)
ChangeStreamEventDeserializer.lastQuery should be(Some(sqlString))
inputStream.available() should be (0)
}
}
"When configured to truncate the SQL at a given length" should {
"Capture all of the SQL when the SQL is shorter or equal to the limit" in {
val bigLimitConfig = ConfigFactory.parseString("changestream.sql-character-limit = 1000").withFallback(testConfig).getConfig("changestream")
ChangestreamEventDeserializerConfig.setConfig(bigLimitConfig)
RowsQueryDeserializer.deserialize(inputStream)
ChangeStreamEventDeserializer.lastQuery should be(Some(sqlString))
inputStream.available() should be (0)
}
"When the SQL is longer than the limit" should {
"Only capture <limit> characters of SQL" in {
val smallLimitConfig = ConfigFactory.parseString("changestream.sql-character-limit = 5").withFallback(testConfig).getConfig("changestream")
ChangestreamEventDeserializerConfig.setConfig(smallLimitConfig)
RowsQueryDeserializer.deserialize(inputStream)
ChangeStreamEventDeserializer.lastQuery should be(Some(sqlString.substring(0, 5)))
inputStream.available() should be (0)
}
}
}
}
Example 158
| Source File: EmTest.scala From Scala-for-Machine-Learning-Second-Edition with MIT License | 5 votes |
|
package org.scalaml.unsupervised.em
import org.scalaml.{Logging, Resource}
import org.scalaml.Predef.{DblMatrix, DblVec}
import org.scalaml.filtering.movaverage.SimpleMovingAverage
import org.scalaml.workflow.data.DataSource
import org.scalatest.{FlatSpec, Matchers}
import org.scalaml.unsupervised.env.TestEnv
import scala.util.Try
final class EMTest extends FlatSpec with Matchers with TestEnv with Logging with Resource {
import scala.language.postfixOps
protected[this] val name: String = "Expectation-Maximization"
final val rawPath = "unsupervised/em"
it should s"$name evaluation with K=2, sampling rate=40" in {
show(s"$name evaluation with K=2, sampling rate=40")
execute(Array[String]("2", "40"))
}
it should s"$name evaluation with K=3, sampling rate=25" in {
show(s"$name evaluation with K=3, sampling rate=25")
execute(Array[String]("3", "25"))
}
it should s"$name evaluation with K=4, sampling rate=15" in {
show(s"$name evaluation with K=4, sampling rate=15")
execute(Array[String]("4", "15"))
}
it should s"$name evaluation with K=5, sampling rate=10" in {
show(s"$name evaluation with K=5, sampling rate=10")
execute(Array[String]("5", "10"))
}
private def execute(args: Array[String]): Unit = {
require(args.length > 0, s"$name Cannot be evaluated with undefined arguments")
val K = args(0).toInt
val samplingRate = args(1).toInt
val period = 8
val smAve = SimpleMovingAverage[Double](period)
val symbols = symbolFiles(getPath(rawPath).get)
// extracts the observations from a set of csv files.
assert(symbols.size > 0, s"$name The symbol files are undefined")
// Retrieve the partial function for the simple moving average
val pfnSmAve = smAve |>
// Walk through the stock ticker symbols and load the historical data from each
// ticker symbol using the data source extractor
val obs = symbols.map(sym => {
(for {
path <- getPath(s"$rawPath/")
src <- DataSource(sym, path, true, 1)
// Extract data from the files containing historical financial data
xs <- src |> (extractor)
// Apply the simple moving average of the first variable
if pfnSmAve.isDefinedAt(xs.head.toVector)
values <- pfnSmAve(xs.head.toVector)
// Generate the features by filtering using a sampling rate
y <- filter(period, values, samplingRate)
} yield y).getOrElse(Array.empty[Double])
})
em(K, obs)
}
private def filter(period: Int, values: DblVec, samplingRate: Int) = Try {
values.view
.indices
.drop(period + 1).toVector
.filter(_ % samplingRate == 0)
.map(values(_)).toArray
}
private def profile(xv: Vector[Array[Double]]) {
import org.scalaml.plots.Legend
val legend = Legend("Density", "EM convergence K=6", "Iterations", "Density")
val em = MultivariateEM[Double](3, xv)
em.display("Density", legend)
}
private def em(K: Int, obs: DblMatrix): Int = {
val em = MultivariateEM[Double](K, obs.toVector)
show(s"${em.toString}")
}
}
// ------------------------------------- EOF -------------------------------
Example 159
| Source File: PCATest.scala From Scala-for-Machine-Learning-Second-Edition with MIT License | 5 votes |
|
package org.scalaml.unsupervised.pca
import org.scalaml.Logging
import org.scalaml.unsupervised.env.TestEnv
import org.scalaml.util.Assertable
import org.scalatest.{FlatSpec, Matchers}
final class PCATest extends FlatSpec with Matchers with Logging with Assertable with TestEnv {
protected[this] val name = "Principal Components Analysis"
// Symbol, PE/PS/PB/ROE/OM
final private val data = Vector[(String, Array[Double])](
("QCOM", Array[Double](20.8, 5.32, 3.65, 17.65, 29.2)),
("IBM", Array[Double](13, 1.22, 12.2, 88.1, 19.9)),
("BAC", Array[Double](21, 2.0, 0.78, 4.12, 24.2)),
("AA", Array[Double](21.7, 0.7, 1.4, -16.3, 5.1)),
("ADM", Array[Double](22.3, 0.33, 1.47, 6.9, 2.4)),
("AET", Array[Double](12.7, 0.54, 1.8, 15.6, 7.6)),
("AFAM", Array[Double](24.8, 0.57, 0.94, 4.0, 4.1)),
("AHS", Array[Double](18.6, 0.57, 2.6, 16.4, 6.5)),
("UNH", Array[Double](13.8, 0.63, 2.4, 17.1, 7.8)),
("THC", Array[Double](17.4, 0.38, 4.9, 7.9, 7.1)),
("BK", Array[Double](19.4, 2.5, 1.0, 5.8, 27.4)),
("JPM", Array[Double](13.7, 2.2, 1.0, 7.8, 38.1)),
("WFC", Array[Double](12.2, 3.1, 1.5, 13.5, 43.7)),
("HCA", Array[Double](14.6, 0.63, 2.5, 17.4, 9.7)),
("FBP", Array[Double](12.8, 2.2, 0.91, -12.1, 29.7)),
("USB", Array[Double](13.4, 4.2, 2.0, 13.7, 43.4)),
("FFBC", Array[Double](20.4, 3.3, 1.4, 7.1, 26.7)),
("CAH", Array[Double](16.4, 0.3, 3.5, 5.8, 2.1)),
("DVA", Array[Double](23.5, 1.25, 3.3, 15.1, 15.7)),
("FHN", Array[Double](13.9, 2.35, 1.2, 1.6, 9.3)),
("FB", Array[Double](96.7, 19.4, 9.9, 11.7, 37.1)),
("TWTR", Array[Double](208, 37.9, 8.9, -34, -96)),
("YELP", Array[Double](231, 21.1, 9.6, -3.1, -4.7)),
("ANGI", Array[Double](67.3, 3.4, 7.9, -16.7, -11.3)),
("LNKD", Array[Double](771, 13.6, 7.9, 1.5, 3.1)),
("TSLA", Array[Double](101.2, 12.9, 36.1, -18.7, -3.8)),
("CYH", Array[Double](21.6, 0.28, 1.1, 6.6, 7.9)),
("PCLN", Array[Double](37.6, 9.5, 9.2, 34.8, 35.9)),
("CVS", Array[Double](19.7, 0.68, 2.3, 12.2, 6.3)),
("FISV", Array[Double](23.7, 3.0, 4.1, 18.6, 22.7)),
("DOW", Array[Double](13.3, 1.0, 2.5, 19.3, 8.0)),
("K", Array[Double](13.4, 1.6, 6.7, 59.8, 21.3))
)
it should s"$name evaluation" in {
import scala.language.postfixOps
show(s"$name evaluation")
val dim = data.head._2.size
val pca = new PCA[Double](data.map(_._2.take(dim)))
show(s"$name model: ${pca.toString}")
}
}
// -------------------------- EOF ------------------------------------------------------
Example 160
| Source File: BakerySpec.scala From Learn-Scala-Programming with MIT License | 5 votes |
|
package ch12
import akka.actor.testkit.typed.Effect.{NoEffects, Spawned}
import akka.actor.testkit.typed.scaladsl._
import akka.actor.typed.DispatcherSelector
import ch12.Baker.BakeCookies
import ch12.Bakery.{Groceries, RawCookies}
import ch12.Boy.GoShopping
import ch12.Chef.Mix
import ch12.Oven.Extract
import ch12.Shop.{SellByList, ShoppingList}
import com.typesafe.config.Config
import org.scalatest._
import scala.concurrent.duration._
import scala.language.postfixOps
class BakerySpec extends WordSpec with ActorTestKit with BeforeAndAfterAll {
override def afterAll: Unit = shutdownTestKit()
"The boy should" should {
"forward given ShoppingList to the seller" in {
val testKit = BehaviorTestKit(Boy.goShopping)
val seller = TestInbox[Shop.SellByList]()
val manager = TestInbox[Manager.Command]()
val list = ShoppingList(1, 1, 1, 1)
testKit.run(GoShopping(list, seller.ref, manager.ref))
testKit.expectEffect(NoEffects)
seller.expectMessage(SellByList(list, manager.ref))
assert(!testKit.isAlive)
}
}
"The chef should" should {
"create and destroy mixers as required" in {
// the mixerFactory is needed because behaviour equals method is not implemented correctly
// currently behaviours compared by reference
// therefore we need to have the same behaviour instance for the test to pass
val mixerFactory = Mixer.mix(0 seconds)
val chef = BehaviorTestKit(Chef.idle(mixerFactory))
val manager = TestInbox[Manager.Command]()
val message = Mix(Groceries(1, 1, 1, 1), manager.ref)
chef.run(message)
chef.expectEffect(
Spawned(mixerFactory,
"Mixer_1",
DispatcherSelector.fromConfig("mixers-dispatcher")))
val expectedByMixer = Mixer.Mix(Groceries(1, 1, 1, 1), chef.ref)
chef.childInbox("Mixer_1").expectMessage(expectedByMixer)
}
}
override def config: Config = ManualTime.config
val manualTime: ManualTime = ManualTime()
"The baker should" should {
"bake cookies in batches" in {
val oven = TestProbe[Oven.Command]()
val manager = TestProbe[Manager.Command]()
val baker = spawn(Baker.idle(oven.ref))
baker ! BakeCookies(RawCookies(1), manager.ref)
oven.expectMessage(Oven.Put(1, baker))
manualTime.expectNoMessageFor(Baker.DefaultBakingTime - 1.millisecond,
oven)
manualTime.timePasses(Baker.DefaultBakingTime)
oven.expectMessage(Extract(baker))
}
}
}
Example 161
| Source File: ShopSpec.scala From Learn-Scala-Programming with MIT License | 5 votes |
|
package ch12
import akka.actor.testkit.typed.Effect.NoEffects
import akka.actor.testkit.typed.scaladsl.{BehaviorTestKit, TestInbox}
import akka.actor.typed.receptionist.Receptionist
import akka.actor.typed.receptionist.Receptionist.Register
import ch12.Bakery.Groceries
import ch12.Manager.ReceiveGroceries
import ch12.Shop.{SellByList, ShoppingList}
import org.scalatest.WordSpec
import scala.language.postfixOps
class ShopSpec extends WordSpec {
"A seller in the shop" should {
"return groceries if given a shopping list" in {
val receptionist = TestInbox[Receptionist.Command]()
val mockReceptionist: Shop.ReceptionistFactory = _ => receptionist.ref
val seller = BehaviorTestKit(Shop.seller(mockReceptionist))
val inbox = TestInbox[Manager.Command]()
val message = ShoppingList(1,1,1,1)
seller.run(SellByList(message, inbox.ref))
inbox.expectMessage(ReceiveGroceries(Groceries(1, 1, 1, 1)))
receptionist.expectMessage(Register(Shop.SellerKey, seller.ref))
seller.expectEffect(NoEffects)
}
}
}
Example 162
| Source File: BakerySpec.scala From Learn-Scala-Programming with MIT License | 5 votes |
|
package ch11
import akka.actor.{ActorSystem, Props}
import akka.testkit.{ImplicitSender, TestKit, TestProbe}
import ch11.Cook.RawCookies
import ch11.Manager.ShoppingList
import ch11.Oven.Cookies
import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.util.Random
class BakerySpec(_system: ActorSystem)
extends TestKit(_system)
with Matchers
with WordSpecLike
with BeforeAndAfterAll
with ImplicitSender {
def this() = this(ActorSystem("BakerySpec"))
override def afterAll: Unit = shutdown(system)
"The boy should" should {
val boyProps = Boy.props(system.actorSelection(testActor.path))
val boy = system.actorOf(boyProps)
"forward given ShoppingList to the seller" in {
val list = ShoppingList(0, 0, 0, 0)
boy ! list
within(3 millis, 20 millis) {
expectMsg(list)
lastSender shouldBe testActor
}
}
"ignore other message types" in {
boy ! 'GoHome
expectNoMessage(500 millis)
}
}
"The baker should" should {
val parent = TestProbe()
val baker = parent.childActorOf(Props(classOf[Baker], 0 millis))
"bake cookies in batches" in {
val count = Random.nextInt(100)
baker ! RawCookies(Oven.size * count)
parent.expectMsgAllOf(List.fill(count)(Cookies(Oven.size)):_*)
}
}
}
Example 163
| Source File: StoreSpec.scala From Learn-Scala-Programming with MIT License | 5 votes |
|
package ch11
import akka.testkit.TestKit
import ch11.Manager.ShoppingList
import ch11.Mixer.Groceries
import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}
import scala.language.postfixOps
class StoreSpec(store: Store) extends TestKit(store.store)
with Matchers with WordSpecLike with BeforeAndAfterAll {
def this() = this(new Store {})
override def afterAll: Unit = shutdown(system)
"A seller in store" should {
"do nothing for all unexpected message types" in {
store.seller ! 'UnexpectedMessage
expectNoMessage()
}
"return groceries if given a shopping list" in {
store.seller.tell(ShoppingList(1, 1, 1, 1), testActor)
expectMsg(Groceries(1,1,1,1))
}
}
}
Example 164
| Source File: RowToVectorBuilder.scala From filo with Apache License 2.0 | 5 votes |
|
package org.velvia.filo
import java.nio.ByteBuffer
import scala.language.existentials
import scala.language.postfixOps
import scalaxy.loops._
import BuilderEncoder.{EncodingHint, AutoDetect}
case class VectorInfo(name: String, dataType: Class[_])
// To help matching against the ClassTag in the VectorBuilder
private object Classes {
val Boolean = classOf[Boolean]
val Byte = java.lang.Byte.TYPE
val Short = java.lang.Short.TYPE
val Int = java.lang.Integer.TYPE
val Long = java.lang.Long.TYPE
val Float = java.lang.Float.TYPE
val Double = java.lang.Double.TYPE
val String = classOf[String]
val DateTime = classOf[org.joda.time.DateTime]
val SqlTimestamp = classOf[java.sql.Timestamp]
val UTF8 = classOf[ZeroCopyUTF8String]
}
object RowToVectorBuilder {
def convertToBytes(hint: EncodingHint = AutoDetect): Map[String, ByteBuffer] = {
val chunks = builders.map(_.toFiloBuffer(hint))
schema.zip(chunks).map { case (VectorInfo(colName, _), bytes) => (colName, bytes) }.toMap
}
private def unsupportedInput(typ: Any) =
throw new RuntimeException("Unsupported input type " + typ)
}
Example 165
| Source File: DictEncodingWrappers.scala From filo with Apache License 2.0 | 5 votes |
|
package org.velvia.filo.codecs
import scala.language.postfixOps
import scalaxy.loops._
import org.velvia.filo.{FiloVector, FastBufferReader}
import org.velvia.filo.vector._
object DictStringWrapper {
// Used to represent no string value or NA. Better than using null.
val NoString = ""
}
abstract class DictStringWrapper(val dsv: DictStringVector) extends FiloVector[String] {
import DictStringWrapper._
private val _len = dsv.len
val reader = FastBufferReader(dsv.codesAsByteBuffer())
// To be mixed in depending on type of code vector
def getCode(index: Int): Int
// Cache the Strings so we only pay cost of deserializing each unique string once
val strCache = Array.fill(dsv.dictionaryLength())(NoString)
final private def dictString(code: Int): String = {
val cacheValue = strCache(code)
if (cacheValue == NoString) {
val strFromDict = dsv.dictionary(code)
strCache(code) = strFromDict
strFromDict
} else {
cacheValue
}
}
final def isAvailable(index: Int): Boolean = getCode(index) != 0
final def apply(index: Int): String = dictString(getCode(index))
final def length: Int = _len
final def foreach[B](fn: String => B): Unit = {
for { i <- 0 until length optimized } {
val code = getCode(i)
if (code != 0) fn(dictString(code))
}
}
}
Example 166
| Source File: SimpleWrappers.scala From filo with Apache License 2.0 | 5 votes |
|
package org.velvia.filo.codecs
import scala.language.postfixOps
import scalaxy.loops._
import org.velvia.filo.{FiloVector, NaMaskAvailable, FastBufferReader}
import org.velvia.filo.vector._
class EmptyFiloVector[A](len: Int) extends FiloVector[A] {
final def isAvailable(index: Int): Boolean = false
final def foreach[B](fn: A => B): Unit = {}
final def apply(index: Int): A =
if (index < len) { null.asInstanceOf[A] }
else { throw new ArrayIndexOutOfBoundsException }
final def length: Int = len
}
abstract class SimplePrimitiveWrapper[@specialized A](spv: SimplePrimitiveVector)
extends NaMaskAvailable[A](spv.naMask) {
val info = spv.info
val _len = spv.len
val reader = FastBufferReader(spv.dataAsByteBuffer())
final def length: Int = _len
final def foreach[B](fn: A => B): Unit = {
if (isEmptyMask) { // every value available!
for { i <- 0 until length optimized } { fn(apply(i)) }
} else {
for { i <- 0 until length optimized } { if (isAvailable(i)) fn(apply(i)) }
}
}
}
// TODO: ditch naMask
class SimpleStringWrapper(ssv: SimpleStringVector)
extends NaMaskAvailable[String](ssv.naMask) {
val _len = ssv.dataLength
final def length: Int = _len
final def apply(i: Int): String = ssv.data(i)
final def foreach[B](fn: String => B): Unit = {
for { i <- 0 until length optimized } { if (isAvailable(i)) fn(apply(i)) }
}
}
Example 167
| Source File: DiffWrappers.scala From filo with Apache License 2.0 | 5 votes |
|
package org.velvia.filo.codecs
import org.joda.time.{DateTime, DateTimeZone}
import scala.language.postfixOps
import scalaxy.loops._
import org.velvia.filo._
import org.velvia.filo.vector._
abstract class DiffPrimitiveWrapper[A: TypedReaderProvider, P](dpv: DiffPrimitiveVector)
extends NaMaskAvailable[P](dpv.naMask) {
val info = dpv.info
val _len = dpv.len
val dataReader = TypedBufferReader[A](FastBufferReader(dpv.dataAsByteBuffer()),
info.nbits, info.signed)
val baseReader = FastBufferReader(dpv.base)
final def length: Int = _len
final def foreach[B](fn: P => B): Unit = {
if (isEmptyMask) { // every value available!
for { i <- 0 until length optimized } { fn(apply(i)) }
} else {
for { i <- 0 until length optimized } { if (isAvailable(i)) fn(apply(i)) }
}
}
}
class DiffDateTimeWithTZWrapper(ddtv: DiffDateTimeVector) extends DiffDateTimeWrapperBase(ddtv) {
import TypedBufferReader._
val tzBase: Byte = ddtv.vars.baseTz
val tzReader = TypedBufferReader[Int](FastBufferReader(ddtv.tzAsByteBuffer),
ddtv.tzInfo.nbits, ddtv.tzInfo.signed)
final def apply(i: Int): DateTime = {
val zone = DateTimeZone.forOffsetMillis((tzBase + tzReader.read(i)) * VectorBuilder.FifteenMinMillis)
new DateTime(millisBase + millisReader.read(i), zone)
}
}
Example 168
| Source File: DictEncodingEncoders.scala From filo with Apache License 2.0 | 5 votes |
|
package org.velvia.filo.codecs
import com.google.flatbuffers.FlatBufferBuilder
import java.nio.ByteBuffer
import java.util.HashMap
import scala.collection.mutable.{ArrayBuffer, BitSet}
import scala.language.postfixOps
import scalaxy.loops._
import org.velvia.filo._
import org.velvia.filo.vector._
object DictEncodingEncoders extends ThreadLocalBuffers {
import Utils._
var count = 0
// Note: This is a way to avoid storing null and dealing with NPEs for NA values
val NaString = ""
def toStringVector(data: Seq[String], naMask: BitSet, stringSet: collection.Set[String]): ByteBuffer = {
import DictStringVector._
count += 1
val builder = AutoIntegralDVBuilders.IntDataVectBuilder
// Convert the set of strings to an encoding
val uniques = stringSet.toSeq
// NOTE: sorry but java's HashMap is just much faster (for the next step)
// This used to be `uniques.zipWithIndex.toMap`
val strToCode = new HashMap[String, Int]()
for { i <- 0 until uniques.length optimized } {
strToCode.put(uniques(i), i)
}
// Encode each string to the code per the map above
// Again we could have used data.zipWithIndex.map(....) but this is much faster.
val codes = ArrayBuffer.fill(data.length)(0)
for { i <- 0 until data.length optimized } {
if (!naMask(i)) codes(i) = strToCode.get(data(i)) + 1
}
val fbb = new FlatBufferBuilder(getBuffer)
val ((dataOffset, nbits), signed) = builder.build(fbb, codes, 0, stringSet.size + 1)
val dictVect = stringVect(fbb, Seq(NaString) ++ uniques)
startDictStringVector(fbb)
addDictionary(fbb, dictVect)
addLen(fbb, data.length)
addCodes(fbb, dataOffset)
addInfo(fbb, DataInfo.createDataInfo(fbb, nbits, signed))
finishDictStringVectorBuffer(fbb, endDictStringVector(fbb))
putHeaderAndGet(fbb, WireFormat.VECTORTYPE_DICT, WireFormat.SUBTYPE_STRING)
}
}
Example 169
| Source File: ScalaReadBenchmark.scala From filo with Apache License 2.0 | 5 votes |
|
package org.velvia.filo
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.{Mode, State, Scope}
import org.openjdk.jmh.annotations.OutputTimeUnit
import scalaxy.loops._
import scala.language.postfixOps
import java.util.concurrent.TimeUnit
@State(Scope.Thread)
class ScalaReadBenchmark {
// Ok, create an IntColumn and benchmark it.
val numValues = 10000
val randomInts = (0 until numValues).map(i => util.Random.nextInt)
val randomIntsAray = randomInts.toArray
// Scala Seq sum
@Benchmark
@BenchmarkMode(Array(Mode.AverageTime))
@OutputTimeUnit(TimeUnit.MICROSECONDS)
def sumAllIntsScalaSeqFoldLeft(): Int = {
randomInts.foldLeft(0)(_ + _)
}
@Benchmark
@BenchmarkMode(Array(Mode.AverageTime))
@OutputTimeUnit(TimeUnit.MICROSECONDS)
def sumAllIntsScalaArrayFoldLeft(): Int = {
randomIntsAray.foldLeft(0)(_ + _)
}
@Benchmark
@BenchmarkMode(Array(Mode.AverageTime))
@OutputTimeUnit(TimeUnit.MICROSECONDS)
def sumAllIntsScalaArrayWhileLoop(): Int = {
var total = 0
for { i <- 0 until numValues optimized } {
total += randomIntsAray(i)
}
total
}
}
Example 170
| Source File: DictStringBenchmark.scala From filo with Apache License 2.0 | 5 votes |
|
package org.velvia.filo
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.{Mode, State, Scope}
import org.openjdk.jmh.annotations.OutputTimeUnit
import scalaxy.loops._
import scala.language.postfixOps
import java.util.concurrent.TimeUnit
@State(Scope.Thread)
class DictStringBenchmark {
import scala.util.Random.{alphanumeric, nextInt, nextFloat}
import VectorReader._
val numValues = 10000
// NOTE: results show that time spent is heavily influenced by ratio of unique strings...
val numUniqueStrings = 500
val maxStringLength = 15
val minStringLength = 5
val naChance = 0.05 //5% of values will be NA
def randString(len: Int): String = alphanumeric.take(len).mkString
val uniqueStrings = (0 until numUniqueStrings).map { i =>
randString(minStringLength + nextInt(maxStringLength - minStringLength))
}
val randomStrings = (0 until numValues).map(i => uniqueStrings(nextInt(numUniqueStrings)))
val filoBufferNoNA = VectorBuilder(randomStrings).toFiloBuffer
val scNoNA = FiloVector[String](filoBufferNoNA)
def shouldNA: Boolean = nextFloat < naChance
val filoBufferNA = VectorBuilder.fromOptions(
randomStrings.map(str => if (shouldNA) None else Some(str))
).toFiloBuffer
val scNA = FiloVector[String](filoBufferNA)
@Benchmark
@BenchmarkMode(Array(Mode.AverageTime))
@OutputTimeUnit(TimeUnit.MICROSECONDS)
def rawStringLengthTotal(): Int = {
var totalLen = 0
for { i <- 0 until numValues optimized } {
totalLen += scNoNA(i).length
}
totalLen
}
// TODO: also a benchmark for the foreach/fold of a column with no NA's?
@Benchmark
@BenchmarkMode(Array(Mode.AverageTime))
@OutputTimeUnit(TimeUnit.MICROSECONDS)
// Measures foreach and NA read speed
def withNAlengthTotal(): Unit = {
var totalLen = 0
scNA.foreach { str => totalLen += str.length }
totalLen
}
}
Example 171
| Source File: BasicFiloBenchmark.scala From filo with Apache License 2.0 | 5 votes |
|
package org.velvia.filo
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.{Mode, State, Scope}
import org.openjdk.jmh.annotations.OutputTimeUnit
import scalaxy.loops._
import scala.language.postfixOps
import java.util.concurrent.TimeUnit
@State(Scope.Thread)
class BasicFiloBenchmark {
import VectorReader._
import vectors.IntBinaryVector
// Ok, create an IntColumn and benchmark it.
val numValues = 10000
val randomInts = (0 until numValues).map(i => util.Random.nextInt)
val randomIntsAray = randomInts.toArray
val filoBuffer = VectorBuilder(randomInts).toFiloBuffer
val sc = FiloVector[Int](filoBuffer)
val ivbuilder = IntBinaryVector.appendingVectorNoNA(numValues)
randomInts.foreach(ivbuilder.addData)
val iv = IntBinaryVector(ivbuilder.base, ivbuilder.offset, ivbuilder.numBytes)
val byteFiloBuf = VectorBuilder(randomInts.map(_ % 128)).toFiloBuffer
val byteVect = FiloVector[Int](byteFiloBuf)
val diffFiloBuf = VectorBuilder(randomInts.map(10000 + _ % 128)).toFiloBuffer
val diffVect = FiloVector[Int](diffFiloBuf)
// According to @ktosopl, be sure to return some value if possible so that JVM won't
// optimize out the method body. However JMH is apparently very good at avoiding this.
// fastest loop possible using FiloVectorApply method
@Benchmark
@BenchmarkMode(Array(Mode.AverageTime))
@OutputTimeUnit(TimeUnit.MICROSECONDS)
def sumAllIntsFiloApply(): Int = {
var total = 0
for { i <- 0 until numValues optimized } {
total += sc(i)
}
total
}
@Benchmark
@BenchmarkMode(Array(Mode.AverageTime))
@OutputTimeUnit(TimeUnit.MICROSECONDS)
def sumAllIntsBinaryVectApply(): Int = {
var total = 0
for { i <- 0 until numValues optimized } {
total += iv(i)
}
total
}
@Benchmark
@BenchmarkMode(Array(Mode.AverageTime))
@OutputTimeUnit(TimeUnit.MICROSECONDS)
def sumAllIntsFiloByteApply(): Int = {
var total = 0
for { i <- 0 until numValues optimized } {
total += byteVect(i)
}
total
}
@Benchmark
@BenchmarkMode(Array(Mode.AverageTime))
@OutputTimeUnit(TimeUnit.MICROSECONDS)
def sumAllIntsFiloDiffApply(): Int = {
var total = 0
for { i <- 0 until numValues optimized } {
total += diffVect(i)
}
total
}
@Benchmark
@BenchmarkMode(Array(Mode.AverageTime))
@OutputTimeUnit(TimeUnit.MICROSECONDS)
def sumAllNotNullIntsFiloApply(): Int = {
var total = 0
for { i <- 0 until numValues optimized } {
if (sc.isAvailable(i)) total += sc(i)
}
total
}
// sum which uses foreach from FiloVector
@Benchmark
@BenchmarkMode(Array(Mode.AverageTime))
@OutputTimeUnit(TimeUnit.MICROSECONDS)
def sumAllIntsFiloForeachFoldLeft(): Int = {
sc.foldLeft(0)(_ + _)
}
}
Example 172
| Source File: FastFiloRowReaderBenchmark.scala From filo with Apache License 2.0 | 5 votes |
|
package org.velvia.filo
import java.sql.Timestamp
import org.openjdk.jmh.annotations.Benchmark
import org.openjdk.jmh.annotations.BenchmarkMode
import org.openjdk.jmh.annotations.{Mode, State, Scope}
import org.openjdk.jmh.annotations.OutputTimeUnit
import scalaxy.loops._
import scala.language.postfixOps
import java.util.concurrent.TimeUnit
@State(Scope.Thread)
class FastFiloRowReaderBenchmark {
import VectorReader._
// Ok, create an IntColumn and benchmark it.
val numValues = 10000
val randomInts = (0 until numValues).map(i => util.Random.nextInt)
val randomLongs = randomInts.map(_.toLong)
val randomTs = randomLongs.map(l => new Timestamp(l))
val chunks = Array(VectorBuilder(randomInts).toFiloBuffer,
VectorBuilder(randomLongs).toFiloBuffer,
VectorBuilder(randomTs).toFiloBuffer)
val clazzes = Array[Class[_]](classOf[Int], classOf[Long], classOf[Timestamp])
// According to @ktosopl, be sure to return some value if possible so that JVM won't
// optimize out the method body. However JMH is apparently very good at avoiding this.
// fastest loop possible using FiloVectorApply method
@Benchmark
@BenchmarkMode(Array(Mode.AverageTime))
@OutputTimeUnit(TimeUnit.MICROSECONDS)
def createFastFiloRowReader(): RowReader = {
new FastFiloRowReader(chunks, clazzes)
}
val fastReader = new FastFiloRowReader(chunks, clazzes)
@Benchmark
@BenchmarkMode(Array(Mode.Throughput))
@OutputTimeUnit(TimeUnit.SECONDS)
def fastFiloRowReaderReadOne(): Int = {
fastReader.setRowNo(0)
if (fastReader.notNull(0)) fastReader.getInt(0) + 1 else 0
}
}
Example 173
| Source File: EmbeddedKafkaCustomConfigSpec.scala From embedded-kafka with MIT License | 5 votes |
|
package net.manub.embeddedkafka
import kafka.server.KafkaConfig
import net.manub.embeddedkafka.EmbeddedKafka._
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.clients.producer.ProducerConfig
import scala.language.postfixOps
import scala.util.Random
class EmbeddedKafkaCustomConfigSpec extends EmbeddedKafkaSpecSupport {
final val TwoMegabytes = 2097152
final val ThreeMegabytes = 3145728
"the custom config" should {
"allow pass additional producer parameters" in {
val customBrokerConfig =
Map(
KafkaConfig.ReplicaFetchMaxBytesProp -> s"$ThreeMegabytes",
KafkaConfig.MessageMaxBytesProp -> s"$ThreeMegabytes"
)
val customProducerConfig =
Map(ProducerConfig.MAX_REQUEST_SIZE_CONFIG -> s"$ThreeMegabytes")
val customConsumerConfig =
Map(
ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG -> s"$ThreeMegabytes"
)
implicit val customKafkaConfig: EmbeddedKafkaConfig =
EmbeddedKafkaConfig(
customBrokerProperties = customBrokerConfig,
customProducerProperties = customProducerConfig,
customConsumerProperties = customConsumerConfig
)
val bigMessage = generateMessageOfLength(TwoMegabytes)
val topic = "big-message-topic"
withRunningKafka {
publishStringMessageToKafka(topic, bigMessage)
consumeFirstStringMessageFrom(topic) shouldBe bigMessage
}
}
}
def generateMessageOfLength(length: Int): String =
Iterator.continually(Random.nextPrintableChar) take length mkString
}
Example 174
| Source File: Plugin.scala From sbt-javacpp with MIT License | 5 votes |
|
package org.bytedeco.sbt.javacpp
import scala.language.postfixOps
import sbt._
import sbt.Keys._
import scala.util.Try
import scala.util.matching.Regex
object Plugin extends AutoPlugin {
override def projectSettings: Seq[Setting[_]] = {
import autoImport._
Seq(
autoCompilerPlugins := true,
javaCppPlatform := Platform.current,
javaCppVersion := Versions.javaCppVersion,
javaCppPresetLibs := Seq.empty,
libraryDependencies += {
"org.bytedeco" % "javacpp" % javaCppVersion.value jar
},
javaCppPresetDependencies)
}
object Versions {
val javaCppVersion = "1.5.3"
}
object autoImport {
val javaCppPlatform = SettingKey[Seq[String]]("javaCppPlatform", """The platform that you want to compile for (defaults to the platform of the current computer). You can also set this via the "sbt.javacpp.platform" System Property """)
val javaCppVersion = SettingKey[String]("javaCppVersion", s"Version of Java CPP that you want to use, defaults to ${Versions.javaCppVersion}")
val javaCppPresetLibs = SettingKey[Seq[(String, String)]]("javaCppPresetLibs", "List of additional JavaCPP presets that you would wish to bind lazily, defaults to an empty list")
}
override def requires: Plugins = plugins.JvmPlugin
override def trigger: PluginTrigger = allRequirements
private def javaCppPresetDependencies: Def.Setting[Seq[ModuleID]] = {
import autoImport._
libraryDependencies ++= {
val (cppPresetVersion, groupId) = buildPresetVersion(javaCppVersion.value)
javaCppPresetLibs.value.flatMap {
case (libName, libVersion) =>
implicit class RegexOps(sc: StringContext) {
def r = new Regex(sc.parts.mkString, sc.parts.tail.map(_ => "x"): _*)
}
val (libNamePrefix, libNamePostfix) = libName match {
case r"([^-]+)$prefix(-.+)$postfix" => (prefix, postfix)
case _ => (libName, "")
}
val generic = groupId % libNamePrefix % s"$libVersion-$cppPresetVersion" classifier ""
val platformSpecific = javaCppPlatform.value.map { platform =>
groupId % libNamePrefix % s"$libVersion-$cppPresetVersion" classifier s"$platform$libNamePostfix"
}
generic +: platformSpecific
}
}
}
private def buildPresetVersion(version: String): (String, String) =
version match {
case VersionSplit(a :: b :: _) if a == 0 || (a == 1 && b <= 3) => (s"$a.$b", "org.bytedeco.javacpp-presets")
case VersionSplit(1 :: 4 :: _) => (version, "org.bytedeco.javacpp-presets")
case _ => (version, "org.bytedeco")
}
private object VersionSplit {
def unapply(arg: String): Option[List[Int]] =
Try(arg.split('.').map(_.toInt).toList).toOption
}
}
Example 175
| Source File: ExampleSpark.scala From hyperion with Apache License 2.0 | 5 votes |
|
package com.krux.hyperion.examples
import scala.language.postfixOps
import com.typesafe.config.ConfigFactory
import com.krux.hyperion.action.SnsAlarm
import com.krux.hyperion.activity.{EmrActivity, SparkStep, SparkTaskActivity}
import com.krux.hyperion.common.S3Uri
import com.krux.hyperion.datanode.S3DataNode
import com.krux.hyperion.expression.{Parameter, RuntimeNode}
import com.krux.hyperion.Implicits._
import com.krux.hyperion.resource.{EmrCluster, EmrApplication}
import com.krux.hyperion.{DataPipelineDef, HyperionCli, HyperionContext, Schedule}
object ExampleSpark extends DataPipelineDef with HyperionCli {
val target = "the-target"
val jar = s3 / "sample-jars" / "sample-jar-assembly-current.jar"
override implicit val hc: HyperionContext = new HyperionContext(ConfigFactory.load("example"))
override lazy val tags = Map("example" -> None, "ownerGroup" -> Some("spark"))
override lazy val schedule = Schedule.cron
.startAtActivation
.every(1.day)
.stopAfter(3)
val location = Parameter[S3Uri]("S3Location").withValue(s3"your-location")
val instanceType = Parameter[String]("InstanceType").withValue("c3.8xlarge")
val instanceCount = Parameter("InstanceCount", 8)
val instanceBid = Parameter("InstanceBid", 3.40)
override def parameters: Iterable[Parameter[_]] = Seq(location, instanceType, instanceCount, instanceBid)
val dataNode = S3DataNode(s3 / "some-bucket" / "some-path" /)
// Actions
val mailAction = SnsAlarm("arn:aws:sns:us-east-1:28619EXAMPLE:ExampleTopic")
.withSubject(s"Something happened at ${RuntimeNode.ScheduledStartTime}")
.withMessage(s"Some message $instanceCount x $instanceType @ $instanceBid for $location")
.withRole("DataPipelineDefaultResourceRole")
// Resources
val sparkCluster = EmrCluster()
.withApplications(EmrApplication.Spark)
.withTaskInstanceCount(instanceCount)
.withTaskInstanceType(instanceType)
.withInitTimeout(5.hours)
// First activity
val filterActivity = SparkTaskActivity(jar)(sparkCluster)
.withMainClass("com.krux.hyperion.FilterJob")
.named("filterActivity")
.onFail(mailAction)
.withInput(dataNode)
.withArguments(
target,
(EmrActivity.ScheduledStartTime - 3.days).format("yyyy-MM-dd")
)
// Second activity
val scoreStep1 = SparkStep(jar)
.withMainClass("com.krux.hyperion.ScoreJob1")
.withArguments(
target,
(EmrActivity.ScheduledStartTime - 3.days).format("yyyy-MM-dd"),
"denormalized"
)
val scoreStep2 = SparkStep(jar)
.withMainClass("com.krux.hyperion.ScoreJob2")
.withArguments(target, (EmrActivity.ScheduledStartTime - 3.days).format("yyyy-MM-dd"))
val scoreStep3 = SparkStep(jar)
.withMainClass("com.krux.hyperion.ScoreJob3")
.withArguments(
target,
(EmrActivity.ScheduledStartTime - 3.days).format("yyyy-MM-dd"),
"value1,value2"
)
val scoreStep4 = SparkStep(jar)
.withMainClass("com.krux.hyperion.ScoreJob4")
.withArguments(
target,
(EmrActivity.ScheduledStartTime - 3.days).format("yyyy-MM-dd"),
"value1,value2," + (EmrActivity.ScheduledStartTime - 3.days).format("yyyy-MM-dd")
)
val scoreActivity = EmrActivity(sparkCluster)
.named("scoreActivity")
.withSteps(scoreStep1, scoreStep2, scoreStep3, scoreStep4)
.onSuccess(mailAction)
override def workflow = filterActivity ~> scoreActivity
}
Example 176
| Source File: DriverActorTest.scala From sparta with Apache License 2.0 | 5 votes |
|
package com.stratio.sparta.serving.api.actor
import java.nio.file.{Files, Path}
import akka.actor.{ActorSystem, Props}
import akka.testkit.{DefaultTimeout, ImplicitSender, TestKit}
import akka.util.Timeout
import com.stratio.sparta.serving.api.actor.DriverActor.UploadDrivers
import com.stratio.sparta.serving.core.config.{SpartaConfig, SpartaConfigFactory}
import com.stratio.sparta.serving.core.models.SpartaSerializer
import com.stratio.sparta.serving.core.models.files.{SpartaFile, SpartaFilesResponse}
import com.typesafe.config.{Config, ConfigFactory}
import org.junit.runner.RunWith
import org.scalatest._
import org.scalatest.junit.JUnitRunner
import org.scalatest.mock.MockitoSugar
import spray.http.BodyPart
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.util.{Failure, Success}
@RunWith(classOf[JUnitRunner])
class DriverActorTest extends TestKit(ActorSystem("PluginActorSpec"))
with DefaultTimeout
with ImplicitSender
with WordSpecLike
with Matchers
with BeforeAndAfterAll
with BeforeAndAfterEach
with MockitoSugar with SpartaSerializer {
val tempDir: Path = Files.createTempDirectory("test")
tempDir.toFile.deleteOnExit()
val localConfig: Config = ConfigFactory.parseString(
s"""
|sparta{
| api {
| host = local
| port= 7777
| }
|}
|
|sparta.config.driverPackageLocation = "$tempDir"
""".stripMargin)
val fileList = Seq(BodyPart("reference.conf", "file"))
override def beforeEach(): Unit = {
SpartaConfig.initMainConfig(Option(localConfig), SpartaConfigFactory(localConfig))
SpartaConfig.initApiConfig()
}
override def afterAll: Unit = {
shutdown()
}
override implicit val timeout: Timeout = Timeout(15 seconds)
"DriverActor " must {
"Not save files with wrong extension" in {
val driverActor = system.actorOf(Props(new DriverActor()))
driverActor ! UploadDrivers(fileList)
expectMsgPF() {
case SpartaFilesResponse(Success(f: Seq[SpartaFile])) => f.isEmpty shouldBe true
}
}
"Not upload empty files" in {
val driverActor = system.actorOf(Props(new DriverActor()))
driverActor ! UploadDrivers(Seq.empty)
expectMsgPF() {
case SpartaFilesResponse(Failure(f)) => f.getMessage shouldBe "At least one file is expected"
}
}
"Save a file" in {
val driverActor = system.actorOf(Props(new DriverActor()))
driverActor ! UploadDrivers(Seq(BodyPart("reference.conf", "file.jar")))
expectMsgPF() {
case SpartaFilesResponse(Success(f: Seq[SpartaFile])) => f.head.fileName.endsWith("file.jar") shouldBe true
}
}
}
}
Example 177
| Source File: PluginActorTest.scala From sparta with Apache License 2.0 | 5 votes |
|
package com.stratio.sparta.serving.api.actor
import java.nio.file.{Files, Path}
import akka.actor.{ActorSystem, Props}
import akka.testkit.{DefaultTimeout, ImplicitSender, TestKit}
import akka.util.Timeout
import com.stratio.sparta.serving.api.actor.PluginActor.{PluginResponse, UploadPlugins}
import com.stratio.sparta.serving.api.constants.HttpConstant
import com.stratio.sparta.serving.core.config.{SpartaConfig, SpartaConfigFactory}
import com.stratio.sparta.serving.core.models.SpartaSerializer
import com.stratio.sparta.serving.core.models.files.{SpartaFile, SpartaFilesResponse}
import com.typesafe.config.{Config, ConfigFactory}
import org.junit.runner.RunWith
import org.scalatest._
import org.scalatest.junit.JUnitRunner
import org.scalatest.mock.MockitoSugar
import spray.http.BodyPart
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.util.{Failure, Success}
@RunWith(classOf[JUnitRunner])
class PluginActorTest extends TestKit(ActorSystem("PluginActorSpec"))
with DefaultTimeout
with ImplicitSender
with WordSpecLike
with Matchers
with BeforeAndAfterAll
with BeforeAndAfterEach
with MockitoSugar with SpartaSerializer {
val tempDir: Path = Files.createTempDirectory("test")
tempDir.toFile.deleteOnExit()
val localConfig: Config = ConfigFactory.parseString(
s"""
|sparta{
| api {
| host = local
| port= 7777
| }
|}
|
|sparta.config.pluginPackageLocation = "$tempDir"
""".stripMargin)
val fileList = Seq(BodyPart("reference.conf", "file"))
override def beforeEach(): Unit = {
SpartaConfig.initMainConfig(Option(localConfig), SpartaConfigFactory(localConfig))
SpartaConfig.initApiConfig()
}
override def afterAll: Unit = {
shutdown()
}
override implicit val timeout: Timeout = Timeout(15 seconds)
"PluginActor " must {
"Not save files with wrong extension" in {
val pluginActor = system.actorOf(Props(new PluginActor()))
pluginActor ! UploadPlugins(fileList)
expectMsgPF() {
case SpartaFilesResponse(Success(f: Seq[SpartaFile])) => f.isEmpty shouldBe true
}
}
"Not upload empty files" in {
val pluginActor = system.actorOf(Props(new PluginActor()))
pluginActor ! UploadPlugins(Seq.empty)
expectMsgPF() {
case SpartaFilesResponse(Failure(f)) => f.getMessage shouldBe "At least one file is expected"
}
}
"Save a file" in {
val pluginActor = system.actorOf(Props(new PluginActor()))
pluginActor ! UploadPlugins(Seq(BodyPart("reference.conf", "file.jar")))
expectMsgPF() {
case SpartaFilesResponse(Success(f: Seq[SpartaFile])) => f.head.fileName.endsWith("file.jar") shouldBe true
}
}
}
}
Example 178
| Source File: RabbitIntegrationSpec.scala From sparta with Apache License 2.0 | 5 votes |
|
package com.stratio.sparta.plugin.input.rabbitmq
import akka.actor.ActorSystem
import akka.event.slf4j.SLF4JLogging
import akka.util.Timeout
import com.typesafe.config.ConfigFactory
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
import org.scalatest.concurrent.TimeLimitedTests
import org.scalatest.time.{Minute, Span}
import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, Matchers, WordSpec}
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.util.Try
abstract class RabbitIntegrationSpec extends WordSpec with Matchers with SLF4JLogging with TimeLimitedTests
with BeforeAndAfter with BeforeAndAfterAll {
private lazy val config = ConfigFactory.load()
implicit val system = ActorSystem("ActorRabbitMQSystem")
implicit val timeout = Timeout(10 seconds)
val timeLimit = Span(1, Minute)
val RabbitTimeOut = 3 second
val configQueueName = Try(config.getString("rabbitmq.queueName")).getOrElse("rabbitmq-queue")
val configExchangeName = Try(config.getString("rabbitmq.exchangeName")).getOrElse("rabbitmq-exchange")
val exchangeType = Try(config.getString("rabbitmq.exchangeType")).getOrElse("topic")
val routingKey = Try(config.getString("rabbitmq.routingKey")).getOrElse("")
val vHost = Try(config.getString("rabbitmq.vHost")).getOrElse("/")
val hosts = Try(config.getString("rabbitmq.hosts")).getOrElse("127.0.0.1")
val userName = Try(config.getString("rabbitmq.userName")).getOrElse("guest")
val password = Try(config.getString("rabbitmq.password")).getOrElse("guest")
val RabbitConnectionURI = s"amqp://$userName:$password@$hosts/%2F"
var sc: Option[SparkContext] = None
var ssc: Option[StreamingContext] = None
def initSpark(): Unit = {
sc = Some(new SparkContext(conf))
ssc = Some(new StreamingContext(sc.get, Seconds(1)))
}
def stopSpark(): Unit = {
ssc.foreach(_.stop())
sc.foreach(_.stop())
System.gc()
}
def initRabbitMQ(): Unit
def closeRabbitMQ(): Unit
before {
log.info("Init spark")
initSpark()
log.info("Sending messages to queue..")
initRabbitMQ()
log.info("Messages in queue.")
}
after {
log.info("Stop spark")
stopSpark()
log.info("Clean rabbitmq")
closeRabbitMQ()
}
}
Example 179
| Source File: DoobieTests.scala From freestyle with Apache License 2.0 | 5 votes |
|
package freestyle.free
import cats.effect.IO
import cats.syntax.either._
import org.scalatest._
import _root_.doobie._
import _root_.doobie.implicits._
import _root_.doobie.h2.H2Transactor
import freestyle.free.implicits._
import freestyle.free.doobie._
import freestyle.free.doobie.implicits._
import scala.language.postfixOps
class DoobieTests extends AsyncWordSpec with Matchers {
import algebras._
implicit val xa: Transactor[IO] =
H2Transactor.newH2Transactor[IO]("jdbc:h2:mem:test;DB_CLOSE_DELAY=-1", "sa", "").unsafeRunSync
val query: ConnectionIO[Int] = sql"SELECT 1 + 1".query[Int].unique
"Doobie Freestyle integration" should {
"allow a doobie ConnectionIO program to be interleaved inside a program monadic flow" in {
val program = for {
a <- app.nonDoobie.x
b <- app.doobieM.transact(query).freeS
c <- FreeS.pure(1)
} yield a + b + c
program.interpret[IO] map { _ shouldBe 4 } unsafeToFuture
}
"allow doobie syntax to lift to FreeS" in {
val program: FreeS[App.Op, Int] = for {
a <- app.nonDoobie.x
b <- query.liftFS[App.Op]
c <- app.nonDoobie.x
} yield a + b + c
program.interpret[IO] map { _ shouldBe 4 } unsafeToFuture
}
"allow doobie syntax to lift to FreeS.Par" in {
val program: FreeS[App.Op, Int] = for {
a <- app.nonDoobie.x
b <- query.liftFSPar[App.Op].freeS
c <- app.nonDoobie.x
} yield a + b + c
program.interpret[IO] map { _ shouldBe 4 } unsafeToFuture
}
}
}
object algebras {
@free
trait NonDoobie {
def x: FS[Int]
}
implicit def nonDoobieHandler: NonDoobie.Handler[IO] =
new NonDoobie.Handler[IO] {
def x: IO[Int] = IO.pure(1)
}
@module
trait App {
val nonDoobie: NonDoobie
val doobieM: DoobieM
}
val app = App[App.Op]
}
Example 180
| Source File: JarUtils.scala From incubator-toree with Apache License 2.0 | 5 votes |
|
package org.apache.spark.toree.test.utils
import sys.process._
import java.net.URL
import java.io.File
import java.util.UUID
import scala.language.postfixOps
import org.apache.spark.TestUtils.{JavaSourceFromString, _}
object JarUtils {
def createTemporaryDir() = {
val tempDir = System.getProperty("java.io.tmpdir")
val dir = new File(tempDir, UUID.randomUUID.toString)
dir.mkdirs()
dir.deleteOnExit()
dir.getCanonicalFile
}
def createDummyJar(destDir: String, packageName: String, className: String) : URL = {
val srcDir = new File(destDir, packageName)
srcDir.mkdirs()
val source =
s"""package $packageName;
|
|public class $className implements java.io.Serializable {
| public static String sayHello(String arg) { return "Hello, " + arg; }
| public static int addStuff(int arg1, int arg2) { return arg1 + arg2; }
|}
""".stripMargin
val sourceFile =
new JavaSourceFromString(new File(srcDir, className).toURI.getPath, source)
val compiledFile = createCompiledClass(className, srcDir, sourceFile, Seq.empty)
val jarFile = new File(destDir,
s"$packageName-$className-%s.jar".format(System.currentTimeMillis()))
val jarURL = createJar(Seq(compiledFile), jarFile, directoryPrefix = Some(packageName))
jarFile.deleteOnExit()
jarURL
}
def downloadJar(destDir: String, artifactURL: String) : URL = {
val fileName = getFileName(artifactURL).
replace(".jar", s"%s.jar".format(System.currentTimeMillis()))
val jarFile = new File(destDir, fileName)
jarFile.deleteOnExit()
(new URL(artifactURL) #> jarFile !!)
jarFile.toURI.toURL
}
private def getFileName(artifactURL: String) = {
artifactURL.split("/").last
}
}
Example 181
| Source File: EchoActorTest.scala From 006877 with MIT License | 5 votes |
|
package aia.testdriven
import akka.testkit.{ TestKit, ImplicitSender }
import akka.actor.{ Props, Actor, ActorSystem }
import org.scalatest.WordSpecLike
import akka.util.Timeout
import scala.concurrent.Await
import scala.util.{ Success, Failure }
import scala.language.postfixOps
class EchoActorTest extends TestKit(ActorSystem("testsystem"))
with WordSpecLike
with ImplicitSender
with StopSystemAfterAll {
"An EchoActor" must {
"Reply with the same message it receives" in {
import akka.pattern.ask
import scala.concurrent.duration._
implicit val timeout = Timeout(3 seconds)
implicit val ec = system.dispatcher
val echo = system.actorOf(Props[EchoActor], "echo1")
val future = echo.ask("some message")
future.onComplete {
case Failure(_) => //실패 처리
case Success(msg) => //성공 처리
}
Await.ready(future, timeout.duration)
}
"Reply with the same message it receives without ask" in {
val echo = system.actorOf(Props[EchoActor], "echo2")
echo ! "some message"
expectMsg("some message")
}
}
}
class EchoActor extends Actor {
def receive = {
case msg =>
sender() ! msg
}
}
Example 182
| Source File: AggregatorTest.scala From 006877 with MIT License | 5 votes |
|
package aia.structure
import java.util.Date
import scala.concurrent.duration._
import akka.testkit._
import akka.actor._
import org.scalatest._
import scala.language.postfixOps
class AggregatorTest
extends TestKit(ActorSystem("AggregatorTest"))
with WordSpecLike with BeforeAndAfterAll {
val timeout = 2 seconds
protected override def afterAll(): Unit = {
system.terminate()
}
"The Agregator" must {
"aggregate two messages" in {
val endProbe = TestProbe()
val actorRef = system.actorOf(
Props(new Aggregator(timeout, endProbe.ref)))
val photoStr = ImageProcessing.createPhotoString(new Date(), 60)
val msg1 = PhotoMessage("id1",
photoStr,
Some(new Date()),
None)
actorRef ! msg1
val msg2 = PhotoMessage("id1",
photoStr,
None,
Some(60))
actorRef ! msg2
val combinedMsg = PhotoMessage("id1",
photoStr,
msg1.creationTime,
msg2.speed)
endProbe.expectMsg(combinedMsg)
}
"send message after timeout" in {
val endProbe = TestProbe()
val actorRef = system.actorOf(
Props(new Aggregator(timeout, endProbe.ref)))
val photoStr = ImageProcessing.createPhotoString(
new Date(), 60)
val msg1 = PhotoMessage("id1",
photoStr,
Some(new Date()),
None)
actorRef ! msg1
endProbe.expectMsg(msg1)
}
"aggregate two messages when restarting" in {
val endProbe = TestProbe()
val actorRef = system.actorOf(
Props(new Aggregator(timeout, endProbe.ref)))
val photoStr = ImageProcessing.createPhotoString(new Date(), 60)
val msg1 = PhotoMessage("id1",
photoStr,
Some(new Date()),
None)
actorRef ! msg1
actorRef ! new IllegalStateException("restart")
val msg2 = PhotoMessage("id1",
photoStr,
None,
Some(60))
actorRef ! msg2
val combinedMsg = PhotoMessage("id1",
photoStr,
msg1.creationTime,
msg2.speed)
endProbe.expectMsg(combinedMsg)
}
}
}
Example 183
| Source File: ScatterGatherTest.scala From 006877 with MIT License | 5 votes |
|
package aia.structure
import java.util.Date
import scala.concurrent.duration._
import akka.actor._
import org.scalatest._
import akka.testkit._
import scala.language.postfixOps
class ScatterGatherTest
extends TestKit(ActorSystem("ScatterGatherTest"))
with WordSpecLike
with BeforeAndAfterAll {
val timeout = 2 seconds
override def afterAll(): Unit = {
system.terminate()
}
"The ScatterGather" must {
"scatter the message and gather them again" in {
val endProbe = TestProbe()
val aggregateRef = system.actorOf(
Props(new Aggregator(timeout, endProbe.ref)))
val speedRef = system.actorOf(
Props(new GetSpeed(aggregateRef)))
val timeRef = system.actorOf(
Props(new GetTime(aggregateRef)))
val actorRef = system.actorOf(
Props(new RecipientList(Seq(speedRef, timeRef))))
val photoDate = new Date()
val photoSpeed = 60
val msg = PhotoMessage("id1",
ImageProcessing.createPhotoString(photoDate, photoSpeed))
actorRef ! msg
val combinedMsg = PhotoMessage(msg.id,
msg.photo,
Some(photoDate),
Some(photoSpeed))
endProbe.expectMsg(combinedMsg)
}
}
}
Example 184
| Source File: PipeAndFilterTest.scala From 006877 with MIT License | 5 votes |
|
package aia.structure
import scala.concurrent.duration._
import akka.actor._
import org.scalatest._
import akka.testkit._
import scala.language.postfixOps
class PipeAndFilterTest
extends TestKit(ActorSystem("PipeAndFilterTest"))
with WordSpecLike
with BeforeAndAfterAll {
val timeout = 2 seconds
override def afterAll(): Unit = {
system.terminate()
}
"The pipe and filter" must {
"filter messages in configuration 1" in {
val endProbe = TestProbe()
val speedFilterRef = system.actorOf(
Props(new SpeedFilter(50, endProbe.ref)))
val licenseFilterRef = system.actorOf(
Props(new LicenseFilter(speedFilterRef)))
val msg = new Photo("123xyz", 60)
licenseFilterRef ! msg
endProbe.expectMsg(msg)
licenseFilterRef ! new Photo("", 60)
endProbe.expectNoMsg(timeout)
licenseFilterRef ! new Photo("123xyz", 49)
endProbe.expectNoMsg(timeout)
}
"filter messages in configuration 2" in {
val endProbe = TestProbe()
val licenseFilterRef = system.actorOf(
Props(new LicenseFilter(endProbe.ref)))
val speedFilterRef = system.actorOf(
Props(new SpeedFilter(50, licenseFilterRef)))
val msg = new Photo("123xyz", 60)
speedFilterRef ! msg
endProbe.expectMsg(msg)
speedFilterRef ! new Photo("", 60)
endProbe.expectNoMsg(timeout)
speedFilterRef ! new Photo("123xyz", 49)
endProbe.expectNoMsg(timeout)
}
}
}
Example 185
| Source File: SessionHeartbeatSpec.scala From incubator-livy with Apache License 2.0 | 5 votes |
|
package org.apache.livy.server.interactive
import scala.concurrent.duration._
import scala.concurrent.Future
import scala.language.postfixOps
import org.mockito.Mockito.{never, verify, when}
import org.scalatest.{FunSpec, Matchers}
import org.scalatest.concurrent.Eventually._
import org.scalatest.mock.MockitoSugar.mock
import org.apache.livy.LivyConf
import org.apache.livy.server.recovery.SessionStore
import org.apache.livy.sessions.{Session, SessionManager}
import org.apache.livy.sessions.Session.RecoveryMetadata
class SessionHeartbeatSpec extends FunSpec with Matchers {
describe("SessionHeartbeat") {
class TestHeartbeat(override val heartbeatTimeout: FiniteDuration) extends SessionHeartbeat {}
it("should not expire if heartbeat was never called.") {
val t = new TestHeartbeat(Duration.Zero)
t.heartbeatExpired shouldBe false
}
it("should expire if time has elapsed.") {
val t = new TestHeartbeat(Duration.fromNanos(1))
t.heartbeat()
eventually(timeout(2 nano), interval(1 nano)) {
t.heartbeatExpired shouldBe true
}
}
it("should not expire if time hasn't elapsed.") {
val t = new TestHeartbeat(Duration.create(1, DAYS))
t.heartbeat()
t.heartbeatExpired shouldBe false
}
}
describe("SessionHeartbeatWatchdog") {
abstract class TestSession
extends Session(0, None, null, null) with SessionHeartbeat {}
class TestWatchdog(conf: LivyConf)
extends SessionManager[TestSession, RecoveryMetadata](
conf,
{ _ => assert(false).asInstanceOf[TestSession] },
mock[SessionStore],
"test",
Some(Seq.empty))
with SessionHeartbeatWatchdog[TestSession, RecoveryMetadata] {}
it("should delete only expired sessions") {
val expiredSession: TestSession = mock[TestSession]
when(expiredSession.id).thenReturn(0)
when(expiredSession.name).thenReturn(None)
when(expiredSession.heartbeatExpired).thenReturn(true)
when(expiredSession.stop()).thenReturn(Future.successful(()))
when(expiredSession.lastActivity).thenReturn(System.nanoTime())
val nonExpiredSession: TestSession = mock[TestSession]
when(nonExpiredSession.id).thenReturn(1)
when(nonExpiredSession.name).thenReturn(None)
when(nonExpiredSession.heartbeatExpired).thenReturn(false)
when(nonExpiredSession.stop()).thenReturn(Future.successful(()))
when(nonExpiredSession.lastActivity).thenReturn(System.nanoTime())
val n = new TestWatchdog(new LivyConf())
n.register(expiredSession)
n.register(nonExpiredSession)
n.deleteExpiredSessions()
verify(expiredSession).stop()
verify(nonExpiredSession, never).stop()
}
}
}
Example 186
| Source File: ReplDriverSuite.scala From incubator-livy with Apache License 2.0 | 5 votes |
|
package org.apache.livy.repl
import java.net.URI
import java.util.concurrent.TimeUnit
import scala.concurrent.duration._
import scala.language.postfixOps
import org.apache.spark.launcher.SparkLauncher
import org.json4s._
import org.json4s.jackson.JsonMethods._
import org.scalatest.FunSuite
import org.scalatest.concurrent.Eventually._
import org.apache.livy._
import org.apache.livy.rsc.{PingJob, RSCClient, RSCConf}
import org.apache.livy.sessions.Spark
class ReplDriverSuite extends FunSuite with LivyBaseUnitTestSuite {
private implicit val formats = DefaultFormats
test("start a repl session using the rsc") {
val client = new LivyClientBuilder()
.setConf(SparkLauncher.DRIVER_MEMORY, "512m")
.setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, sys.props("java.class.path"))
.setConf(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH, sys.props("java.class.path"))
.setConf(RSCConf.Entry.LIVY_JARS.key(), "")
.setURI(new URI("rsc:/"))
.setConf(RSCConf.Entry.DRIVER_CLASS.key(), classOf[ReplDriver].getName())
.setConf(RSCConf.Entry.SESSION_KIND.key(), Spark.toString)
.build()
.asInstanceOf[RSCClient]
try {
// This is sort of what InteractiveSession.scala does to detect an idle session.
client.submit(new PingJob()).get(60, TimeUnit.SECONDS)
val statementId = client.submitReplCode("1 + 1", "spark").get
eventually(timeout(30 seconds), interval(100 millis)) {
val rawResult =
client.getReplJobResults(statementId, 1).get(10, TimeUnit.SECONDS).statements(0)
val result = rawResult.output
assert((parse(result) \ Session.STATUS).extract[String] === Session.OK)
}
} finally {
client.stop(true)
}
}
}
Example 187
| Source File: BaseSessionSpec.scala From incubator-livy with Apache License 2.0 | 5 votes |
|
package org.apache.livy.repl
import java.util.Properties
import java.util.concurrent.atomic.AtomicInteger
import scala.concurrent.Await
import scala.concurrent.duration._
import scala.language.postfixOps
import org.apache.spark.SparkConf
import org.json4s._
import org.scalatest.{FlatSpec, Matchers}
import org.scalatest.concurrent.Eventually._
import org.apache.livy.LivyBaseUnitTestSuite
import org.apache.livy.rsc.RSCConf
import org.apache.livy.rsc.driver.{Statement, StatementState}
import org.apache.livy.sessions._
abstract class BaseSessionSpec(kind: Kind)
extends FlatSpec with Matchers with LivyBaseUnitTestSuite {
implicit val formats = DefaultFormats
private val rscConf = new RSCConf(new Properties()).set(RSCConf.Entry.SESSION_KIND, kind.toString)
private val sparkConf = new SparkConf()
protected def execute(session: Session)(code: String): Statement = {
val id = session.execute(code)
eventually(timeout(30 seconds), interval(100 millis)) {
val s = session.statements(id)
s.state.get() shouldBe StatementState.Available
s
}
}
protected def withSession(testCode: Session => Any): Unit = {
val stateChangedCalled = new AtomicInteger()
val session =
new Session(rscConf, sparkConf, None, { _ => stateChangedCalled.incrementAndGet() })
try {
// Session's constructor should fire an initial state change event.
stateChangedCalled.intValue() shouldBe 1
Await.ready(session.start(), 30 seconds)
assert(session.state === SessionState.Idle)
// There should be at least 1 state change event fired when session transits to idle.
stateChangedCalled.intValue() should (be > 1)
testCode(session)
} finally {
session.close()
}
}
it should "start in the starting or idle state" in {
val session = new Session(rscConf, sparkConf)
val future = session.start()
try {
Await.ready(future, 60 seconds)
session.state should (equal (SessionState.Starting) or equal (SessionState.Idle))
} finally {
session.close()
}
}
it should "eventually become the idle state" in withSession { session =>
session.state should equal (SessionState.Idle)
}
}
Example 188
| Source File: SignInController.scala From play-silhouette-reactivemongo-seed with Apache License 2.0 | 5 votes |
|
package controllers
import javax.inject.Inject
import com.mohiva.play.silhouette.api.Authenticator.Implicits._
import com.mohiva.play.silhouette.api._
import com.mohiva.play.silhouette.api.exceptions.ProviderException
import com.mohiva.play.silhouette.api.repositories.AuthInfoRepository
import com.mohiva.play.silhouette.api.util.{ Clock, Credentials }
import com.mohiva.play.silhouette.impl.exceptions.IdentityNotFoundException
import com.mohiva.play.silhouette.impl.providers._
import forms.SignInForm
import models.services.UserService
import net.ceedubs.ficus.Ficus._
import play.api.Configuration
import play.api.i18n.{ I18nSupport, Messages, MessagesApi }
import play.api.libs.concurrent.Execution.Implicits._
import play.api.mvc.Controller
import utils.auth.DefaultEnv
import scala.concurrent.Future
import scala.concurrent.duration._
import scala.language.postfixOps
def submit = silhouette.UnsecuredAction.async { implicit request =>
SignInForm.form.bindFromRequest.fold(
form => Future.successful(BadRequest(views.html.signIn(form, socialProviderRegistry))),
data => {
val credentials = Credentials(data.email, data.password)
credentialsProvider.authenticate(credentials).flatMap { loginInfo =>
val result = Redirect(routes.ApplicationController.index())
userService.retrieve(loginInfo).flatMap {
case Some(user) if !user.activated =>
Future.successful(Ok(views.html.activateAccount(data.email)))
case Some(user) =>
val c = configuration.underlying
silhouette.env.authenticatorService.create(loginInfo).map {
case authenticator if data.rememberMe =>
authenticator.copy(
expirationDateTime = clock.now + c.as[FiniteDuration]("silhouette.authenticator.rememberMe.authenticatorExpiry"),
idleTimeout = c.getAs[FiniteDuration]("silhouette.authenticator.rememberMe.authenticatorIdleTimeout"),
cookieMaxAge = c.getAs[FiniteDuration]("silhouette.authenticator.rememberMe.cookieMaxAge")
)
case authenticator => authenticator
}.flatMap { authenticator =>
silhouette.env.eventBus.publish(LoginEvent(user, request))
silhouette.env.authenticatorService.init(authenticator).flatMap { v =>
silhouette.env.authenticatorService.embed(v, result)
}
}
case None => Future.failed(new IdentityNotFoundException("Couldn't find user"))
}
}.recover {
case e: ProviderException =>
Redirect(routes.SignInController.view()).flashing("error" -> Messages("invalid.credentials"))
}
}
)
}
}
Example 189
| Source File: ActivateAccountController.scala From play-silhouette-reactivemongo-seed with Apache License 2.0 | 5 votes |
|
package controllers
import java.net.URLDecoder
import java.util.UUID
import javax.inject.Inject
import com.mohiva.play.silhouette.api._
import com.mohiva.play.silhouette.impl.providers.CredentialsProvider
import models.services.{ AuthTokenService, UserService }
import play.api.i18n.{ I18nSupport, Messages, MessagesApi }
import play.api.libs.concurrent.Execution.Implicits._
import play.api.libs.mailer.{ Email, MailerClient }
import play.api.mvc.Controller
import utils.auth.DefaultEnv
import scala.concurrent.Future
import scala.language.postfixOps
def activate(token: UUID) = silhouette.UnsecuredAction.async { implicit request =>
authTokenService.validate(token).flatMap {
case Some(authToken) => userService.retrieve(authToken.userID).flatMap {
case Some(user) if user.loginInfo.providerID == CredentialsProvider.ID =>
userService.save(user.copy(activated = true)).map { _ =>
Redirect(routes.SignInController.view()).flashing("success" -> Messages("account.activated"))
}
case _ => Future.successful(Redirect(routes.SignInController.view()).flashing("error" -> Messages("invalid.activation.link")))
}
case None => Future.successful(Redirect(routes.SignInController.view()).flashing("error" -> Messages("invalid.activation.link")))
}
}
}
Example 190
| Source File: KafkaStreamSuite.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.kafka
import scala.collection.mutable
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.util.Random
import kafka.serializer.StringDecoder
import org.scalatest.BeforeAndAfterAll
import org.scalatest.concurrent.Eventually
import org.apache.spark.{SparkConf, SparkFunSuite}
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Milliseconds, StreamingContext}
class KafkaStreamSuite extends SparkFunSuite with Eventually with BeforeAndAfterAll {
private var ssc: StreamingContext = _
private var kafkaTestUtils: KafkaTestUtils = _
override def beforeAll(): Unit = {
kafkaTestUtils = new KafkaTestUtils
kafkaTestUtils.setup()
}
override def afterAll(): Unit = {
if (ssc != null) {
ssc.stop()
ssc = null
}
if (kafkaTestUtils != null) {
kafkaTestUtils.teardown()
kafkaTestUtils = null
}
}
test("Kafka input stream") {
val sparkConf = new SparkConf().setMaster("local[4]").setAppName(this.getClass.getSimpleName)
ssc = new StreamingContext(sparkConf, Milliseconds(500))
val topic = "topic1"
val sent = Map("a" -> 5, "b" -> 3, "c" -> 10)
kafkaTestUtils.createTopic(topic)
kafkaTestUtils.sendMessages(topic, sent)
val kafkaParams = Map("zookeeper.connect" -> kafkaTestUtils.zkAddress,
"group.id" -> s"test-consumer-${Random.nextInt(10000)}",
"auto.offset.reset" -> "smallest")
val stream = KafkaUtils.createStream[String, String, StringDecoder, StringDecoder](
ssc, kafkaParams, Map(topic -> 1), StorageLevel.MEMORY_ONLY)
val result = new mutable.HashMap[String, Long]()
stream.map(_._2).countByValue().foreachRDD { r =>
r.collect().foreach { kv =>
result.synchronized {
val count = result.getOrElseUpdate(kv._1, 0) + kv._2
result.put(kv._1, count)
}
}
}
ssc.start()
eventually(timeout(10000 milliseconds), interval(100 milliseconds)) {
assert(result.synchronized { sent === result })
}
}
}
Example 191
| Source File: FlumeStreamSuite.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.flume
import java.util.concurrent.ConcurrentLinkedQueue
import scala.collection.JavaConverters._
import scala.concurrent.duration._
import scala.language.postfixOps
import org.jboss.netty.channel.ChannelPipeline
import org.jboss.netty.channel.socket.SocketChannel
import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory
import org.jboss.netty.handler.codec.compression._
import org.scalatest.{BeforeAndAfter, Matchers}
import org.scalatest.concurrent.Eventually._
import org.apache.spark.{SparkConf, SparkFunSuite}
import org.apache.spark.internal.Logging
import org.apache.spark.network.util.JavaUtils
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Milliseconds, StreamingContext, TestOutputStream}
class FlumeStreamSuite extends SparkFunSuite with BeforeAndAfter with Matchers with Logging {
val conf = new SparkConf().setMaster("local[4]").setAppName("FlumeStreamSuite")
var ssc: StreamingContext = null
test("flume input stream") {
testFlumeStream(testCompression = false)
}
test("flume input compressed stream") {
testFlumeStream(testCompression = true)
}
private class CompressionChannelFactory(compressionLevel: Int)
extends NioClientSocketChannelFactory {
override def newChannel(pipeline: ChannelPipeline): SocketChannel = {
val encoder = new ZlibEncoder(compressionLevel)
pipeline.addFirst("deflater", encoder)
pipeline.addFirst("inflater", new ZlibDecoder())
super.newChannel(pipeline)
}
}
}
Example 192
| Source File: StatusTrackerSuite.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark
import scala.concurrent.duration._
import scala.language.implicitConversions
import scala.language.postfixOps
import org.scalatest.Matchers
import org.scalatest.concurrent.Eventually._
import org.apache.spark.JobExecutionStatus._
class StatusTrackerSuite extends SparkFunSuite with Matchers with LocalSparkContext {
test("basic status API usage") {
sc = new SparkContext("local", "test", new SparkConf(false))
val jobFuture = sc.parallelize(1 to 10000, 2).map(identity).groupBy(identity).collectAsync()
val jobId: Int = eventually(timeout(10 seconds)) {
val jobIds = jobFuture.jobIds
jobIds.size should be(1)
jobIds.head
}
val jobInfo = eventually(timeout(10 seconds)) {
sc.statusTracker.getJobInfo(jobId).get
}
jobInfo.status() should not be FAILED
val stageIds = jobInfo.stageIds()
stageIds.size should be(2)
val firstStageInfo = eventually(timeout(10 seconds)) {
sc.statusTracker.getStageInfo(stageIds(0)).get
}
firstStageInfo.stageId() should be(stageIds(0))
firstStageInfo.currentAttemptId() should be(0)
firstStageInfo.numTasks() should be(2)
eventually(timeout(10 seconds)) {
val updatedFirstStageInfo = sc.statusTracker.getStageInfo(stageIds(0)).get
updatedFirstStageInfo.numCompletedTasks() should be(2)
updatedFirstStageInfo.numActiveTasks() should be(0)
updatedFirstStageInfo.numFailedTasks() should be(0)
}
}
test("getJobIdsForGroup()") {
sc = new SparkContext("local", "test", new SparkConf(false))
// Passing `null` should return jobs that were not run in a job group:
val defaultJobGroupFuture = sc.parallelize(1 to 1000).countAsync()
val defaultJobGroupJobId = eventually(timeout(10 seconds)) {
defaultJobGroupFuture.jobIds.head
}
eventually(timeout(10 seconds)) {
sc.statusTracker.getJobIdsForGroup(null).toSet should be (Set(defaultJobGroupJobId))
}
// Test jobs submitted in job groups:
sc.setJobGroup("my-job-group", "description")
sc.statusTracker.getJobIdsForGroup("my-job-group") should be (Seq.empty)
val firstJobFuture = sc.parallelize(1 to 1000).countAsync()
val firstJobId = eventually(timeout(10 seconds)) {
firstJobFuture.jobIds.head
}
eventually(timeout(10 seconds)) {
sc.statusTracker.getJobIdsForGroup("my-job-group") should be (Seq(firstJobId))
}
val secondJobFuture = sc.parallelize(1 to 1000).countAsync()
val secondJobId = eventually(timeout(10 seconds)) {
secondJobFuture.jobIds.head
}
eventually(timeout(10 seconds)) {
sc.statusTracker.getJobIdsForGroup("my-job-group").toSet should be (
Set(firstJobId, secondJobId))
}
}
test("getJobIdsForGroup() with takeAsync()") {
sc = new SparkContext("local", "test", new SparkConf(false))
sc.setJobGroup("my-job-group2", "description")
sc.statusTracker.getJobIdsForGroup("my-job-group2") shouldBe empty
val firstJobFuture = sc.parallelize(1 to 1000, 1).takeAsync(1)
val firstJobId = eventually(timeout(10 seconds)) {
firstJobFuture.jobIds.head
}
eventually(timeout(10 seconds)) {
sc.statusTracker.getJobIdsForGroup("my-job-group2") should be (Seq(firstJobId))
}
}
test("getJobIdsForGroup() with takeAsync() across multiple partitions") {
sc = new SparkContext("local", "test", new SparkConf(false))
sc.setJobGroup("my-job-group2", "description")
sc.statusTracker.getJobIdsForGroup("my-job-group2") shouldBe empty
val firstJobFuture = sc.parallelize(1 to 1000, 2).takeAsync(999)
val firstJobId = eventually(timeout(10 seconds)) {
firstJobFuture.jobIds.head
}
eventually(timeout(10 seconds)) {
sc.statusTracker.getJobIdsForGroup("my-job-group2") should have size 2
}
}
}
Example 193
| Source File: LauncherBackendSuite.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.launcher
import java.util.concurrent.TimeUnit
import scala.concurrent.duration._
import scala.language.postfixOps
import org.scalatest.Matchers
import org.scalatest.concurrent.Eventually._
import org.apache.spark._
class LauncherBackendSuite extends SparkFunSuite with Matchers {
private val tests = Seq(
"local" -> "local",
"standalone/client" -> "local-cluster[1,1,1024]")
tests.foreach { case (name, master) =>
test(s"$name: launcher handle") {
testWithMaster(master)
}
}
private def testWithMaster(master: String): Unit = {
val env = new java.util.HashMap[String, String]()
env.put("SPARK_PRINT_LAUNCH_COMMAND", "1")
val handle = new SparkLauncher(env)
.setSparkHome(sys.props("spark.test.home"))
.setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, System.getProperty("java.class.path"))
.setConf("spark.ui.enabled", "false")
.setConf(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, s"-Dtest.appender=console")
.setMaster(master)
.setAppResource(SparkLauncher.NO_RESOURCE)
.setMainClass(TestApp.getClass.getName().stripSuffix("$"))
.startApplication()
try {
eventually(timeout(30 seconds), interval(100 millis)) {
handle.getAppId() should not be (null)
}
handle.stop()
eventually(timeout(30 seconds), interval(100 millis)) {
handle.getState() should be (SparkAppHandle.State.KILLED)
}
} finally {
handle.kill()
}
}
}
object TestApp {
def main(args: Array[String]): Unit = {
new SparkContext(new SparkConf()).parallelize(Seq(1)).foreach { i =>
Thread.sleep(TimeUnit.SECONDS.toMillis(20))
}
}
}
Example 194
| Source File: StreamingContextUtils.scala From sscheck with Apache License 2.0 | 5 votes |
|
package es.ucm.fdi.sscheck.spark.streaming
import scala.concurrent._
import scala.concurrent.duration._
import scala.language.postfixOps
import ExecutionContext.Implicits.global
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.scheduler.{StreamingListener, StreamingListenerReceiverStarted, StreamingListenerBatchCompleted}
// Inline alternative implementation based on SyncVar
object StreamingContextUtils {
def awaitForNBatchesCompleted(numBatches : Int, atMost : scala.concurrent.duration.Duration = 30 seconds)
(ssc : StreamingContext) : Unit = {
val onBatchCompletedSyncVar = new SyncVar[Unit]
ssc.addStreamingListener(new StreamingListener {
override def onBatchCompleted(batchCompleted: StreamingListenerBatchCompleted) : Unit = {
if (! onBatchCompletedSyncVar.isSet) {
// note only this threads makes puts, so no problem with concurrency
onBatchCompletedSyncVar.put(())
}
}
})
val waitingForBatches = Future {
for (_ <- 1 to numBatches) {
onBatchCompletedSyncVar.take()
}
}
Await.result(waitingForBatches, atMost)
}
}
Example 195
| Source File: ReGen.scala From sscheck with Apache License 2.0 | 5 votes |
|
package es.ucm.fdi.sscheck.gen
import org.scalacheck.Gen
import es.ucm.fdi.sscheck.gen.UtilsGen.containerOfNtoM;
import Buildables.buildableSeq
import UtilsGen.{containerOfNtoM}
import scala.language.postfixOps
object ReGen {
val epsilon = Gen.const(List())
def symbol[A](s : A) : Gen[Seq[A]] = Gen.const(List(s))
// def alt[A](g1 : Gen[Seq[A]], g2 : Gen[Seq[A]]) : Gen[Seq[A]] = Gen.oneOf(g1, g2)
def alt[A](gs : Gen[Seq[A]]*) : Gen[Seq[A]] = {
val l = gs.length
// require (l > 0, "alt needs at least one alternative")
if (l == 0)
epsilon
else if (l == 1)
gs(0)
else
Gen.oneOf(gs(0), gs(1), gs.slice(2, l):_*)
}
def conc[A](g1 : Gen[Seq[A]], g2 : Gen[Seq[A]]) : Gen[Seq[A]] = {
for {
xs <- g1
ys <- g2
} yield xs ++ ys
}
def star[A](g : Gen[Seq[A]]) : Gen[Seq[A]] = {
for {
xs <- Gen.containerOf(g)
} yield xs flatten
}
}
Example 196
| Source File: ReportAndLogSupport.scala From RTran with Apache License 2.0 | 5 votes |
|
package com.ebay.rtran.report
import java.io.{File, FileOutputStream}
import ch.qos.logback.classic
import ch.qos.logback.classic.encoder.PatternLayoutEncoder
import ch.qos.logback.classic.filter.ThresholdFilter
import ch.qos.logback.classic.spi.ILoggingEvent
import ch.qos.logback.core.FileAppender
import org.reflections.Reflections
import com.ebay.rtran.report.api.IReportEventSubscriber
import org.slf4j.{Logger, LoggerFactory}
import scala.collection.JavaConversions._
import scala.language.postfixOps
import scala.util.{Success, Try}
trait ReportAndLogSupport {
val reportFilePrefix: String
val warnLogPrefix: String
val debugLogPrefix: String
def createReportAndLogs[T](projectRoot: File,
taskId: Option[String], packages: String*)(fn: => T): T = {
val appenders = prepareAppenders(projectRoot, taskId)
val rootLogger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME).asInstanceOf[classic.Logger]
appenders foreach rootLogger.addAppender
appenders foreach (_.start)
val reportFile = new File(projectRoot, s"$reportFilePrefix${taskId.map("-" + _) getOrElse ""}.md")
val result = Report.createReport(
new FileOutputStream(reportFile),
subscribers = allSubscribers(projectRoot, packages: _*)
)(fn)
appenders foreach (_.stop)
appenders foreach rootLogger.detachAppender
result
}
def allSubscribers(projectRoot: File, packages: String*) = {
val subscribers = packages flatMap {prefix =>
new Reflections(prefix).getSubTypesOf(classOf[IReportEventSubscriber[_]])
} map {clazz =>
Try(clazz.getDeclaredConstructor(classOf[File]).newInstance(projectRoot)) orElse Try(clazz.newInstance)
} collect {
case Success(subscriber) => subscriber
} toList
subscribers.sortBy(_.sequence)
}
private def prepareAppenders(projectRoot: File, taskId: Option[String]) = {
val lc = LoggerFactory.getILoggerFactory.asInstanceOf[classic.LoggerContext]
val encoders = Array(new PatternLayoutEncoder, new PatternLayoutEncoder)
encoders foreach (_ setContext lc)
encoders foreach (_ setPattern "%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}.%M:%L - %m%n")
encoders foreach (_.start)
val warnFileAppender = new FileAppender[ILoggingEvent]
warnFileAppender.setName("warnFileAppender")
warnFileAppender.setFile(s"${projectRoot.getAbsolutePath}/$warnLogPrefix${taskId.map("-" + _) getOrElse ""}.log")
warnFileAppender.addFilter(new SameThreadFilter)
val warnFilter = new ThresholdFilter
warnFilter.setLevel("WARN")
warnFilter.start()
warnFileAppender.addFilter(warnFilter)
val debugFileAppender = new FileAppender[ILoggingEvent]
debugFileAppender.setName("debugFileAppender")
debugFileAppender.setFile(s"${projectRoot.getAbsolutePath}/$debugLogPrefix${taskId.map("-" + _) getOrElse ""}.log")
debugFileAppender.addFilter(new SameThreadFilter)
val debugFilter = new ThresholdFilter
debugFilter.setLevel("DEBUG")
debugFilter.start()
debugFileAppender.addFilter(debugFilter)
val result = List(warnFileAppender, debugFileAppender)
result.foreach(_ setContext lc)
result zip encoders foreach (entry => entry._1 setEncoder entry._2)
result
}
}
Example 197
| Source File: ExampleBank.scala From affinity with Apache License 2.0 | 5 votes |
|
package io.amient.affinity.example
import akka.http.scaladsl.model.HttpMethods
import akka.util.Timeout
import io.amient.affinity.avro.record.{AvroRecord, Fixed}
import io.amient.affinity.core.ack
import io.amient.affinity.core.actor.{GatewayHttp, GatewayStream, Partition, Routed}
import io.amient.affinity.core.http.RequestMatchers.{HTTP, INT, PATH, QUERY}
import io.amient.affinity.core.storage.Record
import io.amient.affinity.core.util._
import scala.concurrent.duration._
import scala.language.postfixOps
case class Account(sortcode: String, number: Int) extends AvroRecord
case class Transaction(id: Long, amount: Double, timestamp: Long) extends AvroRecord with EventTime {
override def eventTimeUnix() = timestamp
}
class ExampleBank extends GatewayStream with GatewayHttp {
implicit val executor = context.dispatcher
implicit val scheduler = context.system.scheduler
implicit val timeout = Timeout(5 seconds)
val defaultKeyspace = keyspace("default")
input[Account, Transaction]("input-stream") { record: Record[Account, Transaction] =>
defaultKeyspace ?! StoreTransaction(record.key, record.value)
}
override def handle: Receive = {
case HTTP(HttpMethods.GET, PATH("transactions", sortcode, INT(number)), _, response) =>
defaultKeyspace ?! GetAccountTransactions(Account(sortcode, number)) map (handleAsJson(response, _))
case HTTP(HttpMethods.GET, PATH("transactions", sortcode), QUERY(("before", before)), response) =>
defaultKeyspace ?? GetBranchTransactions(sortcode, EventTime.unix(before+"T00:00:00+00:00")) map (handleAsJson(response, _))
case HTTP(HttpMethods.GET, PATH("transactions", sortcode), _, response) =>
defaultKeyspace ?? GetBranchTransactions(sortcode) map (handleAsJson(response, _))
}
}
case class StoreTransaction(key: Account, t: Transaction) extends AvroRecord with Routed with Reply[Option[Transaction]]
case class StorageKey(@Fixed(8) sortcode: String, @Fixed account: Int, txn: Long) extends AvroRecord
case class GetAccountTransactions(key: Account) extends AvroRecord with Routed with Reply[Seq[Transaction]]
case class GetBranchTransactions(sortcode: String, beforeUnixTs: Long = Long.MaxValue) extends AvroRecord with ScatterIterable[Transaction]
class DefaultPartition extends Partition {
val transactions = state[StorageKey, Transaction]("transactions")
implicit val executor = context.dispatcher
override def handle: Receive = {
case request@StoreTransaction(Account(sortcode, number), transaction) => request(sender) ! {
transactions.replace(StorageKey(sortcode, number, transaction.id), transaction)
} map {
_ => context.system.eventStream.publish(request) //this is only to have determinist way of testing all data was processed
}
case request@GetBranchTransactions(sortcode, before) =>
request(sender) ! transactions.range(TimeRange.until(before), sortcode).values.toList
case request@GetAccountTransactions(account) => request(sender) ! {
transactions.range(TimeRange.UNBOUNDED, account.sortcode, account.number).values.toList
}
}
}
Example 198
| Source File: ExampleHttpsGatewaySpec.scala From affinity with Apache License 2.0 | 5 votes |
|
import akka.util.Timeout
import com.typesafe.config.ConfigFactory
import io.amient.affinity.core.ack
import io.amient.affinity.core.cluster.Node
import io.amient.affinity.core.util.AffinityTestBase
import org.scalatest.{FlatSpec, Matchers}
import scala.concurrent.Await
import scala.concurrent.duration._
import scala.language.postfixOps
class ExampleHttpsGatewaySpec extends FlatSpec with AffinityTestBase with Matchers {
behavior of "Simple Api Gateway"
it should "work with multiple listeners" in {
val config = ConfigFactory.load("https-example")
val node = new Node(config)
node.start()
try {
node.awaitClusterReady
node.get_json(node.https_get("/secure/ping", 0)).asText() should be ("pong")
node.get_json(node.http_get("/unsafe/ping", 1)).asText() should be ("pong")
} finally {
node.shutdown()
}
}
it should "work without the http layer" in {
val config = ConfigFactory.load("https-example-test")
val node = new Node(config)
node.start()
try {
node.awaitClusterReady
implicit val scheduler = node.system.scheduler
implicit val context = node.system.dispatcher
implicit val timeout = Timeout(3 seconds)
Await.result(node.gateway ?! GetData("key1"), 3 seconds) should be(None)
Await.result(node.gateway ?! PutData("key1", "value1"), 3 seconds) should be(None)
Await.result(node.gateway ?! GetData("key1"), 3 seconds) should be(Some("value1"))
Await.result(node.gateway ?! PutData("key1", "value2"), 3 seconds) should be(Some("value1"))
Await.result(node.gateway ?! GetData("key1"), 3 seconds) should be(Some("value2"))
} finally {
node.shutdown()
}
}
}
Example 199
| Source File: GraphApi.scala From affinity with Apache License 2.0 | 5 votes |
|
import akka.util.Timeout
import io.amient.affinity.core.ack
import io.amient.affinity.core.actor.Gateway
import message._
import scala.concurrent.duration._
import scala.concurrent.{Future, Promise}
import scala.language.postfixOps
import scala.util.control.NonFatal
trait GraphApi extends Gateway {
import context.dispatcher
implicit val scheduler = context.system.scheduler
val graphService = keyspace("graph")
protected def getVertexProps(vid: Int): Future[Option[VertexProps]] = {
implicit val timeout = Timeout(1 seconds)
graphService ?! GetVertexProps(vid)
}
protected def getGraphComponent(cid: Int): Future[Option[Component]] = {
implicit val timeout = Timeout(1 seconds)
graphService ?! GetComponent(cid)
}
protected def connect(v1: Int, v2: Int): Future[Set[Int]] = {
implicit val timeout = Timeout(5 seconds)
val ts = System.currentTimeMillis
graphService ?! ModifyGraph(v1, Edge(v2, ts, Map(
"hello" -> "world", "time" -> System.currentTimeMillis().toString)), GOP.ADD) flatMap {
case props1 => graphService ?! ModifyGraph(v2, Edge(v1, ts), GOP.ADD) flatMap {
case props2 => collectComponent(v2) flatMap {
case mergedComponent =>
val newComponentID = mergedComponent.connected.min
graphService ?! UpdateComponent(newComponentID, mergedComponent)
if (props1.component != newComponentID) graphService ?! DeleteComponent(props1.component)
if (props2.component != newComponentID) graphService ?! DeleteComponent(props2.component)
Future.sequence(mergedComponent.connected.map { v =>
graphService ?! UpdateVertexComponent(v, newComponentID)
})
}
}
}
}
protected def disconnect(v1: Int, v2: Int): Future[Set[Int]] = {
implicit val timeout = Timeout(5 seconds)
val ts = System.currentTimeMillis
graphService ?! ModifyGraph(v1, Edge(v2, ts), GOP.REMOVE) flatMap {
case props1 => graphService ?! ModifyGraph(v2, Edge(v1, ts), GOP.REMOVE) flatMap {
case props2 => collectComponent(v1) flatMap {
case component1 => collectComponent(v2) flatMap {
case component2 =>
val newComponentIDS = List(component1.connected.min, component2.connected.min)
graphService ?! UpdateComponent(newComponentIDS(0), component1)
graphService ?! UpdateComponent(newComponentIDS(1), component2)
if (!newComponentIDS.contains(props1.component)) graphService ?! DeleteComponent(props1.component)
if (!newComponentIDS.contains(props2.component)) graphService ?! DeleteComponent(props2.component)
Future.sequence {
component1.connected.map { v =>
graphService ?! UpdateVertexComponent(v, newComponentIDS(0))
} ++ component2.connected.map { v =>
graphService ?! UpdateVertexComponent(v, newComponentIDS(1))
}
}
}
}
}
}
}
private def collectComponent(vertex: Int): Future[Component] = {
val promise = Promise[Component]()
val ts = System.currentTimeMillis
implicit val timeout = Timeout(1 seconds)
def collect(queue: Set[Int], agg: Set[Int]): Unit = {
if (queue.isEmpty) {
promise.success(Component(ts, agg))
}
else graphService ?! GetVertexProps(queue.head) map {
_ match {
case None => throw new NoSuchElementException
case Some(VertexProps(_, _, Edges(connected))) => collect(queue.tail ++ (connected -- agg), agg ++ connected)
}
} recover {
case NonFatal(e) => promise.failure(e)
}
}
collect(Set(vertex), Set(vertex))
promise.future
}
}
Example 200
| Source File: Node.scala From affinity with Apache License 2.0 | 5 votes |
|
package io.amient.affinity.core.cluster
import java.util.concurrent.{CountDownLatch, TimeUnit, TimeoutException}
import akka.actor.{Actor, Props}
import akka.event.Logging
import akka.util.Timeout
import com.typesafe.config.{Config, ConfigFactory}
import io.amient.affinity.core.ack
import io.amient.affinity.core.actor.Controller._
import io.amient.affinity.core.actor.Gateway.{GatewayClusterStatus, GatewayConf}
import io.amient.affinity.core.actor._
import io.amient.affinity.core.config._
import io.amient.affinity.{AffinityActorSystem, Conf}
import scala.concurrent.duration._
import scala.concurrent.{Await, Future, Promise}
import scala.language.{implicitConversions, postfixOps}
import scala.reflect.ClassTag
object Node {
class NodeConf extends CfgStruct[NodeConf] {
val Containers: CfgGroup[CfgIntList] = group("container", classOf[CfgIntList], false)
.doc("Array of partitions assigned to this node, <ID> represents the Keyspace, e.g. assigning first four partitions of MyKeySpace: affinity.node.container.MyKeySpace = [0,1,2,3] ")
val Gateway: GatewayConf = struct("gateway", new GatewayConf, false)
val SuspendQueueMaxSize = integer("suspend.queue.max.size", 1000).doc("Size of the queue when the cluster enters suspended mode")
val StartupTimeoutMs = longint("startup.timeout.ms", Integer.MAX_VALUE).doc("Maximum time a node can take to startup - this number must account for any potential state bootstrap")
val ShutdownTimeoutMs = longint("shutdown.timeout.ms", 30000).doc("Maximum time a node can take to shutdown gracefully")
val DataDir = filepath("data.dir", false).doc("Location under which any local state or registers will be kept - this is required if running in a distributed mode or when using persisted kv stores")
val DataAutoAssign = bool("data.auto.assign", true, false).doc("Determines whether this node auto-balances data its containers; if set tot false the fixed list of container partitions will be used")
val DataAutoDelete = bool("data.auto.delete", true, false).doc("If set to true, any unassigned partitions will be deleted from the local storage")
}
}
class Node(config: Config) {
def this(configResource: String) = this(ConfigFactory.parseResources(configResource).resolve)
val conf = Conf(config)
val startupTimeout = conf.Affi.Node.StartupTimeoutMs().toLong milliseconds
val shutdownTimeout = conf.Affi.Node.ShutdownTimeoutMs().toLong milliseconds
implicit val system = AffinityActorSystem.create(config)
private val log = Logging.getLogger(system, this)
private val controller = system.actorOf(Props(new Controller), name = "controller")
private val httpGatewayPort = Promise[List[Int]]()
private val clusterReady = new CountDownLatch(1)
@volatile private var shuttingDown = false
@volatile private var fatalError: Option[Throwable] = None
import scala.concurrent.ExecutionContext.Implicits.global
val systemEventsWatcher = system.actorOf(Props(new Actor {
override def receive: Receive = {
case GatewayClusterStatus(false) => clusterReady.countDown()
case FatalErrorShutdown(e) =>
fatalError = Some(e)
shutdown()
}
}))
system.eventStream.subscribe(systemEventsWatcher, classOf[GatewayClusterStatus])
system.eventStream.subscribe(systemEventsWatcher, classOf[FatalErrorShutdown])
sys.addShutdownHook {
if (!shuttingDown) {
log.info("process killed - attempting graceful shutdown")
fatalError = None
shutdown()
}
Await.ready(system.terminate, shutdownTimeout)
}
def start[T <: Gateway](creator: => T)(implicit tag: ClassTag[T]): Future[List[Int]] = {
controller ! StartRebalance()
implicit val timeout = Timeout(startupTimeout)
val result = controller ?? CreateGateway(Props(creator))
httpGatewayPort.completeWith(result)
result
}
}
