cats.effect.Effect Scala Examples
The following examples show how to use cats.effect.Effect.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
Example 1
| Source File: MetricsMiddleware.scala From iotchain with MIT License | 5 votes |
|
package jbok.network.http.server.middleware
import cats.effect.{Clock, Effect, Sync}
import cats.implicits._
import jbok.common.metrics.PrometheusMetrics
import org.http4s.HttpRoutes
import org.http4s.metrics.prometheus.{Prometheus, PrometheusExportService}
import org.http4s.server.middleware
object MetricsMiddleware {
def exportService[F[_]](implicit F: Sync[F]): F[HttpRoutes[F]] =
for {
_ <- PrometheusExportService.addDefaults[F](PrometheusMetrics.registry)
} yield PrometheusExportService.service[F](PrometheusMetrics.registry)
def apply[F[_]](routes: HttpRoutes[F], enableMetrics: Boolean)(implicit F: Effect[F], clock: Clock[F]): F[HttpRoutes[F]] =
if (enableMetrics) {
Prometheus[F](PrometheusMetrics.registry, "iotchain_http_server").map { metricsOps =>
middleware.Metrics[F](metricsOps)(routes)
}
} else {
F.pure(routes)
}
}
Example 2
| Source File: TracingUtils.scala From opencensus-scala with Apache License 2.0 | 5 votes |
|
package io.opencensus.scala.http4s
import cats.effect.Effect
import io.opencensus.scala.Tracing
import io.opencensus.scala.http.{
StatusTranslator,
HttpAttributes => BaseHttpAttributes
}
import io.opencensus.scala.http4s.HttpAttributes._
import io.opencensus.trace.Span
import org.http4s.Response
object TracingUtils {
def recordResponse[F[_]: Effect](span: Span, tracing: Tracing)(
response: Response[F]
): Response[F] = {
BaseHttpAttributes.setAttributesForResponse(span, response)
tracing.setStatus(span, StatusTranslator.translate(response.status.code))
response.copy(
body = response.body.onFinalize(Effect[F].delay(tracing.endSpan(span)))
)
}
}
Example 3
| Source File: TracingClient.scala From opencensus-scala with Apache License 2.0 | 5 votes |
|
package io.opencensus.scala.http4s
import cats.effect.{Effect, Resource}
import cats.implicits._
import io.opencensus.scala.Tracing
import io.opencensus.scala.http.propagation.Propagation
import io.opencensus.scala.http.{HttpAttributes => BaseHttpAttributes}
import io.opencensus.scala.http4s.HttpAttributes._
import io.opencensus.scala.http4s.TracingUtils.recordResponse
import io.opencensus.scala.http4s.propagation.Http4sFormatPropagation
import io.opencensus.trace.{Span, Status}
import org.http4s.client.Client
import org.http4s.{Header, Request, Response}
abstract class TracingClient[F[_]: Effect] {
protected val tracing: Tracing
protected val propagation: Propagation[Header, Request[F]]
def trace(client: Client[F], parentSpan: Option[Span] = None): Client[F] = {
val tracedOpen: Request[F] => Resource[F, Response[F]] =
req =>
for {
span <- Resource.liftF(startSpan(parentSpan, req))
enrichedReq = addTraceHeaders(req, span)
res <- client
.run(enrichedReq)
.onError(traceError(span).andThen(x => Resource.liftF(x)))
} yield recordResponse(span, tracing)(res)
Client(tracedOpen)
}
private def traceError(span: Span): PartialFunction[Throwable, F[Unit]] = {
case _ => recordException(span)
}
private def startSpan(parentSpan: Option[Span], req: Request[F]) =
Effect[F].delay(startAndEnrichSpan(req, parentSpan))
private def startAndEnrichSpan(
req: Request[F],
parentSpan: Option[Span]
): Span = {
val name = req.uri.path.toString
val span = parentSpan.fold(tracing.startSpan(name))(span =>
tracing.startSpanWithParent(name, span)
)
BaseHttpAttributes.setAttributesForRequest(span, req)
span
}
private def addTraceHeaders(request: Request[F], span: Span): Request[F] =
request.withHeaders(
request.headers.put(propagation.headersWithTracingContext(span): _*)
)
private def recordException(span: Span) =
Effect[F].delay(tracing.endSpan(span, Status.INTERNAL))
}
object TracingClient {
def apply[F[_]: Effect]: TracingClient[F] =
new TracingClient[F] {
override protected val tracing: Tracing = Tracing
override protected val propagation: Propagation[Header, Request[F]] =
new Http4sFormatPropagation[F] {}
}
}
Example 4
| Source File: akkaHttp.scala From sup with Apache License 2.0 | 5 votes |
|
package sup.modules
import akka.http.scaladsl.marshalling.ToEntityMarshaller
import akka.http.scaladsl.model.StatusCode
import akka.http.scaladsl.model.StatusCodes
import akka.http.scaladsl.server.Directives.{path => akkaPath, _}
import akka.http.scaladsl.server.Route
import cats.effect.Effect
import cats.syntax.functor._
import cats.syntax.reducible._
import cats.~>
import cats.Functor
import cats.Reducible
import sup.HealthCheck
import sup.HealthResult
import scala.concurrent.Future
import scala.util.Failure
import scala.util.Success
import akka.http.scaladsl.model.HttpRequest
object akkahttp {
def healthCheckRoutes[F[_]: Effect, H[_]: Reducible](
healthCheck: HealthCheck[F, H],
path: String = "health-check"
)(
implicit marshaller: ToEntityMarshaller[HealthResult[H]]
): Route =
akkaPath(path) {
get {
onComplete(Effect[F].toIO(healthCheckResponse(healthCheck)).unsafeToFuture()) {
case Success(response) => complete(response)
case Failure(error) => failWith(error)
}
}
}
def healthCheckResponse[F[_]: Functor, H[_]: Reducible](
healthCheck: HealthCheck[F, H]
): F[(StatusCode, HealthResult[H])] =
healthCheck.check.map { check =>
if (check.value.reduce.isHealthy) StatusCodes.OK -> check
else StatusCodes.ServiceUnavailable -> check
}
def healthCheckRoutesWithContext[F[_]: Functor, H[_]: Reducible, R](
healthCheck: HealthCheck[F, H],
path: String = "health-check"
)(
run: HttpRequest => F ~> Future
)(
implicit marshaller: ToEntityMarshaller[HealthResult[H]]
): Route =
akkaPath(path) {
get {
extractRequest { request =>
onComplete(run(request)(healthCheckResponse(healthCheck))) {
case Success(response) => complete(response)
case Failure(error) => failWith(error)
}
}
}
}
}
Example 5
| Source File: KamonSupport.scala From kamon-http4s with Apache License 2.0 | 5 votes |
|
package kamon.http4s
package middleware.client
import cats.effect.{Effect, Resource}
import cats.implicits._
import com.typesafe.config.Config
import kamon.Kamon
import kamon.context.Context
import kamon.instrumentation.http.HttpClientInstrumentation
import org.http4s.{Request, Response}
import org.http4s.client.Client
object KamonSupport {
private var _instrumentation = instrumentation(Kamon.config())
private def instrumentation(kamonConfig: Config): HttpClientInstrumentation = {
val httpClientConfig = kamonConfig.getConfig("kamon.instrumentation.http4s.client")
HttpClientInstrumentation.from(httpClientConfig, "http4s.client")
}
Kamon.onReconfigure(newConfig => _instrumentation = instrumentation(newConfig))
def apply[F[_]](underlying: Client[F])(implicit F:Effect[F]): Client[F] = Client { request =>
for {
ctx <- Resource.liftF(F.delay(Kamon.currentContext()))
k <- kamonClient(underlying)(request)(ctx)(_instrumentation)
} yield k
}
private def kamonClient[F[_]](underlying: Client[F])
(request: Request[F])
(ctx: Context)
(instrumentation: HttpClientInstrumentation)
(implicit F:Effect[F]): Resource[F, Response[F]] =
for {
requestHandler <- Resource.liftF(F.delay(instrumentation.createHandler(getRequestBuilder(request), ctx)))
response <- underlying.run(requestHandler.request).attempt
trackedResponse <- Resource.liftF(handleResponse(response, requestHandler, instrumentation.settings))
} yield trackedResponse
def handleResponse[F[_]](
response: Either[Throwable, Response[F]],
requestHandler: HttpClientInstrumentation.RequestHandler[Request[F]],
settings: HttpClientInstrumentation.Settings
)(implicit F:Effect[F]): F[Response[F]] =
response match {
case Right(res) =>
if(res.status.code == 404) requestHandler.span.name(settings.defaultOperationName)
requestHandler.processResponse(getResponseBuilder(res))
F.delay(res)
case Left(error) =>
requestHandler.span.fail(error).finish()
F.raiseError(error)
}
}
Example 6
| Source File: TaskInstances.scala From shims with Apache License 2.0 | 5 votes |
|
package shims.effect.instances
import cats.{Applicative, Monad, Parallel, StackSafeMonad, ~>}
import cats.effect.{Effect, ExitCase, IO, SyncIO}
import scalaz.{Tag, -\/, \/, \/-}
import scalaz.concurrent.Task.ParallelTask
import scalaz.concurrent.{Future, Task}
import shims.conversions.MonadErrorConversions
import java.util.concurrent.atomic.AtomicBoolean
import scala.util.control.NonFatal
trait TaskInstances extends MonadErrorConversions {
// cribbed from quasar, where it was mostly cribbed from scalaz-task-effect
implicit object taskEffect extends Effect[Task] with StackSafeMonad[Task] {
def pure[A](x: A): Task[A] = Task.now(x)
def handleErrorWith[A](fa: Task[A])(f: Throwable => Task[A]): Task[A] =
fa.handleWith(functionToPartial(f))
def raiseError[A](e: Throwable): Task[A] = Task.fail(e)
// In order to comply with `repeatedCallbackIgnored` law
// on async, a custom AtomicBoolean is required to ignore
// second callbacks.
def async[A](k: (Either[Throwable, A] => Unit) => Unit): Task[A] = Task.async { registered =>
val a = new AtomicBoolean(true)
try k(e => if (a.getAndSet(false)) registered(\/.fromEither(e)) else ())
catch { case NonFatal(t) => registered(-\/(t)) }
}
def asyncF[A](k: (Either[Throwable, A] => Unit) => Task[Unit]): Task[A] =
async(k.andThen(_.unsafePerformAsync(forget)))
// emulates using attempt
def bracketCase[A, B](acquire: Task[A])(use: A => Task[B])(release: (A, ExitCase[Throwable]) => Task[Unit]): Task[B] = {
for {
a <- acquire
bOr <- use(a).attempt
ec = bOr.fold(ExitCase.Error(_), _ => ExitCase.Completed)
_ <- release(a, ec)
b <- bOr.fold(Task.fail, Task.now)
} yield b
}
def runAsync[A](fa: Task[A])(cb: Either[Throwable, A] => IO[Unit]): SyncIO[Unit] =
SyncIO {
fa unsafePerformAsync { disjunction =>
cb(disjunction.toEither).unsafeRunAsync(forget)
}
}
def runSyncStep[A](fa: Task[A]): IO[Either[Task[A], A]] =
IO {
fa.get match {
case Future.Now(-\/(_)) => Left(fa)
case other => other.step match {
case Future.Now(\/-(a)) => Right(a)
case other => Left(new Task(other))
}
}
}
override def map[A, B](fa: Task[A])(f: A => B): Task[B] =
fa.map(f)
def flatMap[A, B](fa: Task[A])(f: A => Task[B]): Task[B] = fa.flatMap(f)
override def delay[A](thunk: => A): Task[A] = Task.delay(thunk)
def suspend[A](thunk: => Task[A]): Task[A] = Task.suspend(thunk)
}
implicit val taskParallel: Parallel.Aux[Task, ParallelTask] = new Parallel[Task] {
import Task.taskParallelApplicativeInstance
type F[A] = ParallelTask[A]
val monad: Monad[Task] = taskEffect
val applicative: Applicative[ParallelTask] = Applicative[ParallelTask]
val sequential: ParallelTask ~> Task = λ[ParallelTask ~> Task](Tag.unwrap(_))
val parallel: Task ~> ParallelTask = λ[Task ~> ParallelTask](Tag(_))
}
private def functionToPartial[A, B](f: A => B): PartialFunction[A, B] = {
case a => f(a)
}
private def forget[A](x: A): Unit = ()
}
Example 7
| Source File: package.scala From finch-oauth2 with Apache License 2.0 | 5 votes |
|
package io.finch
import cats.effect.Effect
import cats.implicits._
import com.twitter.finagle.OAuth2
import com.twitter.finagle.http.Status
import com.twitter.finagle.oauth2.{AuthInfo, DataHandler, GrantResult, OAuthError}
import com.twitter.util.{Future => TwitterFuture, Return, Throw}
package object oauth2 {
private def twitterFutureToEffect[F[_], A](f: => TwitterFuture[A])(implicit F: Effect[F]): F[A] =
F.async { cb =>
f.respond {
case Return(r) => cb(Right(r))
case Throw(t) => cb(Left(t))
}
}
private val handleOAuthError: PartialFunction[Throwable, Output[Nothing]] = {
case e: OAuthError =>
val bearer = Seq("error=\"" + e.errorType + "\"") ++
(if (!e.description.isEmpty) Seq("error_description=\"" + e.description + "\"") else Nil)
Output.failure(e, Status(e.statusCode))
.withHeader("WWW-Authenticate" -> s"Bearer ${bearer.mkString(", ")}")
}
def issueAccessToken[F[_], U](dataHandler: DataHandler[U])(
implicit F: Effect[F]
): Endpoint[F, GrantResult] =
new Endpoint[F, GrantResult] {
final def apply(input: Input): Endpoint.Result[F, GrantResult] = {
val out = twitterFutureToEffect(OAuth2.issueAccessToken(input.request, dataHandler))
.map(ai => Output.payload(ai))
.recover(handleOAuthError)
EndpointResult.Matched(input, Trace.empty, out)
}
}
}
Example 8
| Source File: RerunnableInstances.scala From catbird with Apache License 2.0 | 5 votes |
|
package io.catbird.util.effect
import cats.effect.{ Effect, ExitCase, IO, SyncIO }
import com.twitter.util.{ Future, Monitor, Promise, Return, Throw }
import io.catbird.util.{ Rerunnable, RerunnableMonadError }
import java.lang.Throwable
import scala.Unit
import scala.util.{ Either, Left, Right }
trait RerunnableInstances {
implicit final val rerunnableEffectInstance: Effect[Rerunnable] =
new RerunnableMonadError with Effect[Rerunnable] {
final def suspend[A](thunk: => Rerunnable[A]): Rerunnable[A] = Rerunnable.suspend[A](thunk)
override final def delay[A](thunk: => A): Rerunnable[A] = Rerunnable[A](thunk)
final def async[A](k: (Either[Throwable, A] => Unit) => Unit): Rerunnable[A] =
new Rerunnable[A] {
final def run: Future[A] = {
val promise = new Promise[A]
k { e =>
if (promise.isDefined) ()
else
e match {
case Right(a) => promise.setValue(a)
case Left(err) => promise.setException(err)
}
}
promise
}
}
final def asyncF[A](k: (Either[Throwable, A] => Unit) => Rerunnable[Unit]): Rerunnable[A] =
new Rerunnable[A] {
final def run: Future[A] = {
val promise = new Promise[A]
val rerunnable = k { e =>
if (promise.isDefined) ()
else
e match {
case Right(a) => promise.setValue(a)
case Left(err) => promise.setException(err)
}
}
rerunnable.run.flatMap(_ => promise)
}
}
final def runAsync[A](fa: Rerunnable[A])(cb: Either[Throwable, A] => IO[Unit]): SyncIO[Unit] =
rerunnableToIO[A](fa).runAsync(cb)
final def bracketCase[A, B](acquire: Rerunnable[A])(use: A => Rerunnable[B])(
release: (A, ExitCase[Throwable]) => Rerunnable[Unit]
): Rerunnable[B] = new Rerunnable[B] {
final def run: Future[B] =
acquire.run.flatMap { a =>
val future = use(a).run
future.transform {
case Return(b) => release(a, ExitCase.complete).run.handle(Monitor.catcher).flatMap(_ => future)
case Throw(err) => release(a, ExitCase.error(err)).run.handle(Monitor.catcher).flatMap(_ => future)
}
}
}
}
}
Example 9
| Source File: TodoItemApi.scala From freestyle with Apache License 2.0 | 5 votes |
|
package examples.todolist
package http
import cats.effect.Effect
import cats.implicits._
import examples.todolist.service.TodoItemService
import io.circe.generic.auto._
import io.circe.syntax._
import org.http4s._
import org.http4s.circe._
import org.http4s.dsl.Http4sDsl
class TodoItemApi[F[_]: Effect](implicit service: TodoItemService[F]) extends Http4sDsl[F] {
import codecs._
private val prefix = "items"
val endpoints = HttpService[F] {
case POST -> Root / prefix =>
service.reset *> Ok()
case GET -> Root / prefix / IntVar(id) =>
service.retrieve(id) flatMap { item =>
item.fold(NotFound(s"Could not find ${service.model} with $id"))(todoItem =>
Ok(todoItem.asJson))
}
case GET -> Root / prefix =>
service.list.flatMap(l => Ok(l.asJson))
case req @ POST -> Root / prefix =>
for {
todoItem <- req.as[TodoItem]
insertedItem <- service.insert(todoItem)
response <- Ok(insertedItem.asJson)
} yield response
case req @ PUT -> Root / prefix / IntVar(id) =>
for {
todoItem <- req.as[TodoItem]
updatedItem <- service.update(todoItem.copy(id = Some(id)))
reponse <- Ok(updatedItem.asJson)
} yield reponse
case DELETE -> Root / prefix / IntVar(id) =>
service.destroy(id) *> Ok()
}
}
object TodoItemApi {
implicit def instance[F[_]: Effect](implicit service: TodoItemService[F]): TodoItemApi[F] =
new TodoItemApi[F]
}
Example 10
| Source File: AppApi.scala From freestyle with Apache License 2.0 | 5 votes |
|
package examples.todolist
package http
import cats.effect.Effect
import cats.implicits._
import examples.todolist.service.AppService
import io.circe.generic.auto._
import io.circe.syntax._
import org.http4s._
import org.http4s.circe._
import org.http4s.dsl.Http4sDsl
class AppApi[F[_]: Effect](implicit service: AppService[F]) extends Http4sDsl[F] {
import codecs._
val endpoints = HttpService[F] {
case POST -> Root / "reset" =>
service.reset.flatMap(e => Ok(e.asJson))
case GET -> Root / "list" =>
service.list.flatMap(l => Ok(l.asJson))
case req @ POST -> Root / "insert" =>
for {
form <- req.as[TodoForm]
insertedForm <- service.insert(form)
response <- Ok(insertedForm.asJson)
} yield response
case req @ PUT -> Root / "update" =>
for {
form <- req.as[TodoForm]
updatedForm <- service.update(form)
response <- Ok(updatedForm.asJson)
} yield response
case req @ DELETE -> Root / "delete" =>
for {
form <- req.as[TodoForm]
deleted <- service.destroy(form)
response <- Ok(deleted.asJson)
} yield response
}
}
object AppApi {
implicit def instance[F[_]: Effect](implicit service: AppService[F]): AppApi[F] = new AppApi[F]
}
Example 11
package examples.todolist
package http
import cats.effect.Effect
import cats.implicits._
import examples.todolist.http._
import org.http4s.implicits._
class Api[F[_]: Effect](
implicit appApi: AppApi[F],
genericApi: GenericApi[F],
todoItemApi: TodoItemApi[F],
todoListApi: TodoListApi[F],
tagApi: TagApi[F]) {
val endpoints =
appApi.endpoints <+>
genericApi.endpoints <+>
todoItemApi.endpoints <+>
todoListApi.endpoints <+>
tagApi.endpoints
}
object Api {
implicit def instance[F[_]: Effect](
implicit appApi: AppApi[F],
genericApi: GenericApi[F],
todoItemApi: TodoItemApi[F],
todoListApi: TodoListApi[F],
tagApi: TagApi[F]): Api[F] =
new Api[F]
}
Example 12
| Source File: TagApi.scala From freestyle with Apache License 2.0 | 5 votes |
|
package examples.todolist
package http
import cats.effect.Effect
import cats.implicits._
import examples.todolist.service.TagService
import examples.todolist.Tag
import io.circe.generic.auto._
import io.circe.syntax._
import org.http4s._
import org.http4s.circe._
import org.http4s.dsl.Http4sDsl
class TagApi[F[_]: Effect](implicit service: TagService[F]) extends Http4sDsl[F] {
import codecs._
private val prefix = "tags"
val endpoints = HttpService[F] {
case POST -> Root / prefix / "reset" =>
service.reset.flatMap(e => Ok(e.asJson))
case GET -> Root / prefix / IntVar(id) =>
service.retrieve(id) flatMap { item =>
item.fold(NotFound(s"Could not find ${service.model} with $id"))(tag => Ok(tag.asJson))
}
case GET -> Root / prefix =>
service.list.flatMap(l => Ok(l.asJson))
case req @ POST -> Root / prefix =>
for {
tag <- req.as[Tag]
insertedTag <- service.insert(tag)
response <- Ok(insertedTag.asJson)
} yield response
case req @ PUT -> Root / prefix / IntVar(id) =>
for {
tag <- req.as[Tag]
updatedTag <- service.update(tag.copy(id = Some(id)))
reponse <- Ok(updatedTag.asJson)
} yield reponse
case DELETE -> Root / prefix / IntVar(id) =>
service.destroy(id) *> Ok()
}
}
object TagApi {
implicit def instance[F[_]: Effect](implicit service: TagService[F]): TagApi[F] = new TagApi[F]
}
Example 13
| Source File: TodoListApi.scala From freestyle with Apache License 2.0 | 5 votes |
|
package examples.todolist
package http
import cats.effect.Effect
import cats.implicits._
import examples.todolist.service.TodoListService
import io.circe.generic.auto._
import io.circe.syntax._
import org.http4s._
import org.http4s.circe._
import org.http4s.dsl.Http4sDsl
class TodoListApi[F[_]: Effect](implicit service: TodoListService[F]) extends Http4sDsl[F] {
import codecs._
private val prefix = "lists"
val endpoints = HttpService[F] {
case POST -> Root / prefix =>
service.reset.flatMap(e => Ok(e.asJson))
case GET -> Root / prefix / IntVar(id) =>
service.retrieve(id) flatMap { item =>
item.fold(NotFound(s"Could not find ${service.model} with $id"))(todoList =>
Ok(todoList.asJson))
}
case GET -> Root / prefix =>
service.list.flatMap(l => Ok(l.asJson))
case req @ POST -> Root / prefix =>
for {
todoList <- req.as[TodoList]
insertedTodoList <- service.insert(todoList)
response <- Ok(insertedTodoList.asJson)
} yield response
case req @ PUT -> Root / prefix / IntVar(id) =>
for {
todoList <- req.as[TodoList]
updatedTodoList <- service.update(todoList.copy(id = Some(id)))
reponse <- Ok(updatedTodoList.asJson)
} yield reponse
case DELETE -> Root / prefix / IntVar(id) =>
service.destroy(id) *> Ok()
}
}
object TodoListApi {
implicit def instance[F[_]: Effect](implicit service: TodoListService[F]): TodoListApi[F] =
new TodoListApi[F]
}
Example 14
| Source File: GenericApi.scala From freestyle with Apache License 2.0 | 5 votes |
|
package examples.todolist
package http
import cats.effect.Effect
import cats.implicits._
import examples.todolist.model.Pong
import freestyle.tagless.logging.LoggingM
import io.circe.Json
import org.http4s.circe._
import org.http4s.dsl.Http4sDsl
import org.http4s.HttpService
class GenericApi[F[_]: Effect](implicit log: LoggingM[F]) extends Http4sDsl[F] {
val endpoints =
HttpService[F] {
case GET -> Root / "ping" =>
for {
_ <- log.error("Not really an error")
_ <- log.warn("Not really a warn")
_ <- log.debug("GET /ping")
response <- Ok(Json.fromLong(Pong.current.time))
} yield response
case GET -> Root / "hello" =>
for {
_ <- log.error("Not really an error")
_ <- log.warn("Not really a warn")
_ <- log.debug("GET /Hello")
response <- Ok("Hello World")
} yield response
}
}
object GenericApi {
implicit def instance[F[_]: Effect](implicit log: LoggingM[F]): GenericApi[F] =
new GenericApi[F]
}
Example 15
| Source File: TodoListApp.scala From freestyle with Apache License 2.0 | 5 votes |
|
package todo
import cats.effect.{Effect, IO}
import cats.syntax.either._
import cats.syntax.flatMap._
import cats.syntax.functor._
import doobie.util.transactor.Transactor
import examples.todolist.http.Api
import examples.todolist.services.Services
import exapmles.todolist.peristence.Persistence
import freestyle.tagless.config.ConfigM
import freestyle.tagless.config.implicits._
import freestyle.tagless.effects.error.ErrorM
import freestyle.tagless.effects.error.implicits._
import freestyle.tagless.logging.LoggingM
import freestyle.tagless.loggingJVM.log4s.implicits._
import freestyle.tagless.module
import fs2.StreamApp
import org.http4s.HttpService
import org.http4s.implicits._
import org.http4s.server.blaze.BlazeBuilder
@module
trait App[F[_]] {
val persistence: Persistence[F]
val services: Services[F]
}
object TodoListApp extends StreamApp[IO] {
import examples.todolist.runtime.implicits._
override def stream(
args: List[String],
requestShutdown: IO[Unit]): fs2.Stream[IO, StreamApp.ExitCode] =
bootstrap[IO].unsafeRunSync()
def bootstrap[F[_]: Effect](
implicit app: App[F],
T: Transactor[F],
api: Api[F]): F[fs2.Stream[F, StreamApp.ExitCode]] = {
val services: HttpService[F] = api.endpoints
val log: LoggingM[F] = app.services.log
val config: ConfigM[F] = app.services.config
for {
_ <- log.info("Trying to load application.conf")
cfg <- config.load
host: String = cfg.string("http.host").getOrElse("localhost")
port: Int = cfg.int("http.port").getOrElse(8080)
_ <- log.debug(s"Host: $host")
_ <- log.debug(s"Port: $port")
} yield
BlazeBuilder[F]
.bindHttp(port, host)
.mountService(services)
.serve
}
}
Example 16
| Source File: Fs2StreamClientCallListener.scala From fs2-grpc with MIT License | 5 votes |
|
package org.lyranthe.fs2_grpc
package java_runtime
package client
import cats.effect.{Effect, ConcurrentEffect}
import cats.implicits._
import fs2.{Pull, Stream}
import fs2.concurrent.Queue
import io.grpc.{ClientCall, Metadata, Status}
class Fs2StreamClientCallListener[F[_]: Effect, Response](
request: Int => Unit,
queue: Queue[F, Either[GrpcStatus, Response]]
) extends ClientCall.Listener[Response] {
override def onMessage(message: Response): Unit = {
request(1)
queue.enqueue1(message.asRight).unsafeRun()
}
override def onClose(status: Status, trailers: Metadata): Unit =
queue.enqueue1(GrpcStatus(status, trailers).asLeft).unsafeRun()
def stream: Stream[F, Response] = {
def go(q: Stream[F, Either[GrpcStatus, Response]]): Pull[F, Response, Unit] = {
q.pull.uncons1.flatMap {
case Some((Right(v), tl)) => Pull.output1(v) >> go(tl)
case Some((Left(GrpcStatus(status, trailers)), _)) =>
if (!status.isOk)
Pull.raiseError[F](status.asRuntimeException(trailers))
else
Pull.done
case None => Pull.done
}
}
go(queue.dequeue).stream
}
}
object Fs2StreamClientCallListener {
def apply[F[_]: ConcurrentEffect, Response](request: Int => Unit): F[Fs2StreamClientCallListener[F, Response]] =
Queue.unbounded[F, Either[GrpcStatus, Response]].map(new Fs2StreamClientCallListener[F, Response](request, _))
}
Example 17
| Source File: Fs2StreamServerCallListener.scala From fs2-grpc with MIT License | 5 votes |
|
package org.lyranthe.fs2_grpc
package java_runtime
package server
import cats.effect.concurrent.Deferred
import cats.effect.{ConcurrentEffect, Effect}
import cats.implicits._
import io.grpc.ServerCall
import fs2.concurrent.Queue
import fs2._
class Fs2StreamServerCallListener[F[_], Request, Response] private (
requestQ: Queue[F, Option[Request]],
val isCancelled: Deferred[F, Unit],
val call: Fs2ServerCall[F, Request, Response]
)(implicit F: Effect[F])
extends ServerCall.Listener[Request]
with Fs2ServerCallListener[F, Stream[F, ?], Request, Response] {
override def onCancel(): Unit = {
isCancelled.complete(()).unsafeRun()
}
override def onMessage(message: Request): Unit = {
call.call.request(1)
requestQ.enqueue1(message.some).unsafeRun()
}
override def onHalfClose(): Unit = requestQ.enqueue1(none).unsafeRun()
override def source: Stream[F, Request] =
requestQ.dequeue.unNoneTerminate
}
object Fs2StreamServerCallListener {
class PartialFs2StreamServerCallListener[F[_]](val dummy: Boolean = false) extends AnyVal {
def apply[Request, Response](
call: ServerCall[Request, Response],
options: ServerCallOptions = ServerCallOptions.default
)(implicit
F: ConcurrentEffect[F]
): F[Fs2StreamServerCallListener[F, Request, Response]] =
for {
inputQ <- Queue.unbounded[F, Option[Request]]
isCancelled <- Deferred[F, Unit]
serverCall <- Fs2ServerCall[F, Request, Response](call, options)
} yield new Fs2StreamServerCallListener[F, Request, Response](inputQ, isCancelled, serverCall)
}
def apply[F[_]] = new PartialFs2StreamServerCallListener[F]
}
Example 18
| Source File: Fs2UnaryServerCallListener.scala From fs2-grpc with MIT License | 5 votes |
|
package org.lyranthe.fs2_grpc
package java_runtime
package server
import cats.effect.{ConcurrentEffect, Effect}
import cats.effect.concurrent.{Deferred, Ref}
import cats.syntax.all._
import io.grpc._
class Fs2UnaryServerCallListener[F[_], Request, Response] private (
request: Ref[F, Option[Request]],
isComplete: Deferred[F, Unit],
val isCancelled: Deferred[F, Unit],
val call: Fs2ServerCall[F, Request, Response]
)(implicit F: Effect[F])
extends ServerCall.Listener[Request]
with Fs2ServerCallListener[F, F, Request, Response] {
import Fs2UnaryServerCallListener._
override def onCancel(): Unit = {
isCancelled.complete(()).unsafeRun()
}
override def onMessage(message: Request): Unit = {
request.access
.flatMap[Unit] {
case (curValue, modify) =>
if (curValue.isDefined)
F.raiseError(statusException(TooManyRequests))
else
modify(message.some).void
}
.unsafeRun()
}
override def onHalfClose(): Unit =
isComplete.complete(()).unsafeRun()
override def source: F[Request] =
for {
_ <- isComplete.get
valueOrNone <- request.get
value <- valueOrNone.fold[F[Request]](F.raiseError(statusException(NoMessage)))(F.pure)
} yield value
}
object Fs2UnaryServerCallListener {
val TooManyRequests: String = "Too many requests"
val NoMessage: String = "No message for unary call"
private val statusException: String => StatusRuntimeException = msg =>
new StatusRuntimeException(Status.INTERNAL.withDescription(msg))
class PartialFs2UnaryServerCallListener[F[_]](val dummy: Boolean = false) extends AnyVal {
def apply[Request, Response](
call: ServerCall[Request, Response],
options: ServerCallOptions = ServerCallOptions.default
)(implicit
F: ConcurrentEffect[F]
): F[Fs2UnaryServerCallListener[F, Request, Response]] =
for {
request <- Ref.of[F, Option[Request]](none)
isComplete <- Deferred[F, Unit]
isCancelled <- Deferred[F, Unit]
serverCall <- Fs2ServerCall[F, Request, Response](call, options)
} yield new Fs2UnaryServerCallListener[F, Request, Response](request, isComplete, isCancelled, serverCall)
}
def apply[F[_]] = new PartialFs2UnaryServerCallListener[F]
}
Example 19
| Source File: Publish.scala From fs2-rabbit with Apache License 2.0 | 5 votes |
|
package dev.profunktor.fs2rabbit.algebra
import cats.effect.syntax.effect._
import cats.effect.{Blocker, ContextShift, Effect, Sync}
import cats.syntax.functor._
import com.rabbitmq.client.{AMQP, ReturnListener}
import dev.profunktor.fs2rabbit.model._
object Publish {
def make[F[_]: Effect: ContextShift](
blocker: Blocker
): Publish[F] =
new Publish[F] {
override def basicPublish(channel: AMQPChannel,
exchangeName: ExchangeName,
routingKey: RoutingKey,
msg: AmqpMessage[Array[Byte]]): F[Unit] =
blocker.delay {
channel.value.basicPublish(
exchangeName.value,
routingKey.value,
msg.properties.asBasicProps,
msg.payload
)
}
override def basicPublishWithFlag(channel: AMQPChannel,
exchangeName: ExchangeName,
routingKey: RoutingKey,
flag: PublishingFlag,
msg: AmqpMessage[Array[Byte]]): F[Unit] =
blocker.delay {
channel.value.basicPublish(
exchangeName.value,
routingKey.value,
flag.mandatory,
msg.properties.asBasicProps,
msg.payload
)
}
override def addPublishingListener(
channel: AMQPChannel,
listener: PublishReturn => F[Unit]
): F[Unit] =
Sync[F].delay {
val returnListener = new ReturnListener {
override def handleReturn(replyCode: Int,
replyText: String,
exchange: String,
routingKey: String,
properties: AMQP.BasicProperties,
body: Array[Byte]): Unit = {
val publishReturn =
PublishReturn(
ReplyCode(replyCode),
ReplyText(replyText),
ExchangeName(exchange),
RoutingKey(routingKey),
AmqpProperties.unsafeFrom(properties),
AmqpBody(body)
)
listener(publishReturn).toIO.unsafeRunAsync(_ => ())
}
}
channel.value.addReturnListener(returnListener)
}.void
override def clearPublishingListeners(channel: AMQPChannel): F[Unit] =
Sync[F].delay {
channel.value.clearReturnListeners()
}.void
}
}
trait Publish[F[_]] {
def basicPublish(channel: AMQPChannel,
exchangeName: ExchangeName,
routingKey: RoutingKey,
msg: AmqpMessage[Array[Byte]]): F[Unit]
def basicPublishWithFlag(channel: AMQPChannel,
exchangeName: ExchangeName,
routingKey: RoutingKey,
flag: PublishingFlag,
msg: AmqpMessage[Array[Byte]]): F[Unit]
def addPublishingListener(channel: AMQPChannel, listener: PublishReturn => F[Unit]): F[Unit]
def clearPublishingListeners(channel: AMQPChannel): F[Unit]
}
Example 20
| Source File: AckingProgram.scala From fs2-rabbit with Apache License 2.0 | 5 votes |
|
package dev.profunktor.fs2rabbit.program
import cats.Applicative
import cats.effect.{Effect, Sync}
import dev.profunktor.fs2rabbit.algebra.{AMQPInternals, Acking, Consume}
import dev.profunktor.fs2rabbit.arguments.Arguments
import dev.profunktor.fs2rabbit.config.Fs2RabbitConfig
import dev.profunktor.fs2rabbit.model.AckResult.{Ack, NAck, Reject}
import dev.profunktor.fs2rabbit.model._
object AckingProgram {
def make[F[_]: Effect](config: Fs2RabbitConfig): F[AckingProgram[F]] = Sync[F].delay {
WrapperAckingProgram(config, Consume.make)
}
}
trait AckingProgram[F[_]] extends Acking[F] with Consume[F]
case class WrapperAckingProgram[F[_]: Effect] private (
config: Fs2RabbitConfig,
consume: Consume[F]
) extends AckingProgram[F] {
override def createAcker(channel: AMQPChannel): F[AckResult => F[Unit]] = Applicative[F].pure {
case Ack(tag) => consume.basicAck(channel, tag, multiple = false)
case NAck(tag) =>
consume.basicNack(channel, tag, multiple = false, config.requeueOnNack)
case Reject(tag) =>
consume.basicReject(channel, tag, config.requeueOnReject)
}
override def basicAck(channel: AMQPChannel, tag: DeliveryTag, multiple: Boolean): F[Unit] =
consume.basicAck(channel, tag, multiple)
override def basicNack(channel: AMQPChannel, tag: DeliveryTag, multiple: Boolean, requeue: Boolean): F[Unit] =
consume.basicNack(channel, tag, multiple, requeue)
override def basicReject(channel: AMQPChannel, tag: DeliveryTag, requeue: Boolean): F[Unit] =
consume.basicReject(channel, tag, requeue)
override def basicQos(channel: AMQPChannel, basicQos: BasicQos): F[Unit] =
consume.basicQos(channel, basicQos)
override def basicConsume[A](channel: AMQPChannel,
queueName: QueueName,
autoAck: Boolean,
consumerTag: ConsumerTag,
noLocal: Boolean,
exclusive: Boolean,
args: Arguments)(internals: AMQPInternals[F]): F[ConsumerTag] =
consume.basicConsume(channel, queueName, autoAck, consumerTag, noLocal, exclusive, args)(internals)
override def basicCancel(channel: AMQPChannel, consumerTag: ConsumerTag): F[Unit] =
consume.basicCancel(channel, consumerTag)
}
Example 21
| Source File: ConsumingProgram.scala From fs2-rabbit with Apache License 2.0 | 5 votes |
|
package dev.profunktor.fs2rabbit.program
import cats.effect.{Effect, Sync}
import cats.implicits._
import dev.profunktor.fs2rabbit.algebra.ConsumingStream._
import dev.profunktor.fs2rabbit.algebra.{AMQPInternals, Consume, InternalQueue}
import dev.profunktor.fs2rabbit.arguments.Arguments
import dev.profunktor.fs2rabbit.effects.EnvelopeDecoder
import dev.profunktor.fs2rabbit.model._
import fs2.Stream
object ConsumingProgram {
def make[F[_]: Effect](internalQueue: InternalQueue[F]): F[ConsumingProgram[F]] = Sync[F].delay {
WrapperConsumingProgram(internalQueue, Consume.make)
}
}
trait ConsumingProgram[F[_]] extends ConsumingStream[F] with Consume[F]
case class WrapperConsumingProgram[F[_]: Effect] private (
internalQueue: InternalQueue[F],
consume: Consume[F]
) extends ConsumingProgram[F] {
override def createConsumer[A](
queueName: QueueName,
channel: AMQPChannel,
basicQos: BasicQos,
autoAck: Boolean = false,
noLocal: Boolean = false,
exclusive: Boolean = false,
consumerTag: ConsumerTag = ConsumerTag(""),
args: Arguments = Map.empty
)(implicit decoder: EnvelopeDecoder[F, A]): F[Stream[F, AmqpEnvelope[A]]] = {
val setup = for {
internalQ <- internalQueue.create
internals = AMQPInternals[F](Some(internalQ))
_ <- consume.basicQos(channel, basicQos)
consumerTag <- consume.basicConsume(
channel,
queueName,
autoAck,
consumerTag,
noLocal,
exclusive,
args
)(internals)
} yield (consumerTag, internalQ)
Stream
.bracket(setup) {
case (tag, _) =>
consume.basicCancel(channel, tag)
}
.flatMap {
case (_, queue) =>
queue.dequeue.rethrow
.evalMap(env => decoder(env).map(a => env.copy(payload = a)))
}
.pure[F]
}
override def basicAck(channel: AMQPChannel, tag: DeliveryTag, multiple: Boolean): F[Unit] =
consume.basicAck(channel, tag, multiple)
override def basicNack(channel: AMQPChannel, tag: DeliveryTag, multiple: Boolean, requeue: Boolean): F[Unit] =
consume.basicNack(channel, tag, multiple, requeue)
override def basicReject(channel: AMQPChannel, tag: DeliveryTag, requeue: Boolean): F[Unit] =
consume.basicReject(channel, tag, requeue)
override def basicQos(channel: AMQPChannel, basicQos: BasicQos): F[Unit] =
consume.basicQos(channel, basicQos)
override def basicConsume[A](
channel: AMQPChannel,
queueName: QueueName,
autoAck: Boolean,
consumerTag: ConsumerTag,
noLocal: Boolean,
exclusive: Boolean,
args: Arguments
)(internals: AMQPInternals[F]): F[ConsumerTag] =
consume.basicConsume(channel, queueName, autoAck, consumerTag, noLocal, exclusive, args)(internals)
override def basicCancel(channel: AMQPChannel, consumerTag: ConsumerTag): F[Unit] =
consume.basicCancel(channel, consumerTag)
}
Example 22
| Source File: OTag.scala From docspell with GNU General Public License v3.0 | 5 votes |
|
package docspell.backend.ops
import cats.effect.{Effect, Resource}
import cats.implicits._
import docspell.common.{AccountId, Ident}
import docspell.store.records.{RTag, RTagItem}
import docspell.store.{AddResult, Store}
trait OTag[F[_]] {
def findAll(account: AccountId, nameQuery: Option[String]): F[Vector[RTag]]
def add(s: RTag): F[AddResult]
def update(s: RTag): F[AddResult]
def delete(id: Ident, collective: Ident): F[AddResult]
def loadAll(ids: List[Ident]): F[Vector[RTag]]
}
object OTag {
def apply[F[_]: Effect](store: Store[F]): Resource[F, OTag[F]] =
Resource.pure[F, OTag[F]](new OTag[F] {
def findAll(account: AccountId, nameQuery: Option[String]): F[Vector[RTag]] =
store.transact(RTag.findAll(account.collective, nameQuery, _.name))
def add(t: RTag): F[AddResult] = {
def insert = RTag.insert(t)
def exists = RTag.existsByName(t)
val msg = s"A tag '${t.name}' already exists"
store.add(insert, exists).map(_.fold(identity, _.withMsg(msg), identity))
}
def update(t: RTag): F[AddResult] = {
def insert = RTag.update(t)
def exists = RTag.existsByName(t)
val msg = s"A tag '${t.name}' already exists"
store.add(insert, exists).map(_.fold(identity, _.withMsg(msg), identity))
}
def delete(id: Ident, collective: Ident): F[AddResult] = {
val io = for {
optTag <- RTag.findByIdAndCollective(id, collective)
n0 <- optTag.traverse(t => RTagItem.deleteTag(t.tagId))
n1 <- optTag.traverse(t => RTag.delete(t.tagId, collective))
} yield n0.getOrElse(0) + n1.getOrElse(0)
store.transact(io).attempt.map(AddResult.fromUpdate)
}
def loadAll(ids: List[Ident]): F[Vector[RTag]] =
if (ids.isEmpty) Vector.empty.pure[F]
else store.transact(RTag.findAllById(ids))
})
}
Example 23
| Source File: ONode.scala From docspell with GNU General Public License v3.0 | 5 votes |
|
package docspell.backend.ops
import cats.effect.{Effect, Resource}
import cats.implicits._
import docspell.common.syntax.all._
import docspell.common.{Ident, LenientUri, NodeType}
import docspell.store.Store
import docspell.store.records.RNode
import org.log4s._
trait ONode[F[_]] {
def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit]
def unregister(appId: Ident): F[Unit]
}
object ONode {
private[this] val logger = getLogger
def apply[F[_]: Effect](store: Store[F]): Resource[F, ONode[F]] =
Resource.pure[F, ONode[F]](new ONode[F] {
def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] =
for {
node <- RNode(appId, nodeType, uri)
_ <- logger.finfo(s"Registering node ${node.id.id}")
_ <- store.transact(RNode.set(node))
} yield ()
def unregister(appId: Ident): F[Unit] =
logger.finfo(s"Unregister app ${appId.id}") *>
store.transact(RNode.delete(appId)).map(_ => ())
})
}
Example 24
| Source File: OEquipment.scala From docspell with GNU General Public License v3.0 | 5 votes |
|
package docspell.backend.ops
import cats.effect.{Effect, Resource}
import cats.implicits._
import docspell.common.{AccountId, Ident}
import docspell.store.records.{REquipment, RItem}
import docspell.store.{AddResult, Store}
trait OEquipment[F[_]] {
def findAll(account: AccountId, nameQuery: Option[String]): F[Vector[REquipment]]
def add(s: REquipment): F[AddResult]
def update(s: REquipment): F[AddResult]
def delete(id: Ident, collective: Ident): F[AddResult]
}
object OEquipment {
def apply[F[_]: Effect](store: Store[F]): Resource[F, OEquipment[F]] =
Resource.pure[F, OEquipment[F]](new OEquipment[F] {
def findAll(account: AccountId, nameQuery: Option[String]): F[Vector[REquipment]] =
store.transact(REquipment.findAll(account.collective, nameQuery, _.name))
def add(e: REquipment): F[AddResult] = {
def insert = REquipment.insert(e)
def exists = REquipment.existsByName(e.cid, e.name)
val msg = s"An equipment '${e.name}' already exists"
store.add(insert, exists).map(_.fold(identity, _.withMsg(msg), identity))
}
def update(e: REquipment): F[AddResult] = {
def insert = REquipment.update(e)
def exists = REquipment.existsByName(e.cid, e.name)
val msg = s"An equipment '${e.name}' already exists"
store.add(insert, exists).map(_.fold(identity, _.withMsg(msg), identity))
}
def delete(id: Ident, collective: Ident): F[AddResult] = {
val io = for {
n0 <- RItem.removeConcEquip(collective, id)
n1 <- REquipment.delete(id, collective)
} yield n0 + n1
store.transact(io).attempt.map(AddResult.fromUpdate)
}
})
}
Example 25
| Source File: JobQueue.scala From docspell with GNU General Public License v3.0 | 5 votes |
|
package docspell.store.queue
import cats.effect.{Effect, Resource}
import cats.implicits._
import docspell.common._
import docspell.common.syntax.all._
import docspell.store.Store
import docspell.store.queries.QJob
import docspell.store.records.RJob
import org.log4s._
trait JobQueue[F[_]] {
def insertIfNew(job: RJob): F[Unit]
def insertAll(jobs: Seq[RJob]): F[Unit]
def nextJob(
prio: Ident => F[Priority],
worker: Ident,
retryPause: Duration
): F[Option[RJob]]
}
object JobQueue {
private[this] val logger = getLogger
def apply[F[_]: Effect](store: Store[F]): Resource[F, JobQueue[F]] =
Resource.pure[F, JobQueue[F]](new JobQueue[F] {
def nextJob(
prio: Ident => F[Priority],
worker: Ident,
retryPause: Duration
): F[Option[RJob]] =
logger
.ftrace("Select next job") *> QJob.takeNextJob(store)(prio, worker, retryPause)
def insert(job: RJob): F[Unit] =
store
.transact(RJob.insert(job))
.flatMap { n =>
if (n != 1)
Effect[F]
.raiseError(new Exception(s"Inserting job failed. Update count: $n"))
else ().pure[F]
}
def insertIfNew(job: RJob): F[Unit] =
for {
rj <- job.tracker match {
case Some(tid) =>
store.transact(RJob.findNonFinalByTracker(tid))
case None =>
None.pure[F]
}
ret <-
if (rj.isDefined) ().pure[F]
else insert(job)
} yield ret
def insertAll(jobs: Seq[RJob]): F[Unit] =
jobs.toList
.traverse(j => insert(j).attempt)
.map(_.foreach {
case Right(()) =>
case Left(ex) =>
logger.error(ex)("Could not insert job. Skipping it.")
})
})
}
Example 26
| Source File: StoreImpl.scala From docspell with GNU General Public License v3.0 | 5 votes |
|
package docspell.store.impl
import cats.effect.Effect
import cats.implicits._
import docspell.common.Ident
import docspell.store.migrate.FlywayMigrate
import docspell.store.{AddResult, JdbcConfig, Store}
import bitpeace.{Bitpeace, BitpeaceConfig, TikaMimetypeDetect}
import doobie._
import doobie.implicits._
final class StoreImpl[F[_]: Effect](jdbc: JdbcConfig, xa: Transactor[F])
extends Store[F] {
val bitpeaceCfg =
BitpeaceConfig(
"filemeta",
"filechunk",
TikaMimetypeDetect,
Ident.randomId[F].map(_.id)
)
def migrate: F[Int] =
FlywayMigrate.run[F](jdbc)
def transact[A](prg: doobie.ConnectionIO[A]): F[A] =
prg.transact(xa)
def transact[A](prg: fs2.Stream[doobie.ConnectionIO, A]): fs2.Stream[F, A] =
prg.transact(xa)
def bitpeace: Bitpeace[F] =
Bitpeace(bitpeaceCfg, xa)
def add(insert: ConnectionIO[Int], exists: ConnectionIO[Boolean]): F[AddResult] =
for {
save <- transact(insert).attempt
exist <- save.swap.traverse(ex => transact(exists).map(b => (ex, b)))
} yield exist.swap match {
case Right(_) => AddResult.Success
case Left((_, true)) =>
AddResult.EntityExists("Adding failed, because the entity already exists.")
case Left((ex, _)) => AddResult.Failure(ex)
}
}
Example 27
| Source File: JsonLogEncodingTest.scala From cedi-dtrace with Apache License 2.0 | 5 votes |
|
package com.ccadllc.cedi.dtrace
package logging
import cats.effect.{ IO, Effect }
import io.circe._
import io.circe.syntax._
import org.scalacheck.Arbitrary
import org.scalatest.wordspec.AnyWordSpec
import json.encoding._
class JsonLogEncodingTests extends AnyWordSpec with TestSupport {
// format: OFF
val calculateQuarterlySalesTraceContextJson = Json.obj(
"where" -> calculateQuarterlySalesTraceContext.system.data.allValues.asJson,
"root" -> calculateQuarterlySalesTraceContext.currentSpan.root.asJson,
"trace-id" -> calculateQuarterlySalesTraceContext.currentSpan.spanId.traceId.asJson,
"span-id" -> calculateQuarterlySalesTraceContext.currentSpan.spanId.spanId.asJson,
"parent-id" -> calculateQuarterlySalesTraceContext.currentSpan.spanId.parentSpanId.asJson,
"span-name" -> calculateQuarterlySalesTraceContext.currentSpan.spanName.value.asJson,
"start-time" -> calculateQuarterlySalesTraceContext.currentSpan.startTime.asJson,
"span-success" -> calculateQuarterlySalesTraceContext.currentSpan.failure.isEmpty.asJson,
"failure-detail" -> calculateQuarterlySalesTraceContext.currentSpan.failure.map(_.render).asJson,
"span-duration" -> calculateQuarterlySalesTraceContext.currentSpan.duration.toMicros.asJson,
"notes" -> Map(
quarterlySalesUnitsNote.name.value -> quarterlySalesUnitsNoteValue.value.toString,
quarterlySalesGoalReachedNote.name.value -> quarterlySalesGoalReachedNoteValue.value.toString,
salesRegionNote.name.value -> salesRegionNoteValue.value,
quarterlySalesTotalNote.name.value -> quarterlySalesTotalNoteValue.value.toString
).asJson
)
// format: ON
implicit def traceArb[F[_]: Effect]: Arbitrary[TraceContext[F]] = Arbitrary(genTraceContext[F])
"Trace" should { encodeGeneratedJson[TraceContext[IO]] }
"Trace" should { encodeSpecificJson(calculateQuarterlySalesTraceContext, calculateQuarterlySalesTraceContextJson) }
}
Example 28
| Source File: AkkaActorIntermediator.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.sourcing.akka
import akka.actor.ActorSystem
import akka.pattern.ask
import akka.util.Timeout
import cats.effect.{ContextShift, Effect, IO, Timer}
import cats.syntax.all._
import ch.epfl.bluebrain.nexus.sourcing.akka.Msg._
import retry.CatsEffect._
import retry.syntax.all._
import retry.{RetryDetails, RetryPolicy}
import scala.reflect.ClassTag
abstract private[akka] class AkkaActorIntermediator[F[_]: Timer](
name: String,
selection: ActorRefSelection[F],
askTimeout: Timeout
)(implicit F: Effect[F], as: ActorSystem, policy: RetryPolicy[F]) {
implicit private[akka] val contextShift: ContextShift[IO] = IO.contextShift(as.dispatcher)
implicit private[akka] def noop[A]: (A, RetryDetails) => F[Unit] = retry.noop[F, A]
implicit private val timeout: Timeout = askTimeout
private[akka] def send[M <: Msg, Reply, A](id: String, msg: M, f: Reply => A)(implicit
Reply: ClassTag[Reply]
): F[A] =
selection(name, id).flatMap { ref =>
val future = IO(ref ? msg)
val fa = IO.fromFuture(future).to[F]
fa.flatMap[A] {
case Reply(value) => F.pure(f(value))
case te: TypeError => F.raiseError(te)
case um: UnexpectedMsgId => F.raiseError(um)
case cet: CommandEvaluationTimeout[_] => F.raiseError(cet)
case cee: CommandEvaluationError[_] => F.raiseError(cee)
case other => F.raiseError(TypeError(id, Reply.runtimeClass.getSimpleName, other))
}
.retryingOnAllErrors[Throwable]
}
}
Example 29
| Source File: ElasticSearchBaseClient.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.commons.es.client
import akka.http.scaladsl.model.StatusCodes.GatewayTimeout
import akka.http.scaladsl.model.{HttpRequest, StatusCode, StatusCodes}
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.commons.es.client.ElasticSearchBaseClient._
import ch.epfl.bluebrain.nexus.commons.es.client.ElasticSearchFailure.{ElasticServerError, ElasticUnexpectedError}
import ch.epfl.bluebrain.nexus.commons.http.HttpClient.UntypedHttpClient
import ch.epfl.bluebrain.nexus.sourcing.RetryStrategyConfig
import com.typesafe.scalalogging.Logger
import retry.CatsEffect._
import retry.syntax.all._
import retry.{RetryDetails, RetryPolicy}
import scala.util.control.NonFatal
private[client] def sanitize(index: String, allowWildCard: Boolean): String = {
val regex = if (allowWildCard) """[\s|"|\\|<|>|\||,|/|?]""" else """[\s|"|*|\\|<|>|\||,|/|?]"""
index.replaceAll(regex, "_").dropWhile(_ == '_')
}
}
object ElasticSearchBaseClient {
private[client] val docType = "_doc"
private[client] val source = "_source"
private[client] val anyIndexPath = "_all"
private[client] val ignoreUnavailable = "ignore_unavailable"
private[client] val allowNoIndices = "allow_no_indices"
private[client] val trackTotalHits = "track_total_hits"
}
Example 30
| Source File: ArchiveCache.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.archives
import akka.actor.{ActorSystem, NotInfluenceReceiveTimeout}
import cats.Monad
import cats.data.OptionT
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.kg.archives.ArchiveCache._
import ch.epfl.bluebrain.nexus.kg.config.KgConfig.ArchivesConfig
import ch.epfl.bluebrain.nexus.kg.resources.ResId
import ch.epfl.bluebrain.nexus.sourcing.StateMachine
import ch.epfl.bluebrain.nexus.sourcing.akka.StopStrategy
import ch.epfl.bluebrain.nexus.sourcing.akka.statemachine.AkkaStateMachine
import retry.RetryPolicy
class ArchiveCache[F[_]: Monad](ref: StateMachine[F, String, State, Command, Unit]) {
def put(value: Archive): OptionT[F, Archive] =
OptionT(ref.evaluate(value.resId.show, Write(value)).map(_.toOption.flatten))
}
object ArchiveCache {
private[archives] type State = Option[Archive]
private[archives] type Command = Write
final private[archives] case class Write(bundle: Archive) extends NotInfluenceReceiveTimeout
final def apply[F[_]: Timer](implicit as: ActorSystem, cfg: ArchivesConfig, F: Effect[F]): F[ArchiveCache[F]] = {
implicit val retryPolicy: RetryPolicy[F] = cfg.cache.retry.retryPolicy[F]
val invalidationStrategy = StopStrategy.lapsedSinceLastInteraction[State, Command](cfg.cacheInvalidateAfter)
val evaluate: (State, Command) => F[Either[Unit, State]] = {
case (None, Write(bundle)) => F.pure(Right(Some(bundle)))
case (Some(_), _) => F.pure(Left(())) // It already exists, so we don't want to replace it
}
AkkaStateMachine
.sharded[F]("archives", None, evaluate, invalidationStrategy, cfg.cache.akkaStateMachineConfig, cfg.cache.shards)
.map(new ArchiveCache[F](_))
}
}
Example 31
| Source File: ElasticSearchIndexer.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.indexing
import akka.actor.{ActorRef, ActorSystem, Props}
import akka.stream.scaladsl.Source
import akka.util.Timeout
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.admin.client.types.Project
import ch.epfl.bluebrain.nexus.commons.es.client.ElasticSearchClient
import ch.epfl.bluebrain.nexus.commons.es.client.ElasticSearchClient.BulkOp
import ch.epfl.bluebrain.nexus.kg.indexing.View.ElasticSearchView
import ch.epfl.bluebrain.nexus.kg.resources._
import ch.epfl.bluebrain.nexus.kg.routes.Clients
import ch.epfl.bluebrain.nexus.service.config.ServiceConfig
import ch.epfl.bluebrain.nexus.sourcing.projections.ProgressFlow.ProgressFlowElem
import ch.epfl.bluebrain.nexus.sourcing.projections.ProjectionProgress.NoProgress
import ch.epfl.bluebrain.nexus.sourcing.projections._
import com.typesafe.scalalogging.Logger
import scala.concurrent.ExecutionContext
// $COVERAGE-OFF$
@SuppressWarnings(Array("MaxParameters"))
object ElasticSearchIndexer {
implicit private val log: Logger = Logger[ElasticSearchIndexer.type]
final def start[F[_]: Timer](
view: ElasticSearchView,
resources: Resources[F],
project: Project,
restartOffset: Boolean
)(implicit
as: ActorSystem,
actorInitializer: (Props, String) => ActorRef,
projections: Projections[F, String],
F: Effect[F],
clients: Clients[F],
config: ServiceConfig
): StreamSupervisor[F, ProjectionProgress] = {
implicit val ec: ExecutionContext = as.dispatcher
implicit val p: Project = project
implicit val indexing: IndexingConfig = config.kg.elasticSearch.indexing
implicit val metadataOpts: MetadataOptions = MetadataOptions(linksAsIri = true, expandedLinks = true)
implicit val tm: Timeout = Timeout(config.kg.elasticSearch.askTimeout)
val client: ElasticSearchClient[F] = clients.elasticSearch.withRetryPolicy(config.kg.elasticSearch.indexing.retry)
def deleteOrIndex(res: ResourceV): Option[BulkOp] =
if (res.deprecated && !view.filter.includeDeprecated) Some(delete(res))
else view.toDocument(res).map(doc => BulkOp.Index(view.index, res.id.value.asString, doc))
def delete(res: ResourceV): BulkOp =
BulkOp.Delete(view.index, res.id.value.asString)
val initFetchProgressF: F[ProjectionProgress] =
if (restartOffset)
projections.recordProgress(view.progressId, NoProgress) >> view.createIndex >> F.pure(NoProgress)
else view.createIndex >> projections.progress(view.progressId)
val sourceF: F[Source[ProjectionProgress, _]] = initFetchProgressF.map { initial =>
val flow = ProgressFlowElem[F, Any]
.collectCast[Event]
.groupedWithin(indexing.batch, indexing.batchTimeout)
.distinct()
.mapAsync(view.toResource(resources, _))
.collectSome[ResourceV]
.collect {
case res if view.allowedSchemas(res) && view.allowedTypes(res) => deleteOrIndex(res)
case res if view.allowedSchemas(res) => Some(delete(res))
}
.collectSome[BulkOp]
.runAsyncBatch(client.bulk(_))()
.mergeEmit()
.toPersistedProgress(view.progressId, initial)
cassandraSource(s"project=${view.ref.id}", view.progressId, initial.minProgress.offset)
.via(flow)
.via(kamonViewMetricsFlow(view, project))
}
StreamSupervisor.start(sourceF, view.progressId, actorInitializer)
}
}
// $COVERAGE-ON$
Example 32
| Source File: StorageIndexer.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.indexing
import java.time.Instant
import akka.actor.ActorSystem
import akka.stream.scaladsl.{Flow, Source}
import akka.util.Timeout
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.admin.client.AdminClient
import ch.epfl.bluebrain.nexus.iam.auth.AccessToken
import ch.epfl.bluebrain.nexus.kg.cache.{ProjectCache, StorageCache}
import ch.epfl.bluebrain.nexus.kg.config.KgConfig.StorageConfig
import ch.epfl.bluebrain.nexus.kg.resources._
import ch.epfl.bluebrain.nexus.kg.storage.Storage
import ch.epfl.bluebrain.nexus.service.config.ServiceConfig
import ch.epfl.bluebrain.nexus.service.config.Vocabulary.nxv
import ch.epfl.bluebrain.nexus.sourcing.projections.ProgressFlow.{PairMsg, ProgressFlowElem}
import ch.epfl.bluebrain.nexus.sourcing.projections._
import com.typesafe.scalalogging.Logger
import scala.concurrent.ExecutionContext
// $COVERAGE-OFF$
object StorageIndexer {
implicit private val log = Logger[StorageIndexer.type]
def start[F[_]: Timer](storages: Storages[F], storageCache: StorageCache[F])(implicit
projectCache: ProjectCache[F],
F: Effect[F],
as: ActorSystem,
projectInitializer: ProjectInitializer[F],
adminClient: AdminClient[F],
config: ServiceConfig
): StreamSupervisor[F, Unit] = {
implicit val authToken: Option[AccessToken] = config.serviceAccount.credentials
implicit val indexing: IndexingConfig = config.kg.keyValueStore.indexing
implicit val ec: ExecutionContext = as.dispatcher
implicit val tm: Timeout = Timeout(config.kg.keyValueStore.askTimeout)
implicit val storageConfig: StorageConfig = config.kg.storage
val name = "storage-indexer"
def toStorage(event: Event): F[Option[(Storage, Instant)]] =
fetchProject(event.organization, event.id.parent, event.subject).flatMap { implicit project =>
storages.fetchStorage(event.id).value.map {
case Left(err) =>
log.error(s"Error on event '${event.id.show} (rev = ${event.rev})', cause: '${err.msg}'")
None
case Right(timedStorage) => Some(timedStorage)
}
}
val source: Source[PairMsg[Any], _] = cassandraSource(s"type=${nxv.Storage.value.show}", name)
val flow: Flow[PairMsg[Any], Unit, _] = ProgressFlowElem[F, Any]
.collectCast[Event]
.groupedWithin(indexing.batch, indexing.batchTimeout)
.distinct()
.mergeEmit()
.mapAsync(toStorage)
.collectSome[(Storage, Instant)]
.runAsync { case (storage, instant) => storageCache.put(storage)(instant) }()
.flow
.map(_ => ())
StreamSupervisor.startSingleton(F.delay(source.via(flow)), name)
}
}
// $COVERAGE-ON$
Example 33
| Source File: SparqlIndexer.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.indexing
import akka.actor.{ActorRef, ActorSystem, Props}
import akka.stream.scaladsl.Source
import akka.util.Timeout
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.admin.client.types.Project
import ch.epfl.bluebrain.nexus.commons.sparql.client.{BlazegraphClient, SparqlWriteQuery}
import ch.epfl.bluebrain.nexus.kg.indexing.View.SparqlView
import ch.epfl.bluebrain.nexus.kg.resources._
import ch.epfl.bluebrain.nexus.kg.routes.Clients
import ch.epfl.bluebrain.nexus.service.config.ServiceConfig
import ch.epfl.bluebrain.nexus.sourcing.projections.ProgressFlow.ProgressFlowElem
import ch.epfl.bluebrain.nexus.sourcing.projections.ProjectionProgress.NoProgress
import ch.epfl.bluebrain.nexus.sourcing.projections._
import scala.concurrent.ExecutionContext
// $COVERAGE-OFF$
@SuppressWarnings(Array("MaxParameters"))
object SparqlIndexer {
final def start[F[_]: Timer](
view: SparqlView,
resources: Resources[F],
project: Project,
restartOffset: Boolean
)(implicit
as: ActorSystem,
actorInitializer: (Props, String) => ActorRef,
projections: Projections[F, String],
F: Effect[F],
clients: Clients[F],
config: ServiceConfig
): StreamSupervisor[F, ProjectionProgress] = {
implicit val ec: ExecutionContext = as.dispatcher
implicit val p: Project = project
implicit val indexing: IndexingConfig = config.kg.sparql.indexing
implicit val metadataOpts: MetadataOptions = MetadataOptions(linksAsIri = true, expandedLinks = true)
implicit val tm: Timeout = Timeout(config.kg.sparql.askTimeout)
val client: BlazegraphClient[F] =
clients.sparql.copy(namespace = view.index).withRetryPolicy(config.kg.sparql.indexing.retry)
def buildInsertOrDeleteQuery(res: ResourceV): SparqlWriteQuery =
if (res.deprecated && !view.filter.includeDeprecated) view.buildDeleteQuery(res)
else view.buildInsertQuery(res)
val initFetchProgressF: F[ProjectionProgress] =
if (restartOffset)
projections.recordProgress(view.progressId, NoProgress) >> view.createIndex >> F.pure(NoProgress)
else view.createIndex >> projections.progress(view.progressId)
val sourceF: F[Source[ProjectionProgress, _]] = initFetchProgressF.map { initial =>
val flow = ProgressFlowElem[F, Any]
.collectCast[Event]
.groupedWithin(indexing.batch, indexing.batchTimeout)
.distinct()
.mapAsync(view.toResource(resources, _))
.collectSome[ResourceV]
.collect {
case res if view.allowedSchemas(res) && view.allowedTypes(res) => buildInsertOrDeleteQuery(res)
case res if view.allowedSchemas(res) => view.buildDeleteQuery(res)
}
.runAsyncBatch(client.bulk(_))()
.mergeEmit()
.toPersistedProgress(view.progressId, initial)
cassandraSource(s"project=${view.ref.id}", view.progressId, initial.minProgress.offset)
.via(flow)
.via(kamonViewMetricsFlow(view, project))
}
StreamSupervisor.start(sourceF, view.progressId, actorInitializer)
}
}
// $COVERAGE-ON$
Example 34
| Source File: ResolverIndexer.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.indexing
import akka.actor.ActorSystem
import akka.stream.scaladsl.{Flow, Source}
import akka.util.Timeout
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.admin.client.AdminClient
import ch.epfl.bluebrain.nexus.iam.auth.AccessToken
import ch.epfl.bluebrain.nexus.kg.cache.{ProjectCache, ResolverCache}
import ch.epfl.bluebrain.nexus.kg.resolve.Resolver
import ch.epfl.bluebrain.nexus.kg.resources._
import ch.epfl.bluebrain.nexus.service.config.ServiceConfig
import ch.epfl.bluebrain.nexus.service.config.Vocabulary.nxv
import ch.epfl.bluebrain.nexus.sourcing.projections.ProgressFlow.{PairMsg, ProgressFlowElem}
import ch.epfl.bluebrain.nexus.sourcing.projections._
import com.typesafe.scalalogging.Logger
import scala.concurrent.ExecutionContext
// $COVERAGE-OFF$
object ResolverIndexer {
implicit private val log = Logger[ResolverIndexer.type]
final def start[F[_]: Timer](resolvers: Resolvers[F], resolverCache: ResolverCache[F])(implicit
projectCache: ProjectCache[F],
as: ActorSystem,
F: Effect[F],
projectInitializer: ProjectInitializer[F],
adminClient: AdminClient[F],
config: ServiceConfig
): StreamSupervisor[F, Unit] = {
implicit val authToken: Option[AccessToken] = config.serviceAccount.credentials
implicit val indexing: IndexingConfig = config.kg.keyValueStore.indexing
implicit val ec: ExecutionContext = as.dispatcher
implicit val tm: Timeout = Timeout(config.kg.keyValueStore.askTimeout)
val name = "resolver-indexer"
def toResolver(event: Event): F[Option[Resolver]] =
fetchProject(event.organization, event.id.parent, event.subject).flatMap { implicit project =>
resolvers.fetchResolver(event.id).value.map {
case Left(err) =>
log.error(s"Error on event '${event.id.show} (rev = ${event.rev})', cause: '${err.msg}'")
None
case Right(resolver) => Some(resolver)
}
}
val source: Source[PairMsg[Any], _] = cassandraSource(s"type=${nxv.Resolver.value.show}", name)
val flow: Flow[PairMsg[Any], Unit, _] = ProgressFlowElem[F, Any]
.collectCast[Event]
.groupedWithin(indexing.batch, indexing.batchTimeout)
.distinct()
.mergeEmit()
.mapAsync(toResolver)
.collectSome[Resolver]
.runAsync(resolverCache.put)()
.flow
.map(_ => ())
StreamSupervisor.startSingleton(F.delay(source.via(flow)), name)
}
}
// $COVERAGE-ON$
Example 35
| Source File: ViewIndexer.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.indexing
import akka.actor.ActorSystem
import akka.stream.scaladsl.{Flow, Source}
import akka.util.Timeout
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.admin.client.AdminClient
import ch.epfl.bluebrain.nexus.iam.auth.AccessToken
import ch.epfl.bluebrain.nexus.kg.cache.{ProjectCache, ViewCache}
import ch.epfl.bluebrain.nexus.kg.resources._
import ch.epfl.bluebrain.nexus.service.config.ServiceConfig
import ch.epfl.bluebrain.nexus.service.config.Vocabulary.nxv
import ch.epfl.bluebrain.nexus.sourcing.projections.ProgressFlow.{PairMsg, ProgressFlowElem}
import ch.epfl.bluebrain.nexus.sourcing.projections._
import com.typesafe.scalalogging.Logger
import scala.concurrent.ExecutionContext
// $COVERAGE-OFF$
object ViewIndexer {
implicit private val log = Logger[ViewIndexer.type]
def start[F[_]: Timer](views: Views[F], viewCache: ViewCache[F])(implicit
projectCache: ProjectCache[F],
F: Effect[F],
as: ActorSystem,
projectInitializer: ProjectInitializer[F],
adminClient: AdminClient[F],
config: ServiceConfig
): StreamSupervisor[F, Unit] = {
implicit val authToken: Option[AccessToken] = config.serviceAccount.credentials
implicit val indexing: IndexingConfig = config.kg.keyValueStore.indexing
implicit val ec: ExecutionContext = as.dispatcher
implicit val tm: Timeout = Timeout(config.kg.keyValueStore.askTimeout)
val name = "view-indexer"
def toView(event: Event): F[Option[View]] =
fetchProject(event.organization, event.id.parent, event.subject).flatMap { implicit project =>
views.fetchView(event.id).value.map {
case Left(err) =>
log.error(s"Error on event '${event.id.show} (rev = ${event.rev})', cause: '${err.msg}'")
None
case Right(view) => Some(view)
}
}
val source: Source[PairMsg[Any], _] = cassandraSource(s"type=${nxv.View.value.show}", name)
val flow: Flow[PairMsg[Any], Unit, _] = ProgressFlowElem[F, Any]
.collectCast[Event]
.groupedWithin(indexing.batch, indexing.batchTimeout)
.distinct()
.mergeEmit()
.mapAsync(toView)
.collectSome[View]
.runAsync(viewCache.put)()
.flow
.map(_ => ())
StreamSupervisor.startSingleton(F.delay(source.via(flow)), name)
}
}
// $COVERAGE-ON$
Example 36
| Source File: ResolverCache.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.cache
import java.util.UUID
import java.util.concurrent.ConcurrentHashMap
import akka.actor.ActorSystem
import cats.Monad
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.commons.cache.{KeyValueStore, KeyValueStoreConfig}
import ch.epfl.bluebrain.nexus.kg.cache.Cache._
import ch.epfl.bluebrain.nexus.kg.resolve.Resolver
import ch.epfl.bluebrain.nexus.kg.resources.ProjectIdentifier.ProjectRef
import ch.epfl.bluebrain.nexus.rdf.Iri.AbsoluteIri
class ResolverCache[F[_]: Effect: Timer] private (projectToCache: ConcurrentHashMap[UUID, ResolverProjectCache[F]])(
implicit
as: ActorSystem,
config: KeyValueStoreConfig
) {
private class ResolverProjectCache[F[_]: Monad] private (store: KeyValueStore[F, AbsoluteIri, Resolver])
extends Cache[F, AbsoluteIri, Resolver](store) {
implicit private val ordering: Ordering[Resolver] = Ordering.by(_.priority)
def get: F[List[Resolver]] = store.values.map(_.toList.sorted)
def put(resolver: Resolver): F[Unit] =
if (resolver.deprecated) store.remove(resolver.id)
else store.put(resolver.id, resolver)
}
private object ResolverProjectCache {
def apply[F[_]: Effect: Timer](
project: ProjectRef
)(implicit as: ActorSystem, config: KeyValueStoreConfig): ResolverProjectCache[F] =
new ResolverProjectCache(KeyValueStore.distributed(s"resolver-${project.id}", (_, resolver) => resolver.rev))
}
object ResolverCache {
def apply[F[_]: Effect: Timer](implicit as: ActorSystem, config: KeyValueStoreConfig): ResolverCache[F] =
new ResolverCache(new ConcurrentHashMap[UUID, ResolverProjectCache[F]]())
}
Example 37
| Source File: StorageCache.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.cache
import java.time.{Clock, Instant}
import java.util.UUID
import java.util.concurrent.ConcurrentHashMap
import akka.actor.ActorSystem
import cats.Monad
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.commons.cache.{KeyValueStore, KeyValueStoreConfig}
import ch.epfl.bluebrain.nexus.kg.RevisionedValue
import ch.epfl.bluebrain.nexus.kg.cache.Cache._
import ch.epfl.bluebrain.nexus.kg.cache.StorageProjectCache._
import ch.epfl.bluebrain.nexus.kg.resources.ProjectIdentifier.ProjectRef
import ch.epfl.bluebrain.nexus.kg.storage.Storage
import ch.epfl.bluebrain.nexus.rdf.Iri.AbsoluteIri
class StorageCache[F[_]: Effect: Timer] private (projectToCache: ConcurrentHashMap[UUID, StorageProjectCache[F]])(
implicit
as: ActorSystem,
config: KeyValueStoreConfig,
clock: Clock
) {
private class StorageProjectCache[F[_]: Monad] private (store: KeyValueStore[F, AbsoluteIri, RevisionedStorage])
extends Cache[F, AbsoluteIri, RevisionedStorage](store) {
implicit private val ordering: Ordering[RevisionedStorage] = Ordering.by((s: RevisionedStorage) => s.rev).reverse
implicit private def revisioned(storage: Storage)(implicit instant: Instant): RevisionedStorage =
RevisionedValue(instant.toEpochMilli, storage)
def get: F[List[Storage]] =
store.values.map(_.toList.sorted.map(_.value))
def getDefault: F[Option[Storage]] =
get.map(_.collectFirst { case storage if storage.default => storage })
def getBy(id: AbsoluteIri): F[Option[Storage]] =
get(id).map(_.collectFirst { case RevisionedValue(_, storage) if storage.id == id => storage })
def put(storage: Storage)(implicit instant: Instant): F[Unit] =
if (storage.deprecated) store.remove(storage.id)
else store.put(storage.id, storage)
}
private object StorageProjectCache {
type RevisionedStorage = RevisionedValue[Storage]
def apply[F[_]: Effect: Timer](
project: ProjectRef
)(implicit as: ActorSystem, config: KeyValueStoreConfig): StorageProjectCache[F] =
new StorageProjectCache(
KeyValueStore.distributed(s"storage-${project.id}", (_, storage) => storage.value.rev)
)
}
object StorageCache {
def apply[F[_]: Timer: Effect](implicit as: ActorSystem, config: KeyValueStoreConfig, clock: Clock): StorageCache[F] =
new StorageCache(new ConcurrentHashMap[UUID, StorageProjectCache[F]]())
}
Example 38
| Source File: RemoteDiskStorageOperations.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.storage
import akka.http.scaladsl.model.Uri
import cats.Applicative
import cats.effect.Effect
import cats.implicits._
import ch.epfl.bluebrain.nexus.iam.auth.AccessToken
import ch.epfl.bluebrain.nexus.iam.client.types.AuthToken
import ch.epfl.bluebrain.nexus.kg.resources.ResId
import ch.epfl.bluebrain.nexus.kg.resources.file.File._
import ch.epfl.bluebrain.nexus.kg.storage.Storage._
import ch.epfl.bluebrain.nexus.storage.client.StorageClient
import ch.epfl.bluebrain.nexus.storage.client.types.FileAttributes.{Digest => StorageDigest}
import ch.epfl.bluebrain.nexus.storage.client.types.{FileAttributes => StorageFileAttributes}
object RemoteDiskStorageOperations {
// TODO: Remove when migrating ADMIN client
implicit private def oldTokenConversion(implicit token: Option[AccessToken]): Option[AuthToken] =
token.map(t => AuthToken(t.value))
implicit private def toDigest(digest: StorageDigest): Digest = Digest(digest.algorithm, digest.value)
final class FetchAttributes[F[_]](storage: RemoteDiskStorage, client: StorageClient[F])
extends FetchFileAttributes[F] {
implicit val cred = storage.credentials.map(AccessToken)
override def apply(relativePath: Uri.Path): F[StorageFileAttributes] =
client.getAttributes(storage.folder, relativePath)
}
}
Example 39
| Source File: Main.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.storage
import java.nio.file.Paths
import java.time.Clock
import akka.actor.ActorSystem
import akka.event.{Logging, LoggingAdapter}
import akka.http.scaladsl.Http
import akka.http.scaladsl.server.Route
import akka.util.Timeout
import cats.effect.Effect
import ch.epfl.bluebrain.nexus.storage.Storages.DiskStorage
import ch.epfl.bluebrain.nexus.storage.attributes.AttributesCache
import ch.epfl.bluebrain.nexus.storage.config.{AppConfig, Settings}
import ch.epfl.bluebrain.nexus.storage.config.AppConfig._
import ch.epfl.bluebrain.nexus.storage.routes.Routes
import com.typesafe.config.{Config, ConfigFactory}
import kamon.Kamon
import monix.eval.Task
import monix.execution.Scheduler
import scala.concurrent.duration._
import scala.concurrent.{Await, ExecutionContext, Future}
import scala.util.{Failure, Success}
//noinspection TypeAnnotation
// $COVERAGE-OFF$
object Main {
def loadConfig(): Config = {
val cfg = sys.env.get("STORAGE_CONFIG_FILE") orElse sys.props.get("storage.config.file") map { str =>
val file = Paths.get(str).toAbsolutePath.toFile
ConfigFactory.parseFile(file)
} getOrElse ConfigFactory.empty()
(cfg withFallback ConfigFactory.load()).resolve()
}
def setupMonitoring(config: Config): Unit = {
if (sys.env.getOrElse("KAMON_ENABLED", "false").toBoolean) {
Kamon.reconfigure(config)
Kamon.loadModules()
}
}
def shutdownMonitoring(): Unit = {
if (sys.env.getOrElse("KAMON_ENABLED", "false").toBoolean) {
Await.result(Kamon.stopModules(), 10.seconds)
}
}
@SuppressWarnings(Array("UnusedMethodParameter"))
def main(args: Array[String]): Unit = {
val config = loadConfig()
setupMonitoring(config)
implicit val appConfig: AppConfig = Settings(config).appConfig
implicit val as: ActorSystem = ActorSystem(appConfig.description.fullName, config)
implicit val ec: ExecutionContext = as.dispatcher
implicit val eff: Effect[Task] = Task.catsEffect(Scheduler.global)
implicit val iamIdentities: IamIdentitiesClient[Task] = new IamIdentitiesClient[Task](appConfig.iam)
implicit val timeout = Timeout(1.minute)
implicit val clock = Clock.systemUTC
val storages: Storages[Task, AkkaSource] =
new DiskStorage(appConfig.storage, appConfig.digest, AttributesCache[Task, AkkaSource])
val logger: LoggingAdapter = Logging(as, getClass)
logger.info("==== Cluster is Live ====")
val routes: Route = Routes(storages)
val httpBinding: Future[Http.ServerBinding] = {
Http().bindAndHandle(routes, appConfig.http.interface, appConfig.http.port)
}
httpBinding onComplete {
case Success(binding) =>
logger.info(s"Bound to ${binding.localAddress.getHostString}: ${binding.localAddress.getPort}")
case Failure(th) =>
logger.error(th, "Failed to perform an http binding on {}:{}", appConfig.http.interface, appConfig.http.port)
Await.result(as.terminate(), 10.seconds)
}
as.registerOnTermination {
shutdownMonitoring()
}
// attempt to leave the cluster before shutting down
val _ = sys.addShutdownHook {
Await.result(as.terminate().map(_ => ()), 10.seconds)
}
}
}
// $COVERAGE-ON$
Example 40
| Source File: AttributesCache.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.storage.attributes
import java.nio.file.Path
import java.time.Clock
import akka.actor.{ActorRef, ActorSystem}
import akka.pattern.{ask, AskTimeoutException}
import akka.util.Timeout
import cats.effect.{ContextShift, Effect, IO}
import cats.implicits._
import ch.epfl.bluebrain.nexus.storage.File.FileAttributes
import ch.epfl.bluebrain.nexus.storage.StorageError.{InternalError, OperationTimedOut}
import ch.epfl.bluebrain.nexus.storage.attributes.AttributesCacheActor.Protocol._
import ch.epfl.bluebrain.nexus.storage.config.AppConfig.DigestConfig
import com.typesafe.scalalogging.Logger
import scala.util.control.NonFatal
trait AttributesCache[F[_]] {
def asyncComputePut(filePath: Path, algorithm: String): Unit
}
object AttributesCache {
private[this] val logger = Logger[this.type]
def apply[F[_], Source](implicit
system: ActorSystem,
clock: Clock,
tm: Timeout,
F: Effect[F],
computation: AttributesComputation[F, Source],
config: DigestConfig
): AttributesCache[F] =
apply(system.actorOf(AttributesCacheActor.props(computation)))
private[attributes] def apply[F[_]](
underlying: ActorRef
)(implicit system: ActorSystem, tm: Timeout, F: Effect[F]): AttributesCache[F] =
new AttributesCache[F] {
implicit private val contextShift: ContextShift[IO] = IO.contextShift(system.dispatcher)
override def get(filePath: Path): F[FileAttributes] =
IO.fromFuture(IO.shift(system.dispatcher) >> IO(underlying ? Get(filePath)))
.to[F]
.flatMap[FileAttributes] {
case attributes: FileAttributes => F.pure(attributes)
case other =>
logger.error(s"Received unexpected reply from the file attributes cache: '$other'")
F.raiseError(InternalError("Unexpected reply from the file attributes cache"))
}
.recoverWith {
case _: AskTimeoutException =>
F.raiseError(OperationTimedOut("reply from the file attributes cache timed out"))
case NonFatal(th) =>
logger.error("Exception caught while exchanging messages with the file attributes cache", th)
F.raiseError(InternalError("Exception caught while exchanging messages with the file attributes cache"))
}
override def asyncComputePut(filePath: Path, algorithm: String): Unit =
underlying ! Compute(filePath)
}
}
Example 41
| Source File: AttributesComputation.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.storage.attributes
import java.nio.file.{Files, Path}
import java.security.MessageDigest
import akka.http.scaladsl.model.HttpCharsets.`UTF-8`
import akka.http.scaladsl.model.MediaTypes.{`application/octet-stream`, `application/x-tar`}
import akka.http.scaladsl.model.{ContentType, MediaType, MediaTypes}
import akka.stream.Materializer
import akka.stream.scaladsl.{Keep, Sink}
import akka.util.ByteString
import cats.effect.Effect
import cats.implicits._
import ch.epfl.bluebrain.nexus.storage.File.{Digest, FileAttributes}
import ch.epfl.bluebrain.nexus.storage.StorageError.InternalError
import ch.epfl.bluebrain.nexus.storage._
import org.apache.commons.io.FilenameUtils
import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success, Try}
trait AttributesComputation[F[_], Source] {
implicit def akkaAttributes[F[_]](implicit
ec: ExecutionContext,
mt: Materializer,
F: Effect[F]
): AttributesComputation[F, AkkaSource] =
(path: Path, algorithm: String) => {
if (!Files.exists(path)) F.raiseError(InternalError(s"Path not found '$path'"))
else
Try(MessageDigest.getInstance(algorithm)) match {
case Success(msgDigest) =>
val isDir = Files.isDirectory(path)
val source = if (isDir) folderSource(path) else fileSource(path)
source
.alsoToMat(sinkSize)(Keep.right)
.toMat(sinkDigest(msgDigest)) { (bytesF, digestF) =>
(bytesF, digestF).mapN {
case (bytes, digest) => FileAttributes(path.toAkkaUri, bytes, digest, detectMediaType(path, isDir))
}
}
.run()
.to[F]
case Failure(_) => F.raiseError(InternalError(s"Invalid algorithm '$algorithm'."))
}
}
}
Example 42
| Source File: IamIdentitiesClient.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.storage
import akka.actor.ActorSystem
import akka.http.scaladsl.Http
import akka.http.scaladsl.client.RequestBuilding.Get
import akka.http.scaladsl.model.HttpRequest
import akka.http.scaladsl.model.headers.OAuth2BearerToken
import akka.http.scaladsl.unmarshalling.FromEntityUnmarshaller
import akka.util.ByteString
import cats.effect.{ContextShift, Effect, IO}
import cats.implicits._
import ch.epfl.bluebrain.nexus.rdf.implicits._
import ch.epfl.bluebrain.nexus.storage.IamIdentitiesClient.Identity._
import ch.epfl.bluebrain.nexus.storage.IamIdentitiesClient._
import ch.epfl.bluebrain.nexus.storage.IamIdentitiesClientError.IdentitiesSerializationError
import ch.epfl.bluebrain.nexus.storage.config.IamClientConfig
import de.heikoseeberger.akkahttpcirce.ErrorAccumulatingCirceSupport.{DecodingFailures => AccDecodingFailures}
import io.circe.Decoder.Result
import io.circe.{Decoder, DecodingFailure, HCursor}
import scala.concurrent.ExecutionContext
class IamIdentitiesClient[F[_]](config: IamClientConfig)(implicit F: Effect[F], as: ActorSystem)
extends JsonLdCirceSupport {
private val um: FromEntityUnmarshaller[Caller] = unmarshaller[Caller]
implicit private val ec: ExecutionContext = as.dispatcher
implicit private val contextShift: ContextShift[IO] = IO.contextShift(ec)
def apply()(implicit credentials: Option[AccessToken]): F[Caller] =
credentials match {
case Some(token) => execute(Get(config.identitiesIri.asAkka).addCredentials(OAuth2BearerToken(token.value)))
case None => F.pure(Caller.anonymous)
}
private def execute(req: HttpRequest): F[Caller] = {
IO.fromFuture(IO(Http().singleRequest(req))).to[F].flatMap { resp =>
if (resp.status.isSuccess())
IO.fromFuture(IO(um(resp.entity))).to[F].recoverWith {
case err: AccDecodingFailures => F.raiseError(IdentitiesSerializationError(err.getMessage))
case err: Error => F.raiseError(IdentitiesSerializationError(err.getMessage))
}
else
IO.fromFuture(IO(resp.entity.dataBytes.runFold(ByteString(""))(_ ++ _).map(_.utf8String)))
.to[F]
.flatMap { err => F.raiseError(IamIdentitiesClientError.unsafe(resp.status, err)) }
}
}
}
object IamIdentitiesClient {
final case class Authenticated(realm: String) extends Identity
private def decodeAnonymous(hc: HCursor): Result[Subject] =
hc.get[String]("@type").flatMap {
case "Anonymous" => Right(Anonymous)
case _ => Left(DecodingFailure("Cannot decode Anonymous Identity", hc.history))
}
private def decodeUser(hc: HCursor): Result[Subject] =
(hc.get[String]("subject"), hc.get[String]("realm")).mapN {
case (subject, realm) => User(subject, realm)
}
private def decodeGroup(hc: HCursor): Result[Identity] =
(hc.get[String]("group"), hc.get[String]("realm")).mapN {
case (group, realm) => Group(group, realm)
}
private def decodeAuthenticated(hc: HCursor): Result[Identity] =
hc.get[String]("realm").map(Authenticated)
private val attempts =
List[HCursor => Result[Identity]](decodeAnonymous, decodeUser, decodeGroup, decodeAuthenticated)
implicit val identityDecoder: Decoder[Identity] =
Decoder.instance { hc =>
attempts.foldLeft(Left(DecodingFailure("Unexpected", hc.history)): Result[Identity]) {
case (acc @ Right(_), _) => acc
case (_, f) => f(hc)
}
}
}
}
Example 43
| Source File: BookingRoutes.scala From ticket-booking-aecor with Apache License 2.0 | 5 votes |
|
package ru.pavkin.booking.booking.endpoint
import cats.effect.Effect
import cats.implicits._
import io.circe.syntax._
import org.http4s.HttpRoutes
import org.http4s.circe._
import org.http4s.dsl.Http4sDsl
import ru.pavkin.booking.common.models.{BookingKey, ClientId}
final class BookingRoutes[F[_]: Effect](ops: BookingEndpoint[F]) extends Http4sDsl[F] {
implicit val placeBookingDecoder = jsonOf[F, PlaceBookingRequest]
implicit val cancelBookingDecoder = jsonOf[F, CancelBookingRequest]
private val placeBooking: HttpRoutes[F] = HttpRoutes.of {
case r @ POST -> Root / userId / "bookings" =>
r.as[PlaceBookingRequest]
.flatMap(
r =>
ops.placeBooking(ClientId(userId), r.concertId, r.seats).flatMap {
case Left(err) => BadRequest(err.toString)
case Right(_) => Ok()
}
)
}
private val cancelBooking: HttpRoutes[F] = HttpRoutes.of {
case r @ DELETE -> Root / userId / "bookings" / bookingId =>
r.as[CancelBookingRequest]
.flatMap(
r =>
ops.cancelBooking(ClientId(userId), BookingKey(bookingId), r.reason).flatMap {
case Left(err) => BadRequest(err.toString)
case Right(_) => Ok()
}
)
}
private val clientBookings: HttpRoutes[F] = HttpRoutes.of {
case GET -> Root / userId / "bookings" =>
ops.clientBookings(ClientId(userId)).flatMap { bookings =>
Ok(bookings.asJson)
}
}
val routes: HttpRoutes[F] =
placeBooking <+> clientBookings <+> cancelBooking
}
Example 44
| Source File: ProducerOf.scala From skafka with MIT License | 5 votes |
|
package com.evolutiongaming.skafka.producer
import cats.effect.{Bracket, ContextShift, Effect, Resource}
import cats.{Defer, Monad, ~>}
import com.evolutiongaming.smetrics.MeasureDuration
import scala.concurrent.ExecutionContext
trait ProducerOf[F[_]] {
def apply(config: ProducerConfig): Resource[F, Producer[F]]
}
object ProducerOf {
def apply[F[_] : Effect : ContextShift : MeasureDuration](
executorBlocking: ExecutionContext,
metrics: Option[ProducerMetrics[F]] = None
): ProducerOf[F] = new ProducerOf[F] {
def apply(config: ProducerConfig) = {
for {
producer <- Producer.of[F](config, executorBlocking)
} yield {
metrics.fold(producer)(producer.withMetrics[Throwable])
}
}
}
implicit class ProducerOfOps[F[_]](val self: ProducerOf[F]) extends AnyVal {
def mapK[G[_] : Monad : Defer](
fg: F ~> G,
gf: G ~> F)(implicit
B: Bracket[F, Throwable]
): ProducerOf[G] = new ProducerOf[G] {
def apply(config: ProducerConfig) = {
for {
a <- self(config).mapK(fg)
} yield {
a.mapK(fg, gf)
}
}
}
}
}
Example 45
| Source File: CatsEffect.scala From cats-effect-testing with Apache License 2.0 | 5 votes |
|
package cats.effect.testing.specs2
import cats.effect.{Effect, Resource, Sync}
import cats.effect.syntax.effect._
import org.specs2.execute.{AsResult, Failure, Result}
import scala.concurrent.duration._
import scala.language.higherKinds
trait CatsEffect {
protected val Timeout: Duration = 10.seconds
implicit def effectAsResult[F[_]: Effect, R](implicit R: AsResult[R]): AsResult[F[R]] = new AsResult[F[R]] {
def asResult(t: => F[R]): Result =
t.toIO.unsafeRunTimed(Timeout)
.map(R.asResult(_))
.getOrElse(Failure(s"expectation timed out after $Timeout"))
}
implicit def resourceAsResult[F[_]: Effect, R](implicit R: AsResult[R]): AsResult[Resource[F,R]] = new AsResult[Resource[F,R]]{
def asResult(t: => Resource[F, R]): Result =
t.use(r => Sync[F].delay(R.asResult(r)))
.toIO
.unsafeRunTimed(Timeout)
.getOrElse(Failure(s"expectation timed out after $Timeout"))
}
}
Example 46
| Source File: EffectCheckerAsserting.scala From cats-effect-testing with Apache License 2.0 | 5 votes |
|
package cats.effect.testing.scalatest.scalacheck
import cats.effect.Effect
import org.scalactic.source
import org.scalatest.exceptions._
import org.scalatestplus.scalacheck.CheckerAsserting
class EffectCheckerAsserting[F[_], A](implicit F: Effect[F])
extends CheckerAsserting.CheckerAssertingImpl[F[A]] {
override type Result = F[Unit]
override def succeed(result: F[A]): (Boolean, Option[Throwable]) =
F.toIO(result)
.attempt
.unsafeRunSync()
.fold(e => (false, Some(e)), _ => (true, None))
override def indicateSuccess(message: => String): Result = F.unit
override def indicateFailure(
messageFun: StackDepthException => String,
undecoratedMessage: => String,
scalaCheckArgs: List[Any],
scalaCheckLabels: List[String],
optionalCause: Option[Throwable],
pos: source.Position
): Result = {
val error = new GeneratorDrivenPropertyCheckFailedException(
messageFun,
optionalCause,
pos,
None,
undecoratedMessage,
scalaCheckArgs,
None,
scalaCheckLabels
)
F.raiseError(error)
}
}
Example 47
| Source File: TransactionRoute.scala From aecor with MIT License | 5 votes |
|
package aecor.example.transaction
import java.util.UUID
import aecor.example.account
import aecor.example.account.AccountId
import aecor.example.common.Amount
import cats.effect.{Effect, Sync}
import cats.implicits._
import org.http4s.HttpRoutes
import org.http4s.circe.CirceEntityDecoder
import org.http4s.dsl.Http4sDsl
import io.circe.generic.auto._
trait TransactionService[F[_]] {
def authorizePayment(transactionId: TransactionId,
from: From[AccountId],
to: To[AccountId],
amount: Amount): F[TransactionRoute.ApiResult]
}
object TransactionRoute {
sealed trait ApiResult
object ApiResult {
case object Authorized extends ApiResult
case class Declined(reason: String) extends ApiResult
}
final case class CreateTransactionRequest(from: From[AccountId],
to: To[AccountId],
amount: Amount)
object TransactionIdVar {
def unapply(arg: String): Option[TransactionId] = TransactionId(arg).some
}
private final class Builder[F[_]: Sync](service: TransactionService[F]) extends Http4sDsl[F] with CirceEntityDecoder {
def routes: HttpRoutes[F] = HttpRoutes.of[F] {
case req @ PUT -> Root / "transactions" / TransactionIdVar(transactionId) =>
for {
body <- req.as[CreateTransactionRequest]
CreateTransactionRequest(from, to, amount) = body
resp <- service.authorizePayment(transactionId, from, to, amount).flatMap {
case ApiResult.Authorized =>
Ok("Authorized")
case ApiResult.Declined(reason) =>
BadRequest(s"Declined: $reason")
}
} yield resp
case POST -> Root / "test" =>
service
.authorizePayment(
TransactionId(UUID.randomUUID.toString),
From(account.EventsourcedAlgebra.rootAccountId),
To(AccountId("foo")),
Amount(1)
)
.flatMap {
case ApiResult.Authorized =>
Ok("Authorized")
case ApiResult.Declined(reason) =>
BadRequest(s"Declined: $reason")
}
}
}
def apply[F[_]: Effect](api: TransactionService[F]): HttpRoutes[F] =
new Builder(api).routes
}
Example 48
| Source File: deployment.scala From aecor with MIT License | 5 votes |
|
package aecor.example.transaction
import java.util.UUID
import aecor.example.common.Timestamp
import aecor.example.transaction.transaction.Transactions
import aecor.runtime.Eventsourced
import aecor.runtime.akkapersistence.AkkaPersistenceRuntime
import aecor.util.Clock
import cats.implicits._
import cats.effect.{ ContextShift, Effect }
import scodec.codecs.implicits._
object deployment {
def deploy[F[_]: Effect: ContextShift](runtime: AkkaPersistenceRuntime[UUID],
clock: Clock[F]): F[Transactions[F]] =
runtime
.deploy(
"Transaction",
EventsourcedAlgebra.behavior[F].enrich(clock.instant.map(Timestamp(_))),
EventsourcedAlgebra.tagging
)
.map(Eventsourced.Entities.rejectable(_))
}
Example 49
| Source File: deployment.scala From aecor with MIT License | 5 votes |
|
package aecor.example.account
import java.util.UUID
import aecor.example.common.Timestamp
import aecor.runtime.Eventsourced
import aecor.runtime.akkapersistence.AkkaPersistenceRuntime
import aecor.util.Clock
import cats.effect.{ ContextShift, Effect }
import cats.implicits._
object deployment {
def deploy[F[_]: Effect: ContextShift](runtime: AkkaPersistenceRuntime[UUID],
clock: Clock[F]): F[Accounts[F]] =
runtime
.deploy(
"Account",
EventsourcedAlgebra.behavior[F].enrich(clock.instant.map(Timestamp(_))),
EventsourcedAlgebra.tagging
)
.map(Eventsourced.Entities.rejectable(_))
}
Example 50
| Source File: RebalanceEvents.scala From aecor with MIT License | 5 votes |
|
package aecor.kafkadistributedprocessing.internal
import java.util
import aecor.data.Committable
import cats.effect.concurrent.Deferred
import cats.effect.implicits._
import cats.effect.{ ConcurrentEffect, Effect }
import cats.implicits._
import fs2.Stream
import fs2.concurrent.Queue
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener
import org.apache.kafka.common
import org.apache.kafka.common.TopicPartition
import scala.collection.JavaConverters._
private[kafkadistributedprocessing] object RebalanceEvents {
final class UsePartiallyApplied[F[_]] {
def subscribe[A](
f: ConsumerRebalanceListener => F[Unit]
)(implicit F: ConcurrentEffect[F]): F[Stream[F, Committable[F, RebalanceEvent]]] =
for {
queue <- Queue.unbounded[F, Committable[F, RebalanceEvent]]
listener = new Listener[F](
event =>
Deferred[F, Unit]
.flatMap { completion =>
queue.enqueue1(Committable(completion.complete(()), event)) >> completion.get
}
)
_ <- f(listener)
} yield queue.dequeue
}
def apply[F[_]]: UsePartiallyApplied[F] = new UsePartiallyApplied[F]
sealed abstract class RebalanceEvent
object RebalanceEvent {
final case class PartitionsRevoked(partitions: Set[TopicPartition]) extends RebalanceEvent
final case class PartitionsAssigned(partitions: Set[TopicPartition]) extends RebalanceEvent
}
private final class Listener[F[_]: Effect](processEvent: RebalanceEvent => F[Unit])
extends ConsumerRebalanceListener {
override def onPartitionsRevoked(partitions: util.Collection[common.TopicPartition]): Unit =
processEvent(RebalanceEvent.PartitionsRevoked(partitions.asScala.toSet)).toIO
.unsafeRunSync()
override def onPartitionsAssigned(partitions: util.Collection[common.TopicPartition]): Unit =
processEvent(RebalanceEvent.PartitionsAssigned(partitions.asScala.toSet)).toIO
.unsafeRunSync()
}
}
Example 51
| Source File: DefaultScheduleEventJournal.scala From aecor with MIT License | 5 votes |
|
package aecor.schedule.process
import java.util.UUID
import aecor.data.{ Committable, ConsumerId, EntityEvent, EventTag }
import aecor.runtime.akkapersistence.readside.CommittableEventJournalQuery
import aecor.schedule.{ ScheduleBucketId, ScheduleEvent }
import aecor.util.effect._
import akka.stream.Materializer
import akka.stream.scaladsl.{ Keep, Sink }
import cats.effect.Effect
import cats.implicits._
object DefaultScheduleEventJournal {
def apply[F[_]: Effect](
consumerId: ConsumerId,
parallelism: Int,
aggregateJournal: CommittableEventJournalQuery[F, UUID, ScheduleBucketId, ScheduleEvent],
eventTag: EventTag
)(implicit materializer: Materializer): DefaultScheduleEventJournal[F] =
new DefaultScheduleEventJournal(consumerId, parallelism, aggregateJournal, eventTag)
}
final class DefaultScheduleEventJournal[F[_]: Effect](
consumerId: ConsumerId,
parallelism: Int,
aggregateJournal: CommittableEventJournalQuery[F, UUID, ScheduleBucketId, ScheduleEvent],
eventTag: EventTag
)(implicit materializer: Materializer)
extends ScheduleEventJournal[F] {
override def processNewEvents(
f: EntityEvent[ScheduleBucketId, ScheduleEvent] => F[Unit]
): F[Unit] =
Effect[F].fromFuture {
aggregateJournal
.currentEventsByTag(eventTag, consumerId)
.mapAsync(parallelism)(_.map(_.event).traverse(f).unsafeToFuture())
.fold(Committable.unit[F])(Keep.right)
.mapAsync(1)(_.commit.unsafeToFuture())
.runWith(Sink.ignore)
}.void
}
Example 52
| Source File: PeriodicProcessRuntime.scala From aecor with MIT License | 5 votes |
|
package aecor.schedule.process
import aecor.distributedprocessing.{ AkkaStreamProcess, DistributedProcessing }
import aecor.util.effect._
import akka.NotUsed
import akka.actor.ActorSystem
import akka.stream.Materializer
import akka.stream.scaladsl.Source
import cats.effect.{ ContextShift, Effect }
import scala.collection.immutable._
import scala.concurrent.duration.{ FiniteDuration, _ }
object PeriodicProcessRuntime {
def apply[F[_]: Effect: ContextShift](
name: String,
tickInterval: FiniteDuration,
processCycle: F[Unit]
)(implicit materializer: Materializer): PeriodicProcessRuntime[F] =
new PeriodicProcessRuntime[F](name, tickInterval, processCycle)
}
class PeriodicProcessRuntime[F[_]: Effect: ContextShift](
name: String,
tickInterval: FiniteDuration,
processCycle: F[Unit]
)(implicit materializer: Materializer) {
private def source =
Source
.tick(0.seconds, tickInterval, processCycle)
.mapAsync(1)(_.unsafeToFuture())
.mapMaterializedValue(_ => NotUsed)
def run(system: ActorSystem): F[DistributedProcessing.KillSwitch[F]] =
DistributedProcessing(system)
.start[F](s"$name-Process", List(AkkaStreamProcess[F](source)))
}
Example 53
| Source File: DefaultSchedule.scala From aecor with MIT License | 5 votes |
|
package aecor.schedule
import java.time.LocalDateTime
import java.util.UUID
import aecor.data._
import aecor.runtime.akkapersistence.readside.{ CommittableEventJournalQuery, JournalEntry }
import aecor.util.Clock
import akka.NotUsed
import akka.stream.scaladsl.Source
import cats.effect.Effect
import cats.implicits._
import aecor.util.effect._
import scala.concurrent.duration.FiniteDuration
private[schedule] class DefaultSchedule[F[_]: Effect](
clock: Clock[F],
buckets: ScheduleBucketId => ScheduleBucket[F],
bucketLength: FiniteDuration,
aggregateJournal: CommittableEventJournalQuery[F, UUID, ScheduleBucketId, ScheduleEvent],
eventTag: EventTag
) extends Schedule[F] {
override def addScheduleEntry(scheduleName: String,
entryId: String,
correlationId: String,
dueDate: LocalDateTime): F[Unit] =
for {
zone <- clock.zone
scheduleBucket = dueDate.atZone(zone).toEpochSecond / bucketLength.toSeconds
_ <- buckets(ScheduleBucketId(scheduleName, scheduleBucket.toString))
.addScheduleEntry(entryId, correlationId, dueDate)
} yield ()
override def committableScheduleEvents(
scheduleName: String,
consumerId: ConsumerId
): Source[Committable[F, JournalEntry[UUID, ScheduleBucketId, ScheduleEvent]], NotUsed] =
aggregateJournal
.eventsByTag(eventTag, ConsumerId(scheduleName + consumerId.value))
.flatMapConcat {
case m if m.value.event.entityKey.scheduleName == scheduleName => Source.single(m)
case other =>
Source
.fromFuture(other.commit.unsafeToFuture())
.flatMapConcat(
_ => Source.empty[Committable[F, JournalEntry[UUID, ScheduleBucketId, ScheduleEvent]]]
)
}
}
Example 54
| Source File: DistributedProcessingWorker.scala From aecor with MIT License | 5 votes |
|
package aecor.distributedprocessing
import aecor.distributedprocessing.DistributedProcessing._
import aecor.distributedprocessing.DistributedProcessingWorker.KeepRunning
import aecor.distributedprocessing.serialization.Message
import cats.effect.syntax.effect._
import akka.actor.{ Actor, ActorLogging, Props, Status }
import akka.pattern._
import cats.effect.Effect
import cats.implicits._
private[aecor] object DistributedProcessingWorker {
def props[F[_]: Effect](processWithId: Int => Process[F], processName: String): Props =
Props(new DistributedProcessingWorker[F](processWithId, processName))
final case class KeepRunning(workerId: Int) extends Message
}
private[aecor] final class DistributedProcessingWorker[F[_]: Effect](
processFor: Int => Process[F],
processName: String
) extends Actor
with ActorLogging {
import context.dispatcher
case class ProcessStarted(process: RunningProcess[F])
case object ProcessTerminated
var killSwitch: Option[F[Unit]] = None
override def postStop: Unit =
killSwitch.foreach(_.toIO.unsafeRunSync())
def receive: Receive = {
case KeepRunning(workerId) =>
log.info("[{}] Starting process {}", workerId, processName)
processFor(workerId).run
.map(ProcessStarted)
.toIO
.unsafeToFuture() pipeTo self
context.become {
case ProcessStarted(RunningProcess(watchTermination, terminate)) =>
log.info("[{}] Process started {}", workerId, processName)
killSwitch = Some(terminate)
watchTermination.toIO.map(_ => ProcessTerminated).unsafeToFuture() pipeTo self
context.become {
case Status.Failure(e) =>
log.error(e, "Process failed {}", processName)
throw e
case ProcessTerminated =>
log.error("Process terminated {}", processName)
throw new IllegalStateException(s"Process terminated $processName")
}
case Status.Failure(e) =>
log.error(e, "Process failed to start {}", processName)
throw e
case KeepRunning(_) => ()
}
}
}
Example 55
| Source File: DistributedProcessing.scala From aecor with MIT License | 5 votes |
|
package aecor.distributedprocessing
import java.net.URLEncoder
import java.nio.charset.StandardCharsets
import aecor.distributedprocessing.DistributedProcessing.{ KillSwitch, Process }
import aecor.distributedprocessing.DistributedProcessingWorker.KeepRunning
import aecor.util.effect._
import akka.actor.ActorSystem
import akka.cluster.sharding.{ ClusterSharding, ClusterShardingSettings }
import akka.pattern.{ BackoffOpts, BackoffSupervisor, ask }
import akka.util.Timeout
import cats.effect.Effect
import cats.implicits._
import scala.concurrent.duration.{ FiniteDuration, _ }
final class DistributedProcessing private (system: ActorSystem) {
def start[F[_]: Effect](name: String,
processes: List[Process[F]],
settings: DistributedProcessingSettings =
DistributedProcessingSettings.default(system)): F[KillSwitch[F]] =
Effect[F].delay {
val opts = BackoffOpts
.onFailure(
DistributedProcessingWorker.props(processes, name),
"worker",
settings.minBackoff,
settings.maxBackoff,
settings.randomFactor
)
val props = BackoffSupervisor.props(opts)
val region = ClusterSharding(system).start(
typeName = name,
entityProps = props,
settings = settings.clusterShardingSettings,
extractEntityId = {
case c @ KeepRunning(workerId) => (workerId.toString, c)
},
extractShardId = {
case KeepRunning(workerId) => (workerId % settings.numberOfShards).toString
case other => throw new IllegalArgumentException(s"Unexpected message [$other]")
}
)
val regionSupervisor = system.actorOf(
DistributedProcessingSupervisor
.props(processes.size, region, settings.heartbeatInterval),
"DistributedProcessingSupervisor-" + URLEncoder
.encode(name, StandardCharsets.UTF_8.name())
)
implicit val timeout = Timeout(settings.shutdownTimeout)
KillSwitch {
Effect[F].fromFuture {
regionSupervisor ? DistributedProcessingSupervisor.GracefulShutdown
}.void
}
}
}
object DistributedProcessing {
def apply(system: ActorSystem): DistributedProcessing = new DistributedProcessing(system)
final case class KillSwitch[F[_]](shutdown: F[Unit]) extends AnyVal
final case class RunningProcess[F[_]](watchTermination: F[Unit], shutdown: F[Unit])
final case class Process[F[_]](run: F[RunningProcess[F]]) extends AnyVal
}
final case class DistributedProcessingSettings(minBackoff: FiniteDuration,
maxBackoff: FiniteDuration,
randomFactor: Double,
shutdownTimeout: FiniteDuration,
numberOfShards: Int,
heartbeatInterval: FiniteDuration,
clusterShardingSettings: ClusterShardingSettings)
object DistributedProcessingSettings {
def default(clusterShardingSettings: ClusterShardingSettings): DistributedProcessingSettings =
DistributedProcessingSettings(
minBackoff = 3.seconds,
maxBackoff = 10.seconds,
randomFactor = 0.2,
shutdownTimeout = 10.seconds,
numberOfShards = 100,
heartbeatInterval = 2.seconds,
clusterShardingSettings = clusterShardingSettings
)
def default(system: ActorSystem): DistributedProcessingSettings =
default(ClusterShardingSettings(system))
}
Example 56
| Source File: GenericAkkaRuntime.scala From aecor with MIT License | 5 votes |
|
package aecor.runtime.akkageneric
import aecor.encoding.WireProtocol.Encoded
import aecor.encoding.syntax._
import aecor.encoding.{ KeyDecoder, KeyEncoder, WireProtocol }
import aecor.runtime.akkageneric.GenericAkkaRuntime.KeyedCommand
import aecor.runtime.akkageneric.GenericAkkaRuntimeActor.CommandResult
import aecor.runtime.akkageneric.serialization.Message
import aecor.util.effect._
import akka.actor.ActorSystem
import akka.cluster.sharding.{ ClusterSharding, ShardRegion }
import akka.pattern._
import akka.util.Timeout
import cats.effect.Effect
import cats.implicits._
import cats.tagless.FunctorK
import cats.tagless.syntax.functorK._
import cats.~>
import scodec.bits.BitVector
object GenericAkkaRuntime {
def apply(system: ActorSystem): GenericAkkaRuntime =
new GenericAkkaRuntime(system)
private[akkageneric] final case class KeyedCommand(key: String, bytes: BitVector) extends Message
}
final class GenericAkkaRuntime private (system: ActorSystem) {
def runBehavior[K: KeyEncoder: KeyDecoder, M[_[_]]: FunctorK, F[_]](
typeName: String,
createBehavior: K => F[M[F]],
settings: GenericAkkaRuntimeSettings = GenericAkkaRuntimeSettings.default(system)
)(implicit M: WireProtocol[M], F: Effect[F]): F[K => M[F]] =
F.delay {
val props = GenericAkkaRuntimeActor.props[K, M, F](createBehavior, settings.idleTimeout)
val extractEntityId: ShardRegion.ExtractEntityId = {
case KeyedCommand(entityId, c) =>
(entityId, GenericAkkaRuntimeActor.Command(c))
}
val numberOfShards = settings.numberOfShards
val extractShardId: ShardRegion.ExtractShardId = {
case KeyedCommand(key, _) =>
String.valueOf(scala.math.abs(key.hashCode) % numberOfShards)
case other => throw new IllegalArgumentException(s"Unexpected message [$other]")
}
val shardRegion = ClusterSharding(system).start(
typeName = typeName,
entityProps = props,
settings = settings.clusterShardingSettings,
extractEntityId = extractEntityId,
extractShardId = extractShardId
)
val keyEncoder = KeyEncoder[K]
key =>
M.encoder.mapK(new (Encoded ~> F) {
implicit val askTimeout: Timeout = Timeout(settings.askTimeout)
override def apply[A](fa: Encoded[A]): F[A] = F.suspend {
val (bytes, decoder) = fa
F.fromFuture {
shardRegion ? KeyedCommand(keyEncoder(key), bytes)
}
.flatMap {
case result: CommandResult =>
decoder.decodeValue(result.bytes).lift[F]
case other =>
F.raiseError(
new IllegalArgumentException(s"Unexpected response [$other] from shard region")
)
}
}
})
}
}
Example 57
| Source File: effect.scala From aecor with MIT License | 5 votes |
|
package aecor.util
import cats.effect.{ Async, Effect, IO }
import scala.concurrent.{ Future, Promise }
object effect {
implicit final class AecorEffectOps[F[_], A](val self: F[A]) extends AnyVal {
@inline final def unsafeToFuture()(implicit F: Effect[F]): Future[A] = {
val p = Promise[A]
F.runAsync(self) {
case Right(a) => IO { p.success(a); () }
case Left(e) => IO { p.failure(e); () }
}
.unsafeRunSync()
p.future
}
}
implicit final class AecorLiftIOOps[F[_]](val self: Async[F]) extends AnyVal {
def fromFuture[A](future: => Future[A]): F[A] =
IO.fromFuture(IO(future))(IO.contextShift(scala.concurrent.ExecutionContext.global)).to(self)
}
}
Example 58
| Source File: JournalQuery.scala From aecor with MIT License | 5 votes |
|
package aecor.runtime.akkapersistence.readside
import aecor.Has
import aecor.data.{ EntityEvent, EventTag, TagConsumer }
import aecor.runtime.KeyValueStore
import akka.NotUsed
import akka.stream.scaladsl.Source
import cats.effect.Effect
final case class JournalEntry[O, K, A](offset: O, event: EntityEvent[K, A]) {
def map[B](f: A => B): JournalEntry[O, K, B] = copy(event = event.map(f))
}
object JournalEntry {
implicit def aecorHasInstanceForEvent[X, O, K, A](
implicit A: Has[EntityEvent[K, A], X]
): Has[JournalEntry[O, K, A], X] =
A.contramap(_.event)
implicit def aecorHasInstanceForOffset[X, O, K, A](
implicit A: Has[O, X]
): Has[JournalEntry[O, K, A], X] = A.contramap(_.offset)
}
trait JournalQuery[O, K, E] {
def eventsByTag(tag: EventTag,
offset: Option[O]): Source[JournalEntry[O, K, E], NotUsed]
def currentEventsByTag(tag: EventTag,
offset: Option[O]): Source[JournalEntry[O, K, E], NotUsed]
def committable[F[_]: Effect](
offsetStore: KeyValueStore[F, TagConsumer, O]
): CommittableEventJournalQuery[F, O, K, E] =
new CommittableEventJournalQuery(this, offsetStore)
}
Example 59
| Source File: CommittableEventJournalQuery.scala From aecor with MIT License | 5 votes |
|
package aecor.runtime.akkapersistence.readside
import aecor.data.{ Committable, ConsumerId, EventTag, TagConsumer }
import aecor.runtime.KeyValueStore
import aecor.util.effect._
import akka.NotUsed
import akka.stream.scaladsl.Source
import cats.effect.Effect
final class CommittableEventJournalQuery[F[_]: Effect, O, K, E] private[akkapersistence] (
underlying: JournalQuery[O, K, E],
offsetStore: KeyValueStore[F, TagConsumer, O]
) {
private def mkCommittableSource(tag: EventTag,
consumerId: ConsumerId,
inner: Option[O] => Source[JournalEntry[O, K, E], NotUsed]) = {
val tagConsumerId = TagConsumer(tag, consumerId)
Source
.single(NotUsed)
.mapAsync(1) { _ =>
offsetStore.getValue(tagConsumerId).unsafeToFuture()
}
.flatMapConcat(inner)
.map(x => Committable(offsetStore.setValue(tagConsumerId, x.offset), x))
}
def eventsByTag(tag: EventTag,
consumerId: ConsumerId): Source[Committable[F, JournalEntry[O, K, E]], NotUsed] =
mkCommittableSource(tag, consumerId, underlying.eventsByTag(tag, _))
def currentEventsByTag(
tag: EventTag,
consumerId: ConsumerId
): Source[Committable[F, JournalEntry[O, K, E]], NotUsed] =
mkCommittableSource(tag, consumerId, underlying.currentEventsByTag(tag, _))
}
private[akkapersistence] object CommittableEventJournalQuery {
def apply[F[_]: Effect, Offset, K, E](
underlying: JournalQuery[Offset, K, E],
offsetStore: KeyValueStore[F, TagConsumer, Offset]
): CommittableEventJournalQuery[F, Offset, K, E] =
new CommittableEventJournalQuery(underlying, offsetStore)
}
Example 60
| Source File: CassandraOffsetStore.scala From aecor with MIT License | 5 votes |
|
package aecor.runtime.akkapersistence.readside
import java.util.UUID
import aecor.data.TagConsumer
import aecor.runtime.KeyValueStore
import aecor.util.effect._
import akka.persistence.cassandra.Session.Init
import akka.persistence.cassandra.session.scaladsl.CassandraSession
import cats.Functor
import cats.data.Kleisli
import cats.effect.Effect
import cats.implicits._
object CassandraOffsetStore {
final case class Queries(keyspace: String, tableName: String = "consumer_offset") {
def createTableQuery: String =
s"CREATE TABLE IF NOT EXISTS $keyspace.$tableName (consumer_id text, tag text, offset uuid, PRIMARY KEY ((consumer_id, tag)))"
def updateOffsetQuery: String =
s"UPDATE $keyspace.$tableName SET offset = ? where consumer_id = ? AND tag = ?"
def deleteOffsetQuery: String =
s"DELETE FROM $keyspace.$tableName where consumer_id = ? AND tag = ?"
def selectOffsetQuery: String =
s"SELECT offset FROM $keyspace.$tableName WHERE consumer_id = ? AND tag = ?"
}
def apply[F[_]]: Builder[F] = builderInstance.asInstanceOf[Builder[F]]
private val builderInstance = new Builder[Any]()
final class Builder[F[_]] private[CassandraOffsetStore] () {
def createTable(config: Queries)(implicit F: Functor[F]): Init[F] =
Kleisli(_.execute(config.createTableQuery).void)
def apply(session: CassandraSession, config: CassandraOffsetStore.Queries)(
implicit F: Effect[F]
): CassandraOffsetStore[F] =
new CassandraOffsetStore(session, config)
}
}
class CassandraOffsetStore[F[_]] private[akkapersistence] (
session: CassandraSession,
config: CassandraOffsetStore.Queries
)(implicit F: Effect[F])
extends KeyValueStore[F, TagConsumer, UUID] {
private val selectOffsetStatement =
session.prepare(config.selectOffsetQuery)
private val updateOffsetStatement =
session.prepare(config.updateOffsetQuery)
private val deleteOffsetStatement =
session.prepare(config.deleteOffsetQuery)
override def setValue(key: TagConsumer, value: UUID): F[Unit] =
F.fromFuture {
updateOffsetStatement
}
.map { stmt =>
stmt
.bind()
.setUUID("offset", value)
.setString("tag", key.tag.value)
.setString("consumer_id", key.consumerId.value)
}
.flatMap(x => F.fromFuture(session.executeWrite(x)))
.void
override def getValue(key: TagConsumer): F[Option[UUID]] =
F.fromFuture {
selectOffsetStatement
}
.map(_.bind(key.consumerId.value, key.tag.value))
.flatMap(x => F.fromFuture(session.selectOne(x)))
.map(_.map(_.getUUID("offset")))
override def deleteValue(key: TagConsumer): F[Unit] =
F.fromFuture {
deleteOffsetStatement
}
.map(_.bind(key.consumerId.value, key.tag.value))
.flatMap(x => F.fromFuture(session.executeWrite(x)))
.void
}
Example 61
| Source File: AkkaPersistenceRuntime.scala From aecor with MIT License | 5 votes |
|
package aecor.runtime.akkapersistence
import aecor.data.{ EventsourcedBehavior, Tagging }
import aecor.encoding.WireProtocol.Encoded
import aecor.encoding.syntax._
import aecor.encoding.{ KeyDecoder, KeyEncoder, WireProtocol }
import aecor.runtime.akkapersistence.AkkaPersistenceRuntime._
import aecor.runtime.akkapersistence.AkkaPersistenceRuntimeActor.CommandResult
import aecor.runtime.akkapersistence.readside.{ AkkaPersistenceEventJournalQuery, JournalQuery }
import aecor.runtime.akkapersistence.serialization.{ Message, PersistentDecoder, PersistentEncoder }
import aecor.util.effect._
import akka.actor.ActorSystem
import akka.cluster.sharding.{ ClusterSharding, ShardRegion }
import akka.pattern.ask
import akka.util.Timeout
import cats.effect.Effect
import cats.implicits._
import cats.tagless.FunctorK
import cats.tagless.syntax.functorK._
import cats.~>
import scodec.bits.BitVector
object AkkaPersistenceRuntime {
def apply[O](system: ActorSystem, journalAdapter: JournalAdapter[O]): AkkaPersistenceRuntime[O] =
new AkkaPersistenceRuntime(system, journalAdapter)
private[akkapersistence] final case class EntityCommand(entityKey: String,
commandBytes: BitVector)
extends Message
}
class AkkaPersistenceRuntime[O] private[akkapersistence] (system: ActorSystem,
journalAdapter: JournalAdapter[O]) {
def deploy[M[_[_]]: FunctorK, F[_], State, Event: PersistentEncoder: PersistentDecoder, K: KeyEncoder: KeyDecoder](
typeName: String,
behavior: EventsourcedBehavior[M, F, State, Event],
tagging: Tagging[K],
snapshotPolicy: SnapshotPolicy[State] = SnapshotPolicy.never,
settings: AkkaPersistenceRuntimeSettings = AkkaPersistenceRuntimeSettings.default(system)
)(implicit M: WireProtocol[M], F: Effect[F]): F[K => M[F]] =
F.delay {
val props =
AkkaPersistenceRuntimeActor.props(
typeName,
behavior,
snapshotPolicy,
tagging,
settings.idleTimeout,
journalAdapter.writeJournalId,
snapshotPolicy.pluginId
)
val extractEntityId: ShardRegion.ExtractEntityId = {
case EntityCommand(entityId, bytes) =>
(entityId, AkkaPersistenceRuntimeActor.HandleCommand(bytes))
}
val numberOfShards = settings.numberOfShards
val extractShardId: ShardRegion.ExtractShardId = {
case EntityCommand(entityId, _) =>
(scala.math.abs(entityId.hashCode) % numberOfShards).toString
case other => throw new IllegalArgumentException(s"Unexpected message [$other]")
}
val shardRegion = ClusterSharding(system).start(
typeName = typeName,
entityProps = props,
settings = settings.clusterShardingSettings,
extractEntityId = extractEntityId,
extractShardId = extractShardId
)
val keyEncoder = KeyEncoder[K]
key =>
M.encoder.mapK(new (Encoded ~> F) {
implicit val askTimeout: Timeout = Timeout(settings.askTimeout)
override def apply[A](fa: Encoded[A]): F[A] = F.suspend {
val (bytes, decoder) = fa
F.fromFuture {
shardRegion ? EntityCommand(keyEncoder(key), bytes)
}
.flatMap {
case CommandResult(resultBytes) =>
decoder.decodeValue(resultBytes).lift[F]
case other =>
F.raiseError(
new IllegalArgumentException(s"Unexpected response [$other] from shard region")
)
}
}
})
}
def journal[K: KeyDecoder, E: PersistentDecoder]: JournalQuery[O, K, E] =
AkkaPersistenceEventJournalQuery[O, K, E](journalAdapter)
}
Example 62
| Source File: DefaultJournalCassandraSession.scala From aecor with MIT License | 5 votes |
|
package akka.persistence.cassandra
import akka.Done
import akka.actor.{ ActorSystem, ExtendedActorSystem }
import akka.event.Logging
import akka.persistence.cassandra.Session.Init
import akka.persistence.cassandra.session.CassandraSessionSettings
import akka.persistence.cassandra.session.scaladsl.CassandraSession
import cats.effect.{ ContextShift, Effect }
import cats.implicits._
object DefaultJournalCassandraSession {
def apply[F[_]: ContextShift](
system: ActorSystem,
metricsCategory: String,
init: Init[F],
sessionProvider: Option[SessionProvider] = None
)(implicit F: Effect[F]): F[CassandraSession] = F.delay {
val log = Logging(system, classOf[CassandraSession])
val provider = sessionProvider.getOrElse(
SessionProvider(
system.asInstanceOf[ExtendedActorSystem],
system.settings.config.getConfig("cassandra-journal")
)
)
val settings = CassandraSessionSettings(system.settings.config.getConfig("cassandra-journal"))
new CassandraSession(system, provider, settings, system.dispatcher, log, metricsCategory, { x =>
F.toIO(init(Session[F](x)).as(Done)).unsafeToFuture()
})
}
}
Example 63
| Source File: EffectStep.scala From cornichon with Apache License 2.0 | 5 votes |
|
package com.github.agourlay.cornichon.steps.cats
import cats.data.{ EitherT, NonEmptyList }
import cats.effect.Effect
import cats.syntax.either._
import com.github.agourlay.cornichon.core.ScenarioRunner.{ errorsToFailureStep, successLog }
import com.github.agourlay.cornichon.core._
import monix.eval.Task
import monix.execution.Scheduler
import scala.concurrent.duration.Duration
case class EffectStep[F[_]: Effect](title: String, effect: ScenarioContext => F[Either[CornichonError, Session]], show: Boolean = true) extends SessionValueStep {
def setTitle(newTitle: String): Step = copy(title = newTitle)
override def runSessionValueStep(runState: RunState): Task[Either[NonEmptyList[CornichonError], Session]] =
Task.fromEffect(effect(runState.scenarioContext)).map(_.leftMap(NonEmptyList.one))
override def onError(errors: NonEmptyList[CornichonError], runState: RunState, executionTime: Duration): (LogInstruction, FailedStep) =
errorsToFailureStep(this, runState.depth, errors, Some(executionTime))
override def logOnSuccess(result: Session, runState: RunState, executionTime: Duration): LogInstruction =
successLog(title, runState.depth, show, executionTime)
}
object EffectStep {
def fromEitherT[F[_]: Effect](title: String, effect: ScenarioContext => EitherT[F, CornichonError, Session], show: Boolean = true): Step = {
val effectT: ScenarioContext => F[Either[CornichonError, Session]] = s => effect(s).value
EffectStep(title, effectT, show)
}
def fromSync(title: String, effect: ScenarioContext => Session, show: Boolean = true): Step = {
import Scheduler.Implicits.global
val effectF: ScenarioContext => Task[Either[CornichonError, Session]] = s => Task.now(effect(s).asRight)
EffectStep(title, effectF, show)
}
def fromSyncE(title: String, effect: ScenarioContext => Either[CornichonError, Session], show: Boolean = true): Step = {
import Scheduler.Implicits.global
val effectF: ScenarioContext => Task[Either[CornichonError, Session]] = s => Task.now(effect(s))
EffectStep(title, effectF, show)
}
def fromAsync[F[_]: Effect](title: String, effect: ScenarioContext => F[Session], show: Boolean = true): Step = {
import Scheduler.Implicits.global
val effectF: ScenarioContext => Task[Either[CornichonError, Session]] = s => Task.fromEffect(effect(s)).map(Right.apply)
EffectStep(title, effectF, show)
}
}
Example 64
| Source File: implicits.scala From tofu with Apache License 2.0 | 5 votes |
|
package tofu.doobie.instances
import cats.effect.{Effect, IO, SyncEffect}
import doobie.ConnectionIO
import tofu.lift.Lift
object implicits extends DoobieImplicits1
private[instances] trait DoobieImplicits1 extends DoobieImplicits2 {
@inline final implicit def liftToConnectionIOViaIOImplicit[F[_]: Lift[*[_], IO]]: LiftToConnectionIOViaIO[F] =
liftToConnectionIOViaIO
}
private[instances] trait DoobieImplicits2 extends DoobieImplicitsScalaVersionSpecific {
@inline final implicit def liftEffectToConnectionIOImplicit[F[_]: Effect]: LiftEffectToConnectionIO[F] =
liftEffectToConnectionIO
@inline final implicit def liftSyncEffectToConnectionIOImplicit[F[_]: SyncEffect]: LiftSyncEffectToConnectionIO[F] =
liftSyncEffectToConnectionIO
@inline final implicit def liftToConnectionRIOImplicit[F[_], R](implicit
L: Lift[F, ConnectionIO]
): LiftToConnectionRIO[F, R] = liftToConnectionRIO
}
Example 65
| Source File: DoobieInstances.scala From tofu with Apache License 2.0 | 5 votes |
|
package tofu.doobie.instances
import cats.data.ReaderT
import cats.effect.{Effect, IO, SyncEffect}
import cats.effect.syntax.effect._
import cats.effect.syntax.syncEffect._
import doobie.ConnectionIO
import doobie.free.connection.AsyncConnectionIO
import tofu.HasProvide
import tofu.doobie.ConnectionRIO
import tofu.lift.Lift
private[instances] trait DoobieInstances {
final def liftToConnectionIOViaIO[F[_]: Lift[*[_], IO]]: LiftToConnectionIOViaIO[F] = new LiftToConnectionIOViaIO
final def liftEffectToConnectionIO[F[_]: Effect]: LiftEffectToConnectionIO[F] = new LiftEffectToConnectionIO
final def liftSyncEffectToConnectionIO[F[_]: SyncEffect]: LiftSyncEffectToConnectionIO[F] =
new LiftSyncEffectToConnectionIO
final def liftToConnectionRIO[F[_], R](implicit L: Lift[F, ConnectionIO]): LiftToConnectionRIO[F, R] =
new LiftToConnectionRIO
final def liftProvideToConnectionRIO[F[_], G[_], R](implicit
HP: HasProvide[G, F, R],
L: Lift[F, ConnectionIO]
): LiftProvideToConnectionRIO[F, G, R] = new LiftProvideToConnectionRIO
}
final class LiftToConnectionIOViaIO[F[_]](implicit L: Lift[F, IO]) extends Lift[F, ConnectionIO] {
def lift[A](fa: F[A]): ConnectionIO[A] = AsyncConnectionIO.liftIO(L.lift(fa))
}
final class LiftEffectToConnectionIO[F[_]: Effect] extends Lift[F, ConnectionIO] {
def lift[A](fa: F[A]): ConnectionIO[A] = AsyncConnectionIO.liftIO(fa.toIO)
}
final class LiftSyncEffectToConnectionIO[F[_]: SyncEffect] extends Lift[F, ConnectionIO] {
def lift[A](fa: F[A]): ConnectionIO[A] = fa.runSync[ConnectionIO]
}
final class LiftToConnectionRIO[F[_], R](implicit L: Lift[F, ConnectionIO]) extends Lift[F, ConnectionRIO[R, *]] {
def lift[A](fa: F[A]): ConnectionRIO[R, A] = ReaderT.liftF(L.lift(fa))
}
final class LiftProvideToConnectionRIO[F[_], G[_], R](implicit HP: HasProvide[G, F, R], L: Lift[F, ConnectionIO])
extends Lift[G, ConnectionRIO[R, *]] {
def lift[A](fa: G[A]): ConnectionRIO[R, A] = ReaderT(ctx => L.lift(HP.runContext(fa)(ctx)))
}
Example 66
| Source File: unlift.scala From tofu with Apache License 2.0 | 5 votes |
|
package tofu.syntax
import cats.effect.{CancelToken, ConcurrentEffect, Effect, ExitCase, Fiber, IO, SyncIO}
import cats.{FlatMap, Functor, ~>}
import tofu.lift.Unlift
object unlift {
implicit final class UnliftEffectOps[F[_], G[_]](private val U: Unlift[F, G]) extends AnyVal {
def effect(implicit G: Functor[G], E: Effect[F]): G[Effect[G]] =
G.map(U.unlift) { unliftF =>
new EffectInstance[F, G] {
def toG: F ~> G = U.liftF
def toF: G ~> F = unliftF
implicit def F: Effect[F] = E
}
}
def effectWith[A](f: Effect[G] => G[A])(implicit G: FlatMap[G], E: Effect[F]): G[A] =
G.flatMap(U.unlift) { unliftF =>
val eff = new EffectInstance[F, G] {
def toG: F ~> G = U.liftF
def toF: G ~> F = unliftF
implicit def F: Effect[F] = E
}
f(eff)
}
def concurrentEffect(implicit G: Functor[G], CE: ConcurrentEffect[F]): G[ConcurrentEffect[G]] =
G.map(U.unlift) { unliftF =>
new ConcurrentEffectInstance[F, G] {
def toG: F ~> G = U.liftF
def toF: G ~> F = unliftF
implicit def F: ConcurrentEffect[F] = CE
}
}
def concurrentEffectWith[A](f: ConcurrentEffect[G] => G[A])(implicit G: FlatMap[G], CE: ConcurrentEffect[F]): G[A] =
G.flatMap(U.unlift) { unliftF =>
val ce = new ConcurrentEffectInstance[F, G] {
def toG: F ~> G = U.liftF
def toF: G ~> F = unliftF
implicit def F: ConcurrentEffect[F] = CE
}
f(ce)
}
}
private[unlift] trait EffectInstance[F[_], G[_]] extends Effect[G] {
def toG: F ~> G
def toF: G ~> F
implicit def F: Effect[F]
def pure[A](x: A): G[A] = toG(F.pure(x))
def flatMap[A, B](ga: G[A])(f: A => G[B]): G[B] = toG(F.flatMap(toF(ga))(a => toF(f(a))))
def tailRecM[A, B](a: A)(f: A => G[Either[A, B]]): G[B] = toG(F.tailRecM(a)(a => toF(f(a))))
def raiseError[A](e: Throwable): G[A] = toG(F.raiseError(e))
def handleErrorWith[A](ga: G[A])(f: Throwable => G[A]): G[A] = toG(F.handleErrorWith(toF(ga))(t => toF(f(t))))
def bracketCase[A, B](acquire: G[A])(use: A => G[B])(release: (A, ExitCase[Throwable]) => G[Unit]): G[B] =
toG(F.bracketCase(toF(acquire))(a => toF(use(a)))((a, e) => toF(release(a, e))))
def suspend[A](thunk: => G[A]): G[A] = toG(F.suspend(toF(thunk)))
def async[A](k: (Either[Throwable, A] => Unit) => Unit): G[A] = toG(F.async(k))
def asyncF[A](k: (Either[Throwable, A] => Unit) => G[Unit]): G[A] = toG(F.asyncF[A](cb => toF(k(cb))))
def runAsync[A](ga: G[A])(cb: Either[Throwable, A] => IO[Unit]): SyncIO[Unit] = F.runAsync(toF(ga))(cb)
}
private[unlift] trait ConcurrentEffectInstance[F[_], G[_]] extends EffectInstance[F, G] with ConcurrentEffect[G] {
implicit def F: ConcurrentEffect[F]
def start[A](ga: G[A]): G[Fiber[G, A]] = toG(F.map(F.start(toF(ga)))(_.mapK(toG)))
def racePair[A, B](ga: G[A], gb: G[B]): G[Either[(A, Fiber[G, B]), (Fiber[G, A], B)]] =
toG(F.map(F.racePair(toF(ga), toF(gb))) {
case Left((a, fb)) => Left((a, fb.mapK(toG)))
case Right((fa, b)) => Right((fa.mapK(toG), b))
})
def runCancelable[A](ga: G[A])(cb: Either[Throwable, A] => IO[Unit]): SyncIO[CancelToken[G]] =
F.runCancelable(toF(ga))(cb).map(toG(_))
}
}
Example 67
| Source File: UnliftSuite.scala From tofu with Apache License 2.0 | 5 votes |
|
package tofu
import cats.Monad
import cats.effect.{ConcurrentEffect, Effect}
import tofu.lift.Unlift
import tofu.syntax.unlift._
object UnliftSuite {
private def needsEffect[G[_]: Effect]: G[Unit] = Effect[G].unit
private def needsConcurrentEffect[G[_]: ConcurrentEffect]: G[Unit] = ConcurrentEffect[G].unit
def checkUnliftEffectSyntax[F[_]: Effect, G[_]: Monad](implicit U: Unlift[F, G]): Unit = {
Unlift[F, G].effect: G[Effect[G]]
Unlift[F, G].effectWith(implicit E => needsEffect[G])
()
}
def checkUnliftCESyntax[F[_]: ConcurrentEffect, G[_]: Monad](implicit U: Unlift[F, G]): Unit = {
Unlift[F, G].concurrentEffect: G[ConcurrentEffect[G]]
Unlift[F, G].concurrentEffectWith(implicit CE => needsConcurrentEffect[G])
()
}
}
Example 68
| Source File: TestSqsConsumerBuilder.scala From fs2-aws with MIT License | 5 votes |
|
package fs2.aws.testkit
import cats.effect.Effect
import com.amazon.sqs.javamessaging.SQSConnection
import fs2.aws.testkit.TestSqsConsumerBuilder.TestSQSConsumer
import fs2.aws.sqs.{ ConsumerBuilder, SQSConsumer }
import javax.jms.MessageListener
import org.mockito.scalatest.MockitoSugar
class TestSqsConsumerBuilder[F[_]: Effect] extends ConsumerBuilder[F] {
override def start: F[SQSConsumer] =
Effect[F].delay(new TestSQSConsumer)
}
object TestSqsConsumerBuilder extends MockitoSugar {
class TestSQSConsumer extends SQSConsumer {
override def callback: MessageListener = mock[MessageListener]
override def startConsumer(): Unit = ()
override def shutdown(): Unit = ()
override def connection: SQSConnection = mock[SQSConnection]
}
}
Example 69
| Source File: SQSConsumerBuilder.scala From fs2-aws with MIT License | 5 votes |
|
package fs2.aws.sqs
import cats.effect.Effect
import com.amazon.sqs.javamessaging.{ ProviderConfiguration, SQSConnection, SQSConnectionFactory }
import com.amazonaws.services.sqs.AmazonSQSClientBuilder
import eu.timepit.refined.auto._
import javax.jms.{ MessageListener, Session }
import scala.language.higherKinds
class SQSConsumerBuilder[F[_]](val sqsConfig: SqsConfig, val listener: MessageListener)(
implicit F: Effect[F]
) extends ConsumerBuilder[F] {
val start: F[SQSConsumer] = {
F.delay {
new SQSConsumer {
override val callback: MessageListener = listener
val connectionFactory = new SQSConnectionFactory(
new ProviderConfiguration(),
AmazonSQSClientBuilder.defaultClient()
)
override val connection: SQSConnection = connectionFactory.createConnection
override def startConsumer(): Unit = {
val session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE)
val cons = session.createConsumer(session.createQueue(sqsConfig.queueName))
cons.setMessageListener(callback)
connection.start()
}
override def shutdown(): Unit =
connection.stop()
}
}
}
}
object SQSConsumerBuilder {
def apply[F[_]](sqsConfig: SqsConfig, listener: MessageListener)(
implicit F: Effect[F]
): SQSConsumerBuilder[F] = new SQSConsumerBuilder[F](sqsConfig, listener)
}
Example 70
| Source File: S3Client.scala From fs2-aws with MIT License | 5 votes |
|
package fs2.aws.internal
import java.io.InputStream
import cats.effect.Effect
import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model._
import scala.collection.JavaConverters._
import scala.util.control.Exception
private[aws] object S3Client {
def apply[F[_]](s3: AmazonS3) = new S3ClientImpl[F](s3)
}
private[aws] class S3ClientImpl[F[_]](c: AmazonS3) extends S3Client[F] {
override def client: AmazonS3 = c
}
private[aws] trait S3Client[F[_]] {
def client: AmazonS3
def getObjectContentOrError(
getObjectRequest: GetObjectRequest
)(implicit F: Effect[F]): F[Either[Throwable, InputStream]] =
F.delay(Exception.nonFatalCatch either client.getObject(getObjectRequest).getObjectContent)
def getObjectContent(getObjectRequest: GetObjectRequest)(implicit F: Effect[F]): F[InputStream] =
F.delay(client.getObject(getObjectRequest).getObjectContent)
def initiateMultipartUpload(
initiateMultipartUploadRequest: InitiateMultipartUploadRequest
)(implicit F: Effect[F]): F[InitiateMultipartUploadResult] =
F.delay(client.initiateMultipartUpload(initiateMultipartUploadRequest))
def uploadPart(uploadPartRequest: UploadPartRequest)(implicit F: Effect[F]): F[UploadPartResult] =
F.delay(client.uploadPart(uploadPartRequest))
def completeMultipartUpload(
completeMultipartUploadRequest: CompleteMultipartUploadRequest
)(implicit F: Effect[F]): F[CompleteMultipartUploadResult] =
F.delay(client.completeMultipartUpload(completeMultipartUploadRequest))
def s3ObjectSummaries(
listObjectsV2Request: ListObjectsV2Request
)(implicit F: Effect[F]): F[List[S3ObjectSummary]] =
F.delay(client.listObjectsV2(listObjectsV2Request).getObjectSummaries.asScala.toList)
def getObject(objectRequest: GetObjectRequest)(implicit F: Effect[F]): F[S3Object] =
F.delay(client.getObject(objectRequest))
}
Example 71
| Source File: package.scala From fs2-aws with MIT License | 5 votes |
|
package fs2.aws
import java.io._
import cats.effect.{ Effect, IO }
import com.amazonaws.SdkClientException
import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model.{ AmazonS3Exception, GetObjectRequest, S3ObjectInputStream }
import fs2.aws.internal._
import org.apache.http.client.methods.HttpRequestBase
import scala.io.Source
package object utils {
val s3TestClient: S3Client[IO] = new S3Client[IO] {
override def client: AmazonS3 =
throw new NotImplementedError("s3 client shouldn't be used in this test client")
override def getObjectContentOrError(
getObjectRequest: GetObjectRequest
)(implicit e: Effect[IO]): IO[Either[Throwable, InputStream]] =
getObjectRequest match {
case goe: GetObjectRequest => {
IO[Either[Throwable, ByteArrayInputStream]] {
val fileContent: Array[Byte] =
try {
Source.fromResource(goe.getKey).mkString.getBytes
} catch {
case _: FileNotFoundException => throw new AmazonS3Exception("File not found")
case e: Throwable => throw e
}
goe.getRange match {
case Array(x, y) =>
if (y > fileContent.length)
Right(new ByteArrayInputStream(fileContent.slice(x.toInt, fileContent.length)))
else Right(new ByteArrayInputStream(fileContent.slice(x.toInt, y.toInt)))
}
} map {
case Left(e) => Left(e)
case Right(is) =>
Thread.sleep(500) // simulate a call to S3
Right(new S3ObjectInputStream(is, new HttpRequestBase {
def getMethod = ""
}))
}
}
case _ => throw new SdkClientException("Invalid GetObjectRequest")
}
override def getObjectContent(
getObjectRequest: GetObjectRequest
)(implicit e: Effect[IO]): IO[InputStream] =
IO[ByteArrayInputStream] {
val fileContent: Array[Byte] =
try {
val testS3Resource = Option(getObjectRequest.getVersionId) match {
case Some(version) => s"${getObjectRequest.getKey}_v$version"
case None => getObjectRequest.getKey
}
Source.fromResource(testS3Resource).mkString.getBytes
} catch {
case _: FileNotFoundException => throw new AmazonS3Exception("File not found")
case e: Throwable => throw e
}
new ByteArrayInputStream(fileContent)
}.map { is =>
Thread.sleep(500) // simulate a call to S3
new S3ObjectInputStream(is, new HttpRequestBase {
def getMethod = ""
})
}
}
}
Example 72
| Source File: CatsHelpers.scala From nelson with Apache License 2.0 | 5 votes |
|
package nelson
import cats.Eval
import cats.effect.{Effect, IO, Timer}
import cats.free.Cofree
import cats.syntax.functor._
import cats.syntax.monadError._
import fs2.{Pipe, Sink, Stream}
import quiver.{Context, Decomp, Graph}
import java.util.concurrent.TimeoutException
import scala.concurrent.ExecutionContext
import scala.concurrent.duration.FiniteDuration
import scala.collection.immutable.{Stream => SStream}
object CatsHelpers {
implicit class NelsonEnrichedIO[A](val io: IO[A]) extends AnyVal {
private type Tree[A] = Cofree[SStream, A]
private def flattenTree[A](tree: Tree[A]): SStream[A] = {
def go(tree: Tree[A], xs: SStream[A]): SStream[A] =
SStream.cons(tree.head, tree.tail.value.foldRight(xs)(go(_, _)))
go(tree, SStream.Empty)
}
private def Node[A](root: A, forest: => SStream[Tree[A]]): Tree[A] =
Cofree[SStream, A](root, Eval.later(forest))
implicit class NelsonEnrichedGraph[N, A, B](val graph: Graph[N, A, B]) extends AnyVal {
def reachable(v: N): Vector[N] =
xdfWith(Seq(v), _.successors, _.vertex)._1.flatMap(flattenTree)
def xdfWith[C](vs: Seq[N], d: Context[N, A, B] => Seq[N], f: Context[N, A, B] => C): (Vector[Tree[C]], Graph[N, A, B]) =
if (vs.isEmpty || graph.isEmpty) (Vector(), graph)
else graph.decomp(vs.head) match {
case Decomp(None, g) => g.xdfWith(vs.tail, d, f)
case Decomp(Some(c), g) =>
val (xs, _) = g.xdfWith(d(c), d, f)
val (ys, g3) = g.xdfWith(vs.tail, d, f)
(Node(f(c), xs.toStream) +: ys, g3)
}
}
}
Example 73
| Source File: package.scala From scala-pet-store with Apache License 2.0 | 5 votes |
|
package io.github.pauljamescleary.petstore
package infrastructure.repository
import cats.implicits._
import cats.effect.{Async, ContextShift, Effect, IO}
import config._
import _root_.doobie.Transactor
import io.circe.config.parser
import scala.concurrent.ExecutionContext
package object doobie {
def getTransactor[F[_]: Async: ContextShift](cfg: DatabaseConfig): Transactor[F] =
Transactor.fromDriverManager[F](
cfg.driver, // driver classname
cfg.url, // connect URL (driver-specific)
cfg.user, // user
cfg.password, // password
)
def initializedTransactor[F[_]: Effect: Async: ContextShift]: F[Transactor[F]] =
for {
petConfig <- parser.decodePathF[F, PetStoreConfig]("petstore")
_ <- DatabaseConfig.initializeDb(petConfig.db)
} yield getTransactor(petConfig.db)
lazy val testEc = ExecutionContext.Implicits.global
implicit lazy val testCs = IO.contextShift(testEc)
lazy val testTransactor = initializedTransactor[IO].unsafeRunSync()
}
Example 74
| Source File: Fs2AsyncQueue.scala From sttp with Apache License 2.0 | 5 votes |
|
package sttp.client.impl.fs2
import cats.effect.{Effect, IO}
import fs2.concurrent.InspectableQueue
import sttp.client.ws.internal.AsyncQueue
import sttp.model.ws.WebSocketBufferFull
import scala.language.higherKinds
class Fs2AsyncQueue[F[_], A](queue: InspectableQueue[F, A])(implicit F: Effect[F]) extends AsyncQueue[F, A] {
override def offer(t: A): Unit = {
F.toIO(queue.offer1(t))
.flatMap {
case true => IO.unit
case false => IO.raiseError(new WebSocketBufferFull())
}
.unsafeRunSync()
}
override def poll: F[A] = queue.dequeue1
}
Example 75
| Source File: RecoverLog.scala From polynote with Apache License 2.0 | 5 votes |
|
package polynote
import java.nio.channels.FileChannel
import java.nio.file.{Files, Paths, StandardOpenOption}
import java.time.Instant
import cats.effect.Effect
import polynote.app.{Args, MainArgs}
import polynote.kernel.logging.Logging
import polynote.messages.{Message, Notebook, NotebookUpdate, ShortList}
import polynote.server.AppEnv
import zio.{Ref, Runtime, Task, UIO, ZIO}
import zio.ZIO.effectTotal
import zio.blocking.effectBlocking
import fs2.Stream
import polynote.server.repository.{FileBasedRepository, NotebookContent}
import polynote.server.repository.format.ipynb.IPythonFormat
import polynote.server.repository.fs.WAL
import polynote.server.taskConcurrent
import scodec.bits.ByteVector
import scodec.stream.decode
import scodec.codecs
import scodec.stream.decode.StreamDecoder
object RecoverLog {
def replay(messages: Stream[Task, (Instant, Message)], ref: Ref[Notebook], log: Logging.Service): UIO[Unit] = messages.map(_._2).evalMap {
case nb: Notebook => ref.set(nb)
case upd: NotebookUpdate => ref.update {
nb => try {
upd.applyTo(nb)
} catch {
case err: Throwable =>
log.errorSync(Some("Dropped update because an error occurred when applying it"), err)
nb
}
}
case _ => ZIO.unit
}.compile.drain.catchAll {
err =>
log.error(Some("Error occurred while replaying the log; printing the final state anyway."), err)
}
def main(implicit ev: Effect[Task]): ZIO[AppEnv, String, Int] = for {
args <- ZIO.access[MainArgs](_.get[Args].rest)
path <- ZIO(args.head).flatMap(pathStr => effectBlocking(Paths.get(pathStr).toRealPath())).orDie
is <- effectBlocking(FileChannel.open(path, StandardOpenOption.READ)).orDie
log <- Logging.access
_ <- Logging.info(s"Reading log entries from ${path}...")
messages = WAL.decoder.decodeMmap(is)
ref <- Ref.make(Notebook("", ShortList.Nil, None))
_ <- replay(messages, ref, log)
format = new IPythonFormat
result <- ref.get
encoded <- format.encodeNotebook(NotebookContent(result.cells, result.config)).orDie
_ <- effectTotal(println(encoded))
} yield 0
}
Example 76
| Source File: Main.scala From polynote with Apache License 2.0 | 5 votes |
|
package polynote
import java.io.File
import cats.effect.Effect
import polynote.app.Environment
import polynote.config.{KernelIsolation, PolynoteConfig}
import polynote.kernel.{BaseEnv, GlobalEnv, Kernel, LocalKernel, LocalSparkKernel}
import polynote.kernel.environment.{Config, CurrentNotebook}
import polynote.kernel.environment.Env.LayerOps
import polynote.kernel.interpreter.Interpreter
import polynote.kernel.remote.{RemoteKernel, RemoteSparkKernel}
import polynote.messages.NotebookConfig
import polynote.server.{AppEnv, Server}
import polynote.app.{Args, MainArgs, globalEnv}
import polynote.server.auth.IdentityProvider
import polynote.server.repository.NotebookRepository
import polynote.server.repository.fs.FileSystems
import zio.{Has, Task, ULayer, ZIO, ZLayer}
abstract class Main
object Main extends polynote.app.App {
private implicit val taskEffect: Effect[Task] = zio.interop.catz.taskEffectInstance
val main: ZIO[AppEnv, Nothing, Int] =
MainArgs.access.flatMap {
args => args.command match {
case "server" => new Server().main
case "run" => NotebookRunner.main
case "recover" => RecoverLog.main
case other => ZIO.dieMessage(s"Unknown command $other (expected server)")
}
}.catchAll {
str => ZIO.effectTotal {
System.err.println(str)
System.err.println()
}.as(1)
}
override def main(args: List[String]): ZIO[Environment, Nothing, Int] = main.provideSomeLayer[BaseEnv] {
Args.parse(args).orDie andThen
((Config.layer.orDie ++ kernelFactory ++ FileSystems.live) andThen
globalEnv.orDie andThen NotebookRepository.live)
}
private val kernelFactory: ULayer[Kernel.Factory] = ZLayer.succeed {
Kernel.Factory.choose {
for {
notebook <- CurrentNotebook.get
config <- Config.access
} yield {
val notebookConfig = notebook.config.getOrElse(NotebookConfig.empty)
val isSpark = notebookConfig.sparkTemplate.nonEmpty || {
notebookConfig.sparkConfig match {
case None => false
case Some(map) if map.isEmpty => false
case Some(_) => true
}
}
config.behavior.kernelIsolation match {
case KernelIsolation.Always | KernelIsolation.SparkOnly if isSpark => RemoteSparkKernel
case KernelIsolation.Never if isSpark => LocalSparkKernel
case KernelIsolation.Always => RemoteKernel
case _ => LocalKernel
}
}
}
}
}
Example 77
| Source File: AirlinesJob.scala From core with Apache License 2.0 | 5 votes |
|
package com.smartbackpackerapp.airlines
import cats.effect.{Effect, IO}
import cats.syntax.flatMap._
import cats.syntax.functor._
import com.smartbackpackerapp.airlines.parser.{AirlineFile, AllowanceFile}
import fs2.StreamApp.ExitCode
import fs2.{Stream, StreamApp}
object AirlinesApp extends AirlinesJob[IO]
// See: https://wikitravel.org/en/Discount_airlines_in_Europe
class AirlinesJob[F[_]](implicit F: Effect[F]) extends StreamApp[F] {
private val ctx = new AirlinesModule[F]
case object MissingArgument extends Exception("There should be 2 arguments in the following order: `Airline file path` and `Allowance file path`.")
private def putStrLn(value: String): Stream[F, Unit] = Stream.eval(F.delay(println(value)))
def readArgs(args: List[String]): F[(AirlineFile, AllowanceFile)] = {
val ifEmpty = F.raiseError[String](MissingArgument)
for {
x <- args.headOption.fold(ifEmpty)(F.delay(_))
y <- args.lastOption.fold(ifEmpty)(F.delay(_))
} yield (AirlineFile(x), AllowanceFile(y))
}
def program(airlineFile: AirlineFile,
allowanceFile: AllowanceFile): Stream[F, Unit] =
for {
_ <- if (ctx.devDbUrl.nonEmpty) putStrLn(s"DEV DB connection established: ${ctx.devDbUrl}")
else putStrLn(s"DB connection established: ${ctx.dbUrl}")
_ <- putStrLn("Starting job")
_ <- ctx.airlinesInsertData(airlineFile, allowanceFile).run
_ <- putStrLn("Job finished successfully")
} yield ()
def stream(args: List[String], requestShutdown: F[Unit]): Stream[F, ExitCode] =
for {
files <- Stream.eval(readArgs(args))
(x, y) = files
exitCode <- program(x, y).drain
} yield exitCode
}
Example 78
| Source File: Server.scala From core with Apache License 2.0 | 5 votes |
|
package com.smartbackpackerapp
import cats.Parallel
import cats.effect.Effect
import com.smartbackpackerapp.http.auth.JwtTokenAuthMiddleware
import fs2.StreamApp.ExitCode
import fs2.{Scheduler, Stream, StreamApp}
import monix.eval.Task
import monix.execution.Scheduler.Implicits.global
import org.http4s.client.blaze.Http1Client
import org.http4s.server.blaze.BlazeBuilder
object Server extends HttpServer[Task, Task.Par]
class HttpServer[F[_], G[_]](implicit F: Effect[F], P: Parallel[F, G]) extends StreamApp[F] {
// Workaround until something like mirror comes out: https://github.com/typelevel/cats/pull/2019
implicit val parallel: Parallel[F, F] = P.asInstanceOf[Parallel[F, F]]
private lazy val ApiToken: F[Option[String]] = F.delay(sys.env.get("SB_API_TOKEN"))
override def stream(args: List[String], requestShutdown: F[Unit]): Stream[F, ExitCode] =
Scheduler(corePoolSize = 2).flatMap { implicit scheduler =>
for {
httpClient <- Http1Client.stream[F]()
ctx = new Module[F](httpClient)
_ <- Stream.eval(ctx.migrateDb)
_ <- Stream.eval(ctx.startMetricsReporter)
apiToken <- Stream.eval(ApiToken)
authMiddleware <- Stream.eval(JwtTokenAuthMiddleware[F](apiToken))
exitCode <- BlazeBuilder[F]
.bindHttp(sys.env.getOrElse("PORT", "8080").toInt, "0.0.0.0")
.mountService(authMiddleware(ctx.httpEndpointsWithMetrics))
.serve
} yield exitCode
}
}
Example 79
| Source File: ExchangeRateService.scala From core with Apache License 2.0 | 5 votes |
|
package com.smartbackpackerapp.service
import java.net.ConnectException
import cats.MonadError
import cats.effect.Effect
import cats.syntax.all._
import com.smartbackpackerapp.common.Log
import com.smartbackpackerapp.config.SBConfiguration
import com.smartbackpackerapp.model.Currency
import io.circe.generic.auto._
import org.http4s.circe._
import org.http4s.client.{Client, UnexpectedStatus}
class ExchangeRateService[F[_] : Effect](client: Client[F],
sbConfig: SBConfiguration[F])
(implicit L: Log[F]) extends AbstractExchangeRateService[F](sbConfig) {
override protected def retrieveExchangeRate(uri: String): F[CurrencyExchangeDTO] = {
client.expect[CurrencyExchangeDTO](uri)(jsonOf[F, CurrencyExchangeDTO])
}
}
abstract class AbstractExchangeRateService[F[_]](sbConfig: SBConfiguration[F])
(implicit F: MonadError[F, Throwable], L: Log[F]) {
protected val fixerUri: Currency => Currency => F[String] = baseCurrency => foreignCurrency => {
val uri = sbConfig.fixerBaseUri.map(_.getOrElse("http://localhost:8081"))
uri.map(x => s"$x/latest?base=${baseCurrency.value}&symbols=${foreignCurrency.value}")
}
protected def retrieveExchangeRate(uri: String): F[CurrencyExchangeDTO]
// We don't want the whole destination service to fail if the exchange rate service is unavailable
// so the `UnexpectedStatus` and `ConnectException` errors are treated as an empty exchange rate
def exchangeRateFor(baseCurrency: Currency, foreignCurrency: Currency): F[CurrencyExchangeDTO] = {
val ifEmpty = CurrencyExchangeDTO.empty(baseCurrency).pure[F]
def performRequest(uri: String): F[CurrencyExchangeDTO] =
retrieveExchangeRate(uri).recoverWith {
case e: ConnectException => L.error(e).flatMap(_ => ifEmpty)
case _: UnexpectedStatus => ifEmpty
}
validateCurrencies(baseCurrency, foreignCurrency).fold(ifEmpty) { _ =>
for {
uri <- fixerUri(baseCurrency)(foreignCurrency)
_ <- L.info(s"Retrieving currency exchange from: $uri")
er <- performRequest(uri)
} yield {
if (er.rates.nonEmpty) er
else er.copy(rates = Map(baseCurrency.value -> -1.0))
}
}
}
private def validateCurrencies(baseCurrency: Currency, foreignCurrency: Currency): Option[Currency] = {
if (baseCurrency == foreignCurrency) none[Currency]
else foreignCurrency.some
}
}
Example 80
| Source File: ArchiveCache.scala From nexus-kg with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.archives
import akka.actor.{ActorSystem, NotInfluenceReceiveTimeout}
import cats.Monad
import cats.data.OptionT
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.kg.archives.ArchiveCache._
import ch.epfl.bluebrain.nexus.kg.config.AppConfig._
import ch.epfl.bluebrain.nexus.kg.resources.ResId
import ch.epfl.bluebrain.nexus.sourcing.StateMachine
import ch.epfl.bluebrain.nexus.sourcing.akka.StopStrategy
import ch.epfl.bluebrain.nexus.sourcing.akka.statemachine.AkkaStateMachine
import retry.RetryPolicy
class ArchiveCache[F[_]: Monad](ref: StateMachine[F, String, State, Command, Unit]) {
def put(value: Archive): OptionT[F, Archive] =
OptionT(ref.evaluate(value.resId.show, Write(value)).map(_.toOption.flatten))
}
object ArchiveCache {
private[archives] type State = Option[Archive]
private[archives] type Command = Write
private[archives] final case class Write(bundle: Archive) extends NotInfluenceReceiveTimeout
final def apply[F[_]: Timer](implicit as: ActorSystem, cfg: ArchivesConfig, F: Effect[F]): F[ArchiveCache[F]] = {
implicit val retryPolicy: RetryPolicy[F] = cfg.cache.retry.retryPolicy[F]
val invalidationStrategy = StopStrategy.lapsedSinceLastInteraction[State, Command](cfg.cacheInvalidateAfter)
val evaluate: (State, Command) => F[Either[Unit, State]] = {
case (None, Write(bundle)) => F.pure(Right(Some(bundle)))
case (Some(_), _) => F.pure(Left(())) // It already exists, so we don't want to replace it
}
AkkaStateMachine
.sharded[F]("archives", None, evaluate, invalidationStrategy, cfg.cache.akkaStateMachineConfig, cfg.cache.shards)
.map(new ArchiveCache[F](_))
}
}
Example 81
| Source File: ElasticSearchIndexer.scala From nexus-kg with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.indexing
import akka.actor.{ActorRef, ActorSystem, Props}
import akka.stream.scaladsl.Source
import akka.util.Timeout
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.admin.client.types.Project
import ch.epfl.bluebrain.nexus.commons.es.client.ElasticSearchClient
import ch.epfl.bluebrain.nexus.commons.es.client.ElasticSearchClient.BulkOp
import ch.epfl.bluebrain.nexus.kg.config.AppConfig
import ch.epfl.bluebrain.nexus.kg.config.AppConfig._
import ch.epfl.bluebrain.nexus.kg.indexing.View.ElasticSearchView
import ch.epfl.bluebrain.nexus.kg.resources._
import ch.epfl.bluebrain.nexus.kg.routes.Clients
import ch.epfl.bluebrain.nexus.sourcing.projections.ProgressFlow.ProgressFlowElem
import ch.epfl.bluebrain.nexus.sourcing.projections.ProjectionProgress.NoProgress
import ch.epfl.bluebrain.nexus.sourcing.projections._
import com.typesafe.scalalogging.Logger
import scala.concurrent.ExecutionContext
// $COVERAGE-OFF$
@SuppressWarnings(Array("MaxParameters"))
object ElasticSearchIndexer {
private implicit val log: Logger = Logger[ElasticSearchIndexer.type]
final def start[F[_]: Timer](
view: ElasticSearchView,
resources: Resources[F],
project: Project,
restartOffset: Boolean
)(
implicit as: ActorSystem,
actorInitializer: (Props, String) => ActorRef,
projections: Projections[F, String],
F: Effect[F],
clients: Clients[F],
config: AppConfig
): StreamSupervisor[F, ProjectionProgress] = {
implicit val ec: ExecutionContext = as.dispatcher
implicit val p: Project = project
implicit val indexing: IndexingConfig = config.elasticSearch.indexing
implicit val metadataOpts: MetadataOptions = MetadataOptions(linksAsIri = true, expandedLinks = true)
implicit val tm: Timeout = Timeout(config.elasticSearch.askTimeout)
val client: ElasticSearchClient[F] = clients.elasticSearch.withRetryPolicy(config.elasticSearch.indexing.retry)
def deleteOrIndex(res: ResourceV): Option[BulkOp] =
if (res.deprecated && !view.filter.includeDeprecated) Some(delete(res))
else view.toDocument(res).map(doc => BulkOp.Index(view.index, res.id.value.asString, doc))
def delete(res: ResourceV): BulkOp =
BulkOp.Delete(view.index, res.id.value.asString)
val initFetchProgressF: F[ProjectionProgress] =
if (restartOffset)
projections.recordProgress(view.progressId, NoProgress) >> view.createIndex >> F.pure(NoProgress)
else view.createIndex >> projections.progress(view.progressId)
val sourceF: F[Source[ProjectionProgress, _]] = initFetchProgressF.map { initial =>
val flow = ProgressFlowElem[F, Any]
.collectCast[Event]
.groupedWithin(indexing.batch, indexing.batchTimeout)
.distinct()
.mapAsync(view.toResource(resources, _))
.collectSome[ResourceV]
.collect {
case res if view.allowedSchemas(res) && view.allowedTypes(res) => deleteOrIndex(res)
case res if view.allowedSchemas(res) => Some(delete(res))
}
.collectSome[BulkOp]
.runAsyncBatch(client.bulk(_))()
.mergeEmit()
.toPersistedProgress(view.progressId, initial)
cassandraSource(s"project=${view.ref.id}", view.progressId, initial.minProgress.offset)
.via(flow)
.via(kamonViewMetricsFlow(view, project))
}
StreamSupervisor.start(sourceF, view.progressId, actorInitializer)
}
}
// $COVERAGE-ON$
Example 82
| Source File: StorageIndexer.scala From nexus-kg with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.indexing
import java.time.Instant
import akka.actor.ActorSystem
import akka.stream.scaladsl.{Flow, Source}
import akka.util.Timeout
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.admin.client.AdminClient
import ch.epfl.bluebrain.nexus.kg.cache.{ProjectCache, StorageCache}
import ch.epfl.bluebrain.nexus.kg.config.AppConfig
import ch.epfl.bluebrain.nexus.kg.config.AppConfig._
import ch.epfl.bluebrain.nexus.kg.config.Vocabulary.nxv
import ch.epfl.bluebrain.nexus.kg.resources._
import ch.epfl.bluebrain.nexus.kg.storage.Storage
import ch.epfl.bluebrain.nexus.sourcing.projections.ProgressFlow.{PairMsg, ProgressFlowElem}
import ch.epfl.bluebrain.nexus.sourcing.projections._
import com.typesafe.scalalogging.Logger
import scala.concurrent.ExecutionContext
// $COVERAGE-OFF$
object StorageIndexer {
private implicit val log = Logger[StorageIndexer.type]
def start[F[_]: Timer](storages: Storages[F], storageCache: StorageCache[F])(
implicit projectCache: ProjectCache[F],
F: Effect[F],
as: ActorSystem,
projectInitializer: ProjectInitializer[F],
adminClient: AdminClient[F],
config: AppConfig
): StreamSupervisor[F, Unit] = {
implicit val authToken = config.iam.serviceAccountToken
implicit val indexing: IndexingConfig = config.keyValueStore.indexing
implicit val ec: ExecutionContext = as.dispatcher
implicit val tm: Timeout = Timeout(config.keyValueStore.askTimeout)
val name = "storage-indexer"
def toStorage(event: Event): F[Option[(Storage, Instant)]] =
fetchProject(event.organization, event.id.parent, event.subject).flatMap { implicit project =>
storages.fetchStorage(event.id).value.map {
case Left(err) =>
log.error(s"Error on event '${event.id.show} (rev = ${event.rev})', cause: '${err.msg}'")
None
case Right(timedStorage) => Some(timedStorage)
}
}
val source: Source[PairMsg[Any], _] = cassandraSource(s"type=${nxv.Storage.value.show}", name)
val flow: Flow[PairMsg[Any], Unit, _] = ProgressFlowElem[F, Any]
.collectCast[Event]
.groupedWithin(indexing.batch, indexing.batchTimeout)
.distinct()
.mergeEmit()
.mapAsync(toStorage)
.collectSome[(Storage, Instant)]
.runAsync { case (storage, instant) => storageCache.put(storage)(instant) }()
.flow
.map(_ => ())
StreamSupervisor.startSingleton(F.delay(source.via(flow)), name)
}
}
// $COVERAGE-ON$
Example 83
| Source File: SparqlIndexer.scala From nexus-kg with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.indexing
import akka.actor.{ActorRef, ActorSystem, Props}
import akka.stream.scaladsl.Source
import akka.util.Timeout
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.admin.client.types.Project
import ch.epfl.bluebrain.nexus.commons.sparql.client.{BlazegraphClient, SparqlWriteQuery}
import ch.epfl.bluebrain.nexus.kg.config.AppConfig
import ch.epfl.bluebrain.nexus.kg.config.AppConfig._
import ch.epfl.bluebrain.nexus.kg.indexing.View.SparqlView
import ch.epfl.bluebrain.nexus.kg.resources._
import ch.epfl.bluebrain.nexus.kg.routes.Clients
import ch.epfl.bluebrain.nexus.sourcing.projections.ProgressFlow.ProgressFlowElem
import ch.epfl.bluebrain.nexus.sourcing.projections.ProjectionProgress.NoProgress
import ch.epfl.bluebrain.nexus.sourcing.projections._
import scala.concurrent.ExecutionContext
// $COVERAGE-OFF$
@SuppressWarnings(Array("MaxParameters"))
object SparqlIndexer {
final def start[F[_]: Timer](
view: SparqlView,
resources: Resources[F],
project: Project,
restartOffset: Boolean
)(
implicit as: ActorSystem,
actorInitializer: (Props, String) => ActorRef,
projections: Projections[F, String],
F: Effect[F],
clients: Clients[F],
config: AppConfig
): StreamSupervisor[F, ProjectionProgress] = {
implicit val ec: ExecutionContext = as.dispatcher
implicit val p: Project = project
implicit val indexing: IndexingConfig = config.sparql.indexing
implicit val metadataOpts: MetadataOptions = MetadataOptions(linksAsIri = true, expandedLinks = true)
implicit val tm: Timeout = Timeout(config.sparql.askTimeout)
val client: BlazegraphClient[F] =
clients.sparql.copy(namespace = view.index).withRetryPolicy(config.sparql.indexing.retry)
def buildInsertOrDeleteQuery(res: ResourceV): SparqlWriteQuery =
if (res.deprecated && !view.filter.includeDeprecated) view.buildDeleteQuery(res)
else view.buildInsertQuery(res)
val initFetchProgressF: F[ProjectionProgress] =
if (restartOffset)
projections.recordProgress(view.progressId, NoProgress) >> view.createIndex >> F.pure(NoProgress)
else view.createIndex >> projections.progress(view.progressId)
val sourceF: F[Source[ProjectionProgress, _]] = initFetchProgressF.map { initial =>
val flow = ProgressFlowElem[F, Any]
.collectCast[Event]
.groupedWithin(indexing.batch, indexing.batchTimeout)
.distinct()
.mapAsync(view.toResource(resources, _))
.collectSome[ResourceV]
.collect {
case res if view.allowedSchemas(res) && view.allowedTypes(res) => buildInsertOrDeleteQuery(res)
case res if view.allowedSchemas(res) => view.buildDeleteQuery(res)
}
.runAsyncBatch(client.bulk(_))()
.mergeEmit()
.toPersistedProgress(view.progressId, initial)
cassandraSource(s"project=${view.ref.id}", view.progressId, initial.minProgress.offset)
.via(flow)
.via(kamonViewMetricsFlow(view, project))
}
StreamSupervisor.start(sourceF, view.progressId, actorInitializer)
}
}
// $COVERAGE-ON$
Example 84
| Source File: ResolverIndexer.scala From nexus-kg with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.indexing
import akka.actor.ActorSystem
import akka.stream.scaladsl.{Flow, Source}
import akka.util.Timeout
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.admin.client.AdminClient
import ch.epfl.bluebrain.nexus.kg.cache.{ProjectCache, ResolverCache}
import ch.epfl.bluebrain.nexus.kg.config.AppConfig
import ch.epfl.bluebrain.nexus.kg.config.AppConfig._
import ch.epfl.bluebrain.nexus.kg.config.Vocabulary.nxv
import ch.epfl.bluebrain.nexus.kg.resolve.Resolver
import ch.epfl.bluebrain.nexus.kg.resources._
import ch.epfl.bluebrain.nexus.sourcing.projections.ProgressFlow.{PairMsg, ProgressFlowElem}
import ch.epfl.bluebrain.nexus.sourcing.projections._
import com.typesafe.scalalogging.Logger
import scala.concurrent.ExecutionContext
// $COVERAGE-OFF$
object ResolverIndexer {
private implicit val log = Logger[ResolverIndexer.type]
final def start[F[_]: Timer](resolvers: Resolvers[F], resolverCache: ResolverCache[F])(
implicit
projectCache: ProjectCache[F],
as: ActorSystem,
F: Effect[F],
projectInitializer: ProjectInitializer[F],
adminClient: AdminClient[F],
config: AppConfig
): StreamSupervisor[F, Unit] = {
implicit val authToken = config.iam.serviceAccountToken
implicit val indexing: IndexingConfig = config.keyValueStore.indexing
implicit val ec: ExecutionContext = as.dispatcher
implicit val tm: Timeout = Timeout(config.keyValueStore.askTimeout)
val name = "resolver-indexer"
def toResolver(event: Event): F[Option[Resolver]] =
fetchProject(event.organization, event.id.parent, event.subject).flatMap { implicit project =>
resolvers.fetchResolver(event.id).value.map {
case Left(err) =>
log.error(s"Error on event '${event.id.show} (rev = ${event.rev})', cause: '${err.msg}'")
None
case Right(resolver) => Some(resolver)
}
}
val source: Source[PairMsg[Any], _] = cassandraSource(s"type=${nxv.Resolver.value.show}", name)
val flow: Flow[PairMsg[Any], Unit, _] = ProgressFlowElem[F, Any]
.collectCast[Event]
.groupedWithin(indexing.batch, indexing.batchTimeout)
.distinct()
.mergeEmit()
.mapAsync(toResolver)
.collectSome[Resolver]
.runAsync(resolverCache.put)()
.flow
.map(_ => ())
StreamSupervisor.startSingleton(F.delay(source.via(flow)), name)
}
}
// $COVERAGE-ON$
Example 85
| Source File: ViewIndexer.scala From nexus-kg with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.indexing
import akka.actor.ActorSystem
import akka.stream.scaladsl.{Flow, Source}
import akka.util.Timeout
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.admin.client.AdminClient
import ch.epfl.bluebrain.nexus.kg.cache.{ProjectCache, ViewCache}
import ch.epfl.bluebrain.nexus.kg.config.AppConfig
import ch.epfl.bluebrain.nexus.kg.config.AppConfig._
import ch.epfl.bluebrain.nexus.kg.config.Vocabulary.nxv
import ch.epfl.bluebrain.nexus.kg.resources._
import ch.epfl.bluebrain.nexus.sourcing.projections.ProgressFlow.{PairMsg, ProgressFlowElem}
import ch.epfl.bluebrain.nexus.sourcing.projections._
import com.typesafe.scalalogging.Logger
import scala.concurrent.ExecutionContext
// $COVERAGE-OFF$
object ViewIndexer {
private implicit val log = Logger[ViewIndexer.type]
def start[F[_]: Timer](views: Views[F], viewCache: ViewCache[F])(
implicit projectCache: ProjectCache[F],
F: Effect[F],
as: ActorSystem,
projectInitializer: ProjectInitializer[F],
adminClient: AdminClient[F],
config: AppConfig
): StreamSupervisor[F, Unit] = {
implicit val authToken = config.iam.serviceAccountToken
implicit val indexing: IndexingConfig = config.keyValueStore.indexing
implicit val ec: ExecutionContext = as.dispatcher
implicit val tm: Timeout = Timeout(config.keyValueStore.askTimeout)
val name = "view-indexer"
def toView(event: Event): F[Option[View]] =
fetchProject(event.organization, event.id.parent, event.subject).flatMap { implicit project =>
views.fetchView(event.id).value.map {
case Left(err) =>
log.error(s"Error on event '${event.id.show} (rev = ${event.rev})', cause: '${err.msg}'")
None
case Right(view) => Some(view)
}
}
val source: Source[PairMsg[Any], _] = cassandraSource(s"type=${nxv.View.value.show}", name)
val flow: Flow[PairMsg[Any], Unit, _] = ProgressFlowElem[F, Any]
.collectCast[Event]
.groupedWithin(indexing.batch, indexing.batchTimeout)
.distinct()
.mergeEmit()
.mapAsync(toView)
.collectSome[View]
.runAsync(viewCache.put)()
.flow
.map(_ => ())
StreamSupervisor.startSingleton(F.delay(source.via(flow)), name)
}
}
// $COVERAGE-ON$
Example 86
| Source File: ResolverCache.scala From nexus-kg with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.cache
import java.util.UUID
import java.util.concurrent.ConcurrentHashMap
import akka.actor.ActorSystem
import cats.Monad
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.commons.cache.{KeyValueStore, KeyValueStoreConfig}
import ch.epfl.bluebrain.nexus.kg.cache.Cache._
import ch.epfl.bluebrain.nexus.kg.resolve.Resolver
import ch.epfl.bluebrain.nexus.kg.resources.ProjectIdentifier.ProjectRef
import ch.epfl.bluebrain.nexus.rdf.Iri.AbsoluteIri
class ResolverCache[F[_]: Effect: Timer] private (projectToCache: ConcurrentHashMap[UUID, ResolverProjectCache[F]])(
implicit as: ActorSystem,
config: KeyValueStoreConfig
) {
private class ResolverProjectCache[F[_]: Monad] private (store: KeyValueStore[F, AbsoluteIri, Resolver])
extends Cache[F, AbsoluteIri, Resolver](store) {
private implicit val ordering: Ordering[Resolver] = Ordering.by(_.priority)
def get: F[List[Resolver]] = store.values.map(_.toList.sorted)
def put(resolver: Resolver): F[Unit] =
if (resolver.deprecated) store.remove(resolver.id)
else store.put(resolver.id, resolver)
}
private object ResolverProjectCache {
def apply[F[_]: Effect: Timer](
project: ProjectRef
)(implicit as: ActorSystem, config: KeyValueStoreConfig): ResolverProjectCache[F] =
new ResolverProjectCache(KeyValueStore.distributed(s"resolver-${project.id}", (_, resolver) => resolver.rev))
}
object ResolverCache {
def apply[F[_]: Effect: Timer](implicit as: ActorSystem, config: KeyValueStoreConfig): ResolverCache[F] =
new ResolverCache(new ConcurrentHashMap[UUID, ResolverProjectCache[F]]())
}
Example 87
| Source File: StorageCache.scala From nexus-kg with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.cache
import java.time.{Clock, Instant}
import java.util.UUID
import java.util.concurrent.ConcurrentHashMap
import akka.actor.ActorSystem
import cats.Monad
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.commons.cache.{KeyValueStore, KeyValueStoreConfig}
import ch.epfl.bluebrain.nexus.kg.RevisionedValue
import ch.epfl.bluebrain.nexus.kg.cache.Cache._
import ch.epfl.bluebrain.nexus.kg.cache.StorageProjectCache._
import ch.epfl.bluebrain.nexus.kg.resources.ProjectIdentifier.ProjectRef
import ch.epfl.bluebrain.nexus.kg.storage.Storage
import ch.epfl.bluebrain.nexus.rdf.Iri.AbsoluteIri
class StorageCache[F[_]: Effect: Timer] private (projectToCache: ConcurrentHashMap[UUID, StorageProjectCache[F]])(
implicit as: ActorSystem,
config: KeyValueStoreConfig,
clock: Clock
) {
private class StorageProjectCache[F[_]: Monad] private (store: KeyValueStore[F, AbsoluteIri, RevisionedStorage])
extends Cache[F, AbsoluteIri, RevisionedStorage](store) {
private implicit val ordering: Ordering[RevisionedStorage] = Ordering.by((s: RevisionedStorage) => s.rev).reverse
private implicit def revisioned(storage: Storage)(implicit instant: Instant): RevisionedStorage =
RevisionedValue(instant.toEpochMilli, storage)
def get: F[List[Storage]] =
store.values.map(_.toList.sorted.map(_.value))
def getDefault: F[Option[Storage]] =
get.map(_.collectFirst { case storage if storage.default => storage })
def getBy(id: AbsoluteIri): F[Option[Storage]] =
get(id).map(_.collectFirst { case RevisionedValue(_, storage) if storage.id == id => storage })
def put(storage: Storage)(implicit instant: Instant): F[Unit] =
if (storage.deprecated) store.remove(storage.id)
else store.put(storage.id, storage)
}
private object StorageProjectCache {
type RevisionedStorage = RevisionedValue[Storage]
def apply[F[_]: Effect: Timer](
project: ProjectRef
)(implicit as: ActorSystem, config: KeyValueStoreConfig): StorageProjectCache[F] =
new StorageProjectCache(
KeyValueStore.distributed(s"storage-${project.id}", (_, storage) => storage.value.rev)
)
}
object StorageCache {
def apply[F[_]: Timer: Effect](implicit as: ActorSystem, config: KeyValueStoreConfig, clock: Clock): StorageCache[F] =
new StorageCache(new ConcurrentHashMap[UUID, StorageProjectCache[F]]())
}
Example 88
| Source File: AclsCache.scala From nexus-kg with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.kg.cache
import akka.actor.ActorSystem
import cats.Monad
import cats.effect.{Effect, Timer}
import cats.implicits._
import ch.epfl.bluebrain.nexus.commons.cache.KeyValueStore
import ch.epfl.bluebrain.nexus.iam.client.IamClient
import ch.epfl.bluebrain.nexus.iam.client.types.events.Event._
import ch.epfl.bluebrain.nexus.iam.client.types.{AccessControlList, AccessControlLists, ResourceAccessControlList}
import ch.epfl.bluebrain.nexus.kg.config.AppConfig
import ch.epfl.bluebrain.nexus.kg.config.AppConfig._
import ch.epfl.bluebrain.nexus.kg.config.Vocabulary._
import ch.epfl.bluebrain.nexus.rdf.Iri.Path
def apply[F[_]: Effect: Timer](
iamClient: IamClient[F]
)(implicit as: ActorSystem, config: AppConfig): AclsCache[F] = {
val cache = new AclsCache(KeyValueStore.distributed("acls", (_, acls) => acls.rev))
val handle: AclEvent => F[Unit] = {
case event: AclReplaced => cache.replace(event.path, toResourceAcl(event, event.acl))
case event: AclAppended => cache.append(event.path, toResourceAcl(event, event.acl))
case event: AclSubtracted => cache.subtract(event.path, toResourceAcl(event, event.acl))
case event: AclDeleted => cache.remove(event.path)
}
iamClient.aclEvents(handle)(config.iam.serviceAccountToken)
cache
}
}
Example 89
| Source File: CatsInterop.scala From caliban with Apache License 2.0 | 5 votes |
|
package caliban.interop.cats
import caliban.introspection.adt.__Type
import caliban.schema.Step.QueryStep
import caliban.schema.{ Schema, Step }
import caliban.{ CalibanError, GraphQL, GraphQLInterpreter, GraphQLResponse, InputValue }
import cats.effect.implicits._
import cats.effect.{ Async, Effect }
import zio.interop.catz._
import zio.{ Runtime, _ }
import zio.query.ZQuery
object CatsInterop {
def executeAsync[F[_]: Async, R, E](graphQL: GraphQLInterpreter[R, E])(
query: String,
operationName: Option[String] = None,
variables: Map[String, InputValue] = Map(),
extensions: Map[String, InputValue] = Map(),
skipValidation: Boolean = false,
enableIntrospection: Boolean = true
)(implicit runtime: Runtime[R]): F[GraphQLResponse[E]] =
Async[F].async { cb =>
val execution = graphQL.execute(
query,
operationName,
variables,
extensions,
skipValidation = skipValidation,
enableIntrospection = enableIntrospection
)
runtime.unsafeRunAsync(execution)(exit => cb(exit.toEither))
}
def checkAsync[F[_]: Async, R](
graphQL: GraphQLInterpreter[R, Any]
)(query: String)(implicit runtime: Runtime[R]): F[Unit] =
Async[F].async(cb => runtime.unsafeRunAsync(graphQL.check(query))(exit => cb(exit.toEither)))
def interpreterAsync[F[_]: Async, R](
graphQL: GraphQL[R]
)(implicit runtime: Runtime[R]): F[GraphQLInterpreter[R, CalibanError]] =
Async[F].async(cb => runtime.unsafeRunAsync(graphQL.interpreter)(exit => cb(exit.toEither)))
def schema[F[_]: Effect, R, A](implicit ev: Schema[R, A]): Schema[R, F[A]] =
new Schema[R, F[A]] {
override def toType(isInput: Boolean, isSubscription: Boolean): __Type =
ev.toType(isInput, isSubscription)
override def optional: Boolean =
ev.optional
override def resolve(value: F[A]): Step[R] =
QueryStep(ZQuery.fromEffect(value.toIO.to[Task].map(ev.resolve)))
}
}
Example 90
| Source File: package.scala From caliban with Apache License 2.0 | 5 votes |
|
package caliban.interop.cats
import caliban.schema.Schema
import caliban.{ CalibanError, GraphQL, GraphQLInterpreter, GraphQLResponse, InputValue }
import cats.effect.{ Async, Effect }
import zio.Runtime
package object implicits {
implicit class CatsEffectGraphQLInterpreter[R, E](underlying: GraphQLInterpreter[R, E]) {
def executeAsync[F[_]: Async](
query: String,
operationName: Option[String] = None,
variables: Map[String, InputValue] = Map(),
extensions: Map[String, InputValue] = Map(),
skipValidation: Boolean = false,
enableIntrospection: Boolean = true
)(implicit runtime: Runtime[R]): F[GraphQLResponse[E]] =
CatsInterop.executeAsync(underlying)(
query,
operationName,
variables,
extensions,
skipValidation = skipValidation,
enableIntrospection = enableIntrospection
)
def checkAsync[F[_]: Async](query: String)(implicit runtime: Runtime[R]): F[Unit] =
CatsInterop.checkAsync(underlying)(query)
}
implicit class CatsEffectGraphQL[R, E](underlying: GraphQL[R]) {
def interpreterAsync[F[_]: Async](implicit runtime: Runtime[R]): F[GraphQLInterpreter[R, CalibanError]] =
CatsInterop.interpreterAsync(underlying)
}
implicit def effectSchema[F[_]: Effect, R, A](implicit ev: Schema[R, A]): Schema[R, F[A]] =
CatsInterop.schema
}
Example 91
| Source File: package.scala From interop-cats with Apache License 2.0 | 5 votes |
|
package zio
import cats.effect.{ Effect, ExitCase, LiftIO }
import zio.interop.catz.taskEffectInstance
package object interop {
type ParIO[-R, +E, +A] = Par.T[R, E, A]
type Queue[F[+_], A] = CQueue[F, A, A]
@inline private[interop] final def exitToExitCase(exit: Exit[Any, Any]): ExitCase[Throwable] = exit match {
case Exit.Success(_) => ExitCase.Completed
case Exit.Failure(cause) if cause.interrupted => ExitCase.Canceled
case Exit.Failure(cause) =>
cause.failureOrCause match {
case Left(t: Throwable) => ExitCase.Error(t)
case _ => ExitCase.Error(FiberFailure(cause))
}
}
@inline private[interop] final def exitCaseToExit[E](exitCase: ExitCase[E]): Exit[E, Unit] = exitCase match {
case ExitCase.Completed => Exit.unit
case ExitCase.Error(e) => Exit.fail(e)
case ExitCase.Canceled => Exit.interrupt(Fiber.Id.None)
}
private[interop] def fromEffect[F[+_], R, A](
eff: F[A]
)(implicit R: Runtime[R], F: Effect[F]): RIO[R, A] =
taskEffectInstance.liftIO[A](F.toIO(eff))
private[interop] def toEffect[F[+_], R, A](zio: RIO[R, A])(implicit R: Runtime[R], F: LiftIO[F]): F[A] =
F.liftIO(taskEffectInstance.toIO(zio))
}
Example 92
| Source File: MicrometerHttp4sMetricsOpsModule.scala From scala-server-toolkit with MIT License | 5 votes |
|
package com.avast.sst.http4s.server.micrometer
import java.util.concurrent.TimeUnit
import cats.effect.Effect
import cats.effect.concurrent.Ref
import cats.syntax.functor._
import io.micrometer.core.instrument.MeterRegistry
import org.http4s.metrics.{MetricsOps, TerminationType}
import org.http4s.{Method, Status}
object MicrometerHttp4sMetricsOpsModule {
def make[F[_]: Effect](meterRegistry: MeterRegistry): F[MetricsOps[F]] = {
val F = Effect[F]
for {
activeRequests <- Ref.of[F, Long](0L)
} yield new MetricsOps[F] {
private val prefix = "http.global"
private val failureTime = meterRegistry.timer(s"$prefix.failure-time")
meterRegistry.gauge(
s"$prefix.active-requests",
activeRequests,
(_: Ref[F, Long]) => Effect[F].toIO(activeRequests.get).unsafeRunSync().toDouble
)
override def increaseActiveRequests(classifier: Option[String]): F[Unit] = activeRequests.update(_ + 1)
override def decreaseActiveRequests(classifier: Option[String]): F[Unit] = activeRequests.update(_ - 1)
override def recordHeadersTime(method: Method, elapsed: Long, classifier: Option[String]): F[Unit] = {
F.delay(meterRegistry.timer(s"$prefix.headers-time", "method", method.name).record(elapsed, TimeUnit.NANOSECONDS))
}
override def recordTotalTime(method: Method, status: Status, elapsed: Long, classifier: Option[String]): F[Unit] = {
F.delay(
meterRegistry
.timer(s"$prefix.total-time", "status", s"${status.code}", "status-class", s"${status.code / 100}xx")
.record(elapsed, TimeUnit.NANOSECONDS)
)
}
override def recordAbnormalTermination(elapsed: Long, terminationType: TerminationType, classifier: Option[String]): F[Unit] = {
F.delay(failureTime.record(elapsed, TimeUnit.NANOSECONDS))
}
}
}
}
Example 93
| Source File: Decrypt.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.kms
import cats.effect.Effect
import io.circe.{Decoder, Json}
import org.aws4s.Region
import org.aws4s.core.Command.Validator
import org.aws4s.core.{CommandPayload, Param}
private[kms] case class Decrypt[F[_]: Effect](
region: Region,
ciphertext: Ciphertext,
context: Option[EncryptionContext],
grantTokens: Option[GrantTokens],
) extends KmsCommand[F, DecryptSuccess] {
override def action: String = "Decrypt"
override def params: List[Param[Json]] =
CommandPayload.params(ciphertext)(context, grantTokens)
override val validator: Validator[Json] = _ => None
}
case class DecryptSuccess(
plainText: Plaintext,
)
object DecryptSuccess {
implicit val decoder: Decoder[DecryptSuccess] =
Decoder.forProduct1(Plaintext.name) { (plaintext: Plaintext) =>
DecryptSuccess(plaintext)
}
}
Example 94
| Source File: KmsCommand.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.kms
import cats.effect.Effect
import cats.implicits._
import io.circe.{Decoder, Json}
import org.aws4s.core.ExtraEntityDecoderInstances._
import org.aws4s._
import org.aws4s.core.{Command, CommandPayload, RenderedParam, ServiceName}
import org.http4s.circe._
import org.http4s.headers.{Host, `Content-Type`}
import org.http4s.{Header, Headers, MediaType, Method, Request, Uri}
private[kms] abstract class KmsCommand[F[_]: Effect, R: Decoder] extends Command[F, Json, R] {
override def serviceName: ServiceName = ServiceName.Kms
override def payloadSigning: PayloadSigning = PayloadSigning.Signed
def action: String
override final val requestGenerator: List[RenderedParam[Json]] => F[Request[F]] = { params =>
val host = s"kms.${region.name}.amazonaws.com"
val payload: Json = CommandPayload.jsonObject(params)
Request[F](
Method.POST,
Uri.unsafeFromString(s"https://$host/"),
headers = Headers(Header("X-Amz-Target", s"TrentService.$action"), Host(host))
).withBody(payload).map(_.withContentType(`Content-Type`.apply(MediaType.fromKey(("application", "x-amz-json-1.1")))))
}
}
Example 95
| Source File: Encrypt.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.kms
import cats.effect.Effect
import io.circe.{Decoder, Json}
import org.aws4s.Region
import org.aws4s.core.Command.Validator
import org.aws4s.core.{CommandPayload, Param}
private[kms] case class Encrypt[F[_]: Effect](
region: Region,
keyId: KeyId,
plaintext: Plaintext,
context: Option[EncryptionContext],
grantTokens: Option[GrantTokens],
) extends KmsCommand[F, EncryptSuccess] {
override def action: String = "Encrypt"
override def params: List[Param[Json]] =
CommandPayload.params(keyId, plaintext)(context, grantTokens)
override val validator: Validator[Json] = _ => None
}
case class EncryptSuccess(
cipherText: Ciphertext,
)
object EncryptSuccess {
implicit val decoder: Decoder[EncryptSuccess] =
Decoder.forProduct1(Ciphertext.name) { (cipherText: Ciphertext) =>
EncryptSuccess(cipherText)
}
}
Example 96
| Source File: CreateKey.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.kms
import cats.effect.Effect
import io.circe.{Decoder, Json}
import org.aws4s.Region
import org.aws4s.core.Command.Validator
import org.aws4s.core.{CommandPayload, Param}
private[kms] case class CreateKey[F[_]: Effect](
region: Region,
description: Option[KeyDescription],
) extends KmsCommand[F, CreateKeySuccess] {
override val action: String = "CreateKey"
override def params: List[Param[Json]] = CommandPayload.params()(description)
override val validator: Validator[Json] = _ => None
}
case class CreateKeySuccess(
keyMetadata: KeyMetadata,
)
object CreateKeySuccess {
implicit val decoder: Decoder[CreateKeySuccess] =
Decoder.forProduct1("KeyMetadata")(CreateKeySuccess.apply)
}
Example 97
package org.aws4s.kms
import cats.effect.Effect
import io.circe.Json
import org.aws4s.core.ExtraEntityDecoderInstances._
import org.aws4s.core.Service
import org.aws4s.{Credentials, Region}
import org.http4s.client.Client
case class Kms[F[_]: Effect](client: F[Client[F]], region: Region, credentials: () => Credentials) extends Service[F, Json] {
def encrypt(
keyId: KeyId,
plaintext: Plaintext,
context: Option[EncryptionContext] = None,
grantTokens: Option[GrantTokens] = None,
): F[EncryptSuccess] = run {
Encrypt(
region,
keyId,
plaintext,
context,
grantTokens
)
}
def decrypt(
ciphertext: Ciphertext,
context: Option[EncryptionContext] = None,
grantTokens: Option[GrantTokens] = None,
): F[DecryptSuccess] = run {
Decrypt(region, ciphertext, context, grantTokens)
}
def createKey(description: Option[KeyDescription] = None): F[CreateKeySuccess] = run {
CreateKey(region, description)
}
def scheduleKeyDeletion(keyId: KeyId, pendingWindowInDays: Option[PendingWindowInDays] = None): F[ScheduleKeyDeletionSuccess] = run {
ScheduleKeyDeletion(region, keyId, pendingWindowInDays)
}
}
Example 98
| Source File: ScheduleKeyDeletion.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.kms
import java.time.Instant
import cats.effect.Effect
import io.circe.{Decoder, Json}
import org.aws4s.Region
import org.aws4s.core.ExtraCirceDecoders._
import org.aws4s.core.Command.Validator
import org.aws4s.core.{CommandPayload, Param}
private[kms] case class ScheduleKeyDeletion[F[_]: Effect](
region: Region,
keyId: KeyId,
pendingWindowInDays: Option[PendingWindowInDays],
) extends KmsCommand[F, ScheduleKeyDeletionSuccess] {
override def action: String = "ScheduleKeyDeletion"
override val validator: Validator[Json] = _ => None
override def params: List[Param[Json]] = CommandPayload.params(keyId)(pendingWindowInDays)
}
case class ScheduleKeyDeletionSuccess(
keyId: KeyId,
deletionDate: Instant,
)
object ScheduleKeyDeletionSuccess {
implicit val decoder: Decoder[ScheduleKeyDeletionSuccess] =
Decoder.forProduct2(
KeyId.name,
"DeletionDate"
)(ScheduleKeyDeletionSuccess.apply)
}
Example 99
| Source File: ReceiveMessage.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.sqs
import cats.effect.Effect
import org.http4s.EntityDecoder
import cats.implicits._
import org.aws4s.core.{CommandPayload, ExtraEntityDecoderInstances, Param}
private[sqs] case class ReceiveMessage[F[_]: Effect](
q: Queue,
maxNumberOfMessages: Option[MaxNumberOfMessages],
visibilityTimeout: Option[VisibilityTimeout],
waitTimeSeconds: Option[WaitTimeSeconds],
receiveRequestAttemptId: Option[ReceiveRequestAttemptId],
) extends SqsCommand[F, ReceiveMessageSuccess] {
override val action: String = "ReceiveMessage"
override final val params: List[Param[String]] =
CommandPayload.params()(
maxNumberOfMessages,
visibilityTimeout,
waitTimeSeconds,
receiveRequestAttemptId
)
}
case class ReceiveMessageSuccess(
messages: List[Message]
)
object ReceiveMessageSuccess {
implicit def entityDecoder[F[_]: Effect]: EntityDecoder[F, ReceiveMessageSuccess] =
ExtraEntityDecoderInstances.fromXml { elem =>
if (elem.label == "ReceiveMessageResponse")
(elem \ "ReceiveMessageResult" \ "Message").toList.traverse(Message.parse) map { messages =>
ReceiveMessageSuccess(messages)
} else None
}
}
Example 100
| Source File: SqsCommand.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.sqs
import cats.effect.Effect
import org.http4s.headers.Host
import org.http4s.{EntityDecoder, Headers, Method, Request, UrlForm}
import org.aws4s._
import org.aws4s.core.Command.Validator
import org.aws4s.core.{Command, RenderedParam, ServiceName}
private[sqs] abstract class SqsCommand[F[_]: Effect, R: EntityDecoder[F, ?]] extends Command[F, String, R] {
val q: Queue
val action: String
override final val serviceName: ServiceName = ServiceName.Sqs
override final val payloadSigning: PayloadSigning = PayloadSigning.Signed
override final val region: Region = q.region
override final val validator: Validator[String] = _ => None
override final val requestGenerator: List[RenderedParam[String]] => F[Request[F]] = { params =>
val body = params.map(p => (p.name, p.value)).foldLeft(UrlForm())((form, newPair) => form + newPair) + ("Action" -> action)
Request[F](Method.POST, q.uri, headers = Headers(Host(q.host))).withBody[UrlForm](body)
}
}
Example 101
package org.aws4s.sqs
import cats.effect.Effect
import org.aws4s.core.Service
import org.aws4s.Credentials
import org.http4s.client.Client
case class Sqs[F[_]: Effect](client: F[Client[F]], credentials: () => Credentials) extends Service[F, String] {
def sendMessage(
q: Queue,
messageBody: MessageBody,
delaySeconds: Option[DelaySeconds] = None,
messageDeduplicationId: Option[MessageDeduplicationId] = None
): F[SendMessageSuccess] = run {
SendMessage(
q,
messageBody,
delaySeconds,
messageDeduplicationId
)
}
def receiveMessage(
q: Queue,
maxNumberOfMessages: Option[MaxNumberOfMessages] = None,
visibilityTimeout: Option[VisibilityTimeout] = None,
waitTimeSeconds: Option[WaitTimeSeconds] = None,
receiveRequestAttemptId: Option[ReceiveRequestAttemptId] = None,
): F[ReceiveMessageSuccess] = run {
ReceiveMessage(
q,
maxNumberOfMessages,
visibilityTimeout,
waitTimeSeconds,
receiveRequestAttemptId
)
}
def deleteMessage(
q: Queue,
receiptHandle: ReceiptHandle,
): F[Unit] = run {
DeleteMessage(
q,
receiptHandle
)
}
}
Example 102
| Source File: SendMessage.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.sqs
import cats.effect.Effect
import org.http4s.EntityDecoder
import org.aws4s.core.XmlParsing._
import org.aws4s.core.{CommandPayload, ExtraEntityDecoderInstances, Param}
private[sqs] case class SendMessage[F[_]: Effect](
q: Queue,
messageBody: MessageBody,
delaySeconds: Option[DelaySeconds] = None,
messageDeduplicationId: Option[MessageDeduplicationId] = None,
) extends SqsCommand[F, SendMessageSuccess] {
override val action: String = "SendMessage"
override def params: List[Param[String]] =
CommandPayload.params(
messageBody
)(
delaySeconds,
messageDeduplicationId
)
}
case class SendMessageSuccess(
messageId: MessageId,
md5OfMessageBody: String,
sequenceNumber: Option[SequenceNumber]
)
object SendMessageSuccess {
implicit def entityDecoder[F[_]: Effect]: EntityDecoder[F, SendMessageSuccess] =
ExtraEntityDecoderInstances.fromXml { elem =>
if (elem.label == "SendMessageResponse")
Some(
SendMessageSuccess(
MessageId(text(elem)("SendMessageResult", "MessageId")),
text(elem)("SendMessageResult", "MD5OfMessageBody"),
integer(elem)("SendMessageResult", "SequenceNumber") map SequenceNumber
)
)
else
None
}
}
Example 103
| Source File: CreateTable.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.dynamodb
import cats.data.NonEmptyList
import cats.effect.Effect
import io.circe.{Decoder, Json}
import org.aws4s.Region
import org.aws4s.core.{Command, Param}
private[dynamodb] case class CreateTable[F[_]: Effect](
region: Region,
indices: NonEmptyList[Index],
tableName: TableName,
provisionedThroughput: ProvisionedThroughput,
) extends DynamoDbCommand[F, CreateTableSuccess] {
override def action: String = "CreateTable"
override def params: List[Param[Json]] = {
val attributeDefinitions = AttributeDefinitions(indices.map(ix => AttributeDefinition(ix.attributeName, ix.attributeType)))
val keySchema = KeySchema(indices.map(ix => KeySchemaElement(ix.attributeName, ix.keyType)))
List(attributeDefinitions, tableName, keySchema, provisionedThroughput)
}
override val validator: Command.Validator[Json] = _ => None
}
case class CreateTableSuccess(tableDescription: TableDescription)
object CreateTableSuccess {
implicit val decoder: Decoder[CreateTableSuccess] =
Decoder.forProduct1(TableDescription.name)(CreateTableSuccess.apply)
}
Example 104
| Source File: DynamoDbCommand.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.dynamodb
import cats.effect.Effect
import io.circe.{Decoder, Json}
import org.aws4s.PayloadSigning
import org.http4s.headers.{Host, `Content-Type`}
import org.http4s.{Header, Headers, MediaType, Method, Request, Uri}
import org.http4s.circe._
import cats.implicits._
import org.aws4s.core.ExtraEntityDecoderInstances._
import org.aws4s.core.{Command, CommandPayload, RenderedParam, ServiceName}
private[dynamodb] abstract class DynamoDbCommand[F[_]: Effect, R: Decoder] extends Command[F, Json, R] {
override def serviceName: ServiceName = ServiceName.DynamoDb
override def payloadSigning: PayloadSigning = PayloadSigning.Signed
def action: String
override final val requestGenerator: List[RenderedParam[Json]] => F[Request[F]] = params => {
val host = s"dynamodb.${region.name}.amazonaws.com"
val payload: Json = CommandPayload.jsonObject(params)
Request[F](
Method.POST,
Uri.unsafeFromString(s"https://$host/"),
headers = Headers(Header("X-Amz-Target", s"DynamoDB_20120810.$action"), Host(host))
).withBody(payload).map(_.withContentType(`Content-Type`.apply(MediaType.fromKey(("application", "x-amz-json-1.0")))))
}
}
Example 105
| Source File: DynamoDb.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.dynamodb
import cats.data.NonEmptyList
import cats.effect.Effect
import io.circe.Json
import org.aws4s.core.Service
import org.aws4s.{Credentials, Region}
import org.http4s.client.Client
import org.aws4s.core.ExtraEntityDecoderInstances._
case class DynamoDb[F[_]: Effect](client: F[Client[F]], region: Region, credentials: () => Credentials) extends Service[F, Json] {
def createTable(tableName: TableName, indices: NonEmptyList[Index], provisionedThroughput: ProvisionedThroughput): F[CreateTableSuccess] =
run {
CreateTable(region, indices, tableName, provisionedThroughput)
}
def deleteTable(tableName: TableName): F[DeleteTableSuccess] =
run {
DeleteTable(region, tableName)
}
}
Example 106
| Source File: S3ObjectCommand.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.s3
import cats.effect.Effect
import org.aws4s.core.{Command, Param, RenderedParam, ServiceName}
import org.http4s.{EntityDecoder, Headers, Method, Request, Uri}
import fs2._
import org.http4s.headers.Host
import cats.implicits._
private[aws4s] abstract class S3ObjectCommand[F[_]: Effect, R: EntityDecoder[F, ?]] extends Command[F, Nothing, R] {
override final val serviceName: ServiceName = ServiceName.S3
val action: Method
val bucketName: BucketName
val objectPath: ObjectPath
val payload: F[Stream[F, Byte]]
override final val params: List[Param[Nothing]] = List.empty
override final val validator: Command.Validator[Nothing] = _ => None
override final val requestGenerator: List[RenderedParam[Nothing]] => F[Request[F]] = { _ =>
val host = s"${bucketName.value}.s3.${region.name}.amazonaws.com"
val uri = Uri.unsafeFromString(s"https://$host/").withPath(objectPath.value)
for {
pStream <- payload
pBytes <- pStream.compile.toVector
r <- Request[F](action, uri, headers = Headers(Host(host))).withBody(pBytes.toArray)
} yield r
}
}
Example 107
package org.aws4s.s3
import cats.effect.Effect
import org.aws4s._
import org.http4s.client.Client
import fs2.Stream
import org.aws4s.core.ExtraEntityDecoderInstances._
import org.aws4s.core.Service
case class S3[F[_]: Effect](client: F[Client[F]], region: Region, credentials: () => Credentials) extends Service[F, Nothing] {
val listBuckets: F[ListBucketsSuccess] = run {
ListBuckets(region)
}
def putObject(
bucket: BucketName,
objectPath: ObjectPath,
obj: F[Stream[F, Byte]],
payloadSigning: PayloadSigning = PayloadSigning.Unsigned,
): F[Unit] = run {
PutObject(region, bucket, objectPath, obj, payloadSigning)
}
def deleteObject(
bucket: BucketName,
objectPath: ObjectPath
): F[Unit] = run {
DeleteObject(region, bucket, objectPath)
}
def getObject(
bucket: BucketName,
objectPath: ObjectPath
): F[Stream[F, Byte]] = run {
GetObject(region, bucket, objectPath)
}
}
Example 108
| Source File: PutObject.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.s3
import cats.effect.Effect
import org.aws4s.{PayloadSigning, _}
import org.http4s.Method
import fs2.Stream
private[aws4s] case class PutObject[F[_]: Effect](
region: Region,
bucketName: BucketName,
objectPath: ObjectPath,
obj: F[Stream[F, Byte]],
payloadSigning: PayloadSigning
) extends S3ObjectCommand[F, Unit] {
override val action: Method = Method.PUT
override val payload: F[Stream[F, Byte]] = obj
}
Example 109
| Source File: S3ServiceCommand.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.s3
import cats.effect.Effect
import cats.implicits._
import fs2._
import org.aws4s.PayloadSigning
import org.aws4s.core.{Command, Param, RenderedParam, ServiceName}
import org.http4s.headers.Host
import org.http4s.{EntityDecoder, Headers, Method, Request, Uri}
private[aws4s] abstract class S3ServiceCommand[F[_]: Effect, R: EntityDecoder[F, ?]] extends Command[F, Nothing, R] {
override final val serviceName: ServiceName = ServiceName.S3
val action: Method
val payload: F[Stream[F, Byte]]
override final val payloadSigning: PayloadSigning = PayloadSigning.Signed
override final val params: List[Param[Nothing]] = List.empty
override final val validator: Command.Validator[Nothing] = _ => None
override final val requestGenerator: List[RenderedParam[Nothing]] => F[Request[F]] = { _ =>
val host = s"s3.${region.name}.amazonaws.com"
val uri = Uri.unsafeFromString(s"https://$host/").withPath("/")
payload map { p =>
Request[F](action, uri, headers = Headers(Host(host))).withBodyStream(p)
}
}
}
Example 110
| Source File: ListBuckets.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.s3
import cats.effect.Effect
import org.aws4s._
import org.http4s.{EntityDecoder, Method}
import cats.implicits._
import fs2.Stream
import org.aws4s.core.ExtraEntityDecoderInstances
private[s3] case class ListBuckets[F[_]: Effect](
region: Region
) extends S3ServiceCommand[F, ListBucketsSuccess] {
override val action: Method = Method.GET
override val payload: F[Stream[F, Byte]] = (Stream.empty: Stream[F, Byte]).pure[F]
}
case class ListBucketsSuccess(
buckets: List[BucketName]
)
object ListBucketsSuccess {
implicit def entityDecoder[F[_]: Effect]: EntityDecoder[F, ListBucketsSuccess] =
ExtraEntityDecoderInstances.fromXml { elem =>
if (elem.label == "ListAllMyBucketsResult")
(elem \ "Buckets" \ "Bucket").toList.traverse(BucketName.parse) map { buckets =>
ListBucketsSuccess(buckets)
} else None
}
}
Example 111
| Source File: ExtraEntityDecoderInstances.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.core
import cats.Applicative
import cats.data.EitherT
import cats.effect.Effect
import cats.implicits._
import fs2.Stream
import io.circe.Decoder
import org.http4s.scalaxml._
import org.http4s.{DecodeFailure, EntityDecoder, InvalidMessageBodyFailure, MediaRange}
private[aws4s] object ExtraEntityDecoderInstances {
implicit def streamEntityDecoder[F[_]: Applicative]: EntityDecoder[F, Stream[F, Byte]] =
EntityDecoder.decodeBy(MediaRange.`*/*`) { msg =>
EitherT.fromEither(msg.body.asRight[DecodeFailure])
}
def fromXml[F[_]: Effect, A](f: scala.xml.Elem => Option[A]): EntityDecoder[F, A] =
EntityDecoder[F, scala.xml.Elem] flatMapR { elem =>
val result = f(elem)
EitherT.fromEither(result.toRight(InvalidMessageBodyFailure("Response was not as expected")))
}
implicit def circeEntityDecoder[F[_]: Effect, A: Decoder]: EntityDecoder[F, A] =
org.http4s.circe.jsonOf[F, A]
}
Example 112
| Source File: ResponseContent.scala From aws4s with MIT License | 5 votes |
|
package org.aws4s.core
import cats.data.NonEmptyList
import cats.effect.{Effect, Sync}
import io.circe.Json
import org.http4s.{EntityDecoder, MediaRange, Message}
import org.http4s.scalaxml._
import org.http4s.circe._
private[aws4s] sealed trait ResponseContent {
final def tryParse[A](pf: PartialFunction[ResponseContent, Option[A]]): Option[A] =
pf.orElse[ResponseContent, Option[A]]({ case _ => None })(this)
}
private[aws4s] case class XmlContent(elem: scala.xml.Elem) extends ResponseContent
private[aws4s] case class JsonContent(json: Json) extends ResponseContent
private[aws4s] case class StringContent(text: String) extends ResponseContent
private[aws4s] case object NoContent extends ResponseContent
private[aws4s] object ResponseContent {
implicit def entityDecoder[F[_]: Effect]: EntityDecoder[F, ResponseContent] =
EntityDecoder[F, scala.xml.Elem].map(elem => XmlContent(elem)).widen[ResponseContent] orElse
EntityDecoder[F, Json].map(json => JsonContent(json)).widen[ResponseContent] orElse
inclusiveJsonEntityDecoder.map(json => JsonContent(json)).widen[ResponseContent] orElse
EntityDecoder[F, String].map(text => StringContent(text)).widen[ResponseContent] orElse
EntityDecoder[F, Unit].map(_ => NoContent).widen[ResponseContent]
private def inclusiveJsonEntityDecoder[F[_]: Sync]: EntityDecoder[F, Json] = {
val json = jsonDecoder[F]
val extraMediaRanges = NonEmptyList.of(
"application/x-amz-json-1.0"
) map (mr => MediaRange.parse(mr).getOrElse(throw new RuntimeException(s"Invalid Media Range: $mr")))
val allMediaRanges = extraMediaRanges concat json.consumes.toList
EntityDecoder.decodeBy[F, Json](allMediaRanges.head, allMediaRanges.tail: _*)((msg: Message[F]) => json.decode(msg, strict = false))
}
}
Example 113
| Source File: Fs2OutputStream.scala From fs2-blobstore with Apache License 2.0 | 4 votes |
|
package blobstore.gcs
import java.io.OutputStream
import cats.effect.{ConcurrentEffect, Effect}
import fs2.{Chunk, Stream}
import fs2.concurrent.Queue
import cats.syntax.functor._
import scala.annotation.tailrec
private[gcs] class Fs2OutputStream[F[_]](
queue: Queue[F, Option[Chunk[Byte]]],
chunkSize: Int
)(implicit eff: Effect[F])
extends OutputStream {
@SuppressWarnings(Array("scalafix:DisableSyntax.var"))
private var bufferedChunk: Chunk[Byte] = Chunk.empty
@tailrec
private def addChunk(newChunk: Chunk[Byte]): Unit = {
val newChunkSize = newChunk.size
val bufferedChunkSize = bufferedChunk.size
val spaceLeftInTheBuffer = chunkSize - bufferedChunkSize
if (newChunkSize > spaceLeftInTheBuffer) {
bufferedChunk = Chunk.concatBytes(Seq(bufferedChunk, newChunk.take(spaceLeftInTheBuffer)), chunkSize)
flushBuffer()
addChunk(newChunk.drop(spaceLeftInTheBuffer))
} else {
bufferedChunk =
Chunk.concatBytes(Seq(bufferedChunk, newChunk), bufferedChunkSize + newChunkSize)
}
}
private def flushBuffer(): Unit = {
enqueueChunkSync(Some(bufferedChunk))
bufferedChunk = Chunk.empty
}
private def enqueueChunkSync(c: Option[Chunk[Byte]]): Unit =
eff.toIO(queue.enqueue1(c)).unsafeRunSync()
val stream: Stream[F, Byte] = queue.dequeue.unNoneTerminate.flatMap(Stream.chunk)
override def write(bytes: Array[Byte]): Unit =
addChunk(Chunk.bytes(bytes))
override def write(bytes: Array[Byte], off: Int, len: Int): Unit =
addChunk(Chunk.bytes(bytes, off, len))
override def write(b: Int): Unit =
addChunk(Chunk.singleton(b.toByte))
override def flush(): Unit = flushBuffer()
override def close(): Unit = {
flush()
enqueueChunkSync(None)
}
}
private[gcs] object Fs2OutputStream {
def apply[F[_]: ConcurrentEffect](chunkSize: Int, queueSize: Option[Int]): F[Fs2OutputStream[F]] = {
val fQueue = queueSize match {
case None => Queue.unbounded[F, Option[Chunk[Byte]]]
case Some(size) => Queue.bounded[F, Option[Chunk[Byte]]](size)
}
fQueue.map { queue => new Fs2OutputStream[F](queue, chunkSize) }
}
}
