java.net.URLEncoder Scala Examples
The following examples show how to use java.net.URLEncoder.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
Example 1
| Source File: Launcher.scala From sparkplug with MIT License | 7 votes |
|
package springnz.sparkplug.client
import java.net.{ URLEncoder, InetAddress }
import better.files._
import com.typesafe.config.{ ConfigRenderOptions, Config }
import org.apache.spark.launcher.SparkLauncher
import springnz.sparkplug.util.{ BuilderOps, ConfigUtils, Logging, Pimpers }
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future
import scala.util.{ Properties, Try }
object Launcher extends Logging {
import BuilderOps._
import Pimpers._
def startProcess(launcher: SparkLauncher): Future[Unit] = {
val processFuture = Future {
launcher.launch()
}.withErrorLog("Failed to launch: ")
processFuture.flatMap {
process ⇒ executeProcess(process)
}
}
private def executeProcess(process: Process): Future[Unit] = Future {
val outStream = scala.io.Source.fromInputStream(process.getInputStream)
for (line ← outStream.getLines()) {
log.info(line)
}
val errorStream = scala.io.Source.fromInputStream(process.getErrorStream)
for (line ← errorStream.getLines()) {
log.info(line)
}
process.waitFor()
}
def launch(clientAkkaAddress: String,
jarPath: File,
mainJarPattern: String,
mainClass: String,
sparkConfig: Config,
akkaRemoteConfig: Option[Config],
sendJars: Boolean = true): Try[Future[Unit]] = Try {
val fullExtraJarFolder = jarPath.pathAsString
val sparkHome = Properties.envOrNone("SPARK_HOME")
val sparkMaster = Properties.envOrElse("SPARK_MASTER", s"spark://${InetAddress.getLocalHost.getHostAddress}:7077")
log.debug(s"Spark master set to: $sparkMaster")
// TODO: enable this functionality (need Spark 1.5 for this)
// val sparkArgs: Array[String] = config.getString("spark.submit.sparkargs").split(' ')
if (!sparkMaster.startsWith("local[") && !sparkHome.isDefined)
throw new RuntimeException("If 'SPARK_MASTER' is not set to local, 'SPARK_HOME' must be set.")
val appName = mainClass.split('.').last
val mainJar = jarPath.glob(mainJarPattern).collectFirst { case f ⇒ f.pathAsString }
val configVars: Seq[(String, String)] = ConfigUtils.configFields(sparkConfig).toSeq
val akkaRemoteConfigString = akkaRemoteConfig.map { config ⇒
val configString = config.root().render(ConfigRenderOptions.concise())
URLEncoder.encode(configString, "UTF-8")
}
val launcher = (new SparkLauncher)
.setIfSome[String](mainJar) { (l, mj) ⇒ l.setAppResource(mj) }
.setMainClass(mainClass)
.setAppName(appName)
.setMaster(sparkMaster)
.setIfSome[String](sparkHome) { (l, sh) ⇒ l.setSparkHome(sh) }
.addAppArgs("appName", appName)
.addAppArgs("clientAkkaAddress", clientAkkaAddress)
.setIfSome(akkaRemoteConfigString) { (l, config) ⇒ l.addAppArgs("remoteAkkaConfig", config) }
.setFoldLeft(configVars) { case (launcher, (key, value)) ⇒ launcher.setConf(key, value) }
.setDeployMode(sparkConfig.getString("spark.deploymode"))
val extraJarFiles = jarPath.glob("*.jar")
.map { case f ⇒ f.pathAsString }
.filterNot(_.contains("/akka-"))
val launcherWithJars =
if (sendJars)
extraJarFiles.foldLeft(launcher) { case (l, jarFile) ⇒ l.addJar(jarFile) }
else if (extraJarFiles.length == 0) launcher
else launcher
.setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, s"$fullExtraJarFolder/*")
.setConf(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH, s"$fullExtraJarFolder/*")
startProcess(launcherWithJars)
}
}
Example 2
| Source File: package.scala From franklin with Apache License 2.0 | 6 votes |
|
package com.azavea.franklin.api
import com.azavea.franklin.api.commands.ApiConfig
import com.azavea.stac4s._
import eu.timepit.refined.types.string.NonEmptyString
import java.net.URLEncoder
import java.nio.charset.StandardCharsets
package object implicits {
implicit class combineNonEmptyString(s: NonEmptyString) {
// Combining a non-empty string should always return a non-empty string
def +(otherString: String): NonEmptyString =
NonEmptyString.unsafeFrom(s.value.concat(otherString))
}
implicit class StacItemWithCog(item: StacItem) {
def updateLinksWithHost(apiConfig: ApiConfig) = {
val updatedLinks = item.links.map(_.addServerHost(apiConfig))
val updatedAssets = item.assets.mapValues { asset =>
asset.href.startsWith("/") match {
case true => asset.copy(href = s"${apiConfig.apiHost}${asset.href}")
case _ => asset
}
}
item.copy(links = updatedLinks, assets = updatedAssets)
}
def addTilesLink(apiHost: String, collectionId: String, itemId: String) = {
val cogAsset = item.assets.values.exists { asset =>
asset._type match {
case Some(`image/cog`) => true
case _ => false
}
}
val updatedLinks = cogAsset match {
case true => {
val encodedItemId = URLEncoder.encode(itemId, StandardCharsets.UTF_8.toString)
val encodedCollectionId = URLEncoder.encode(collectionId, StandardCharsets.UTF_8.toString)
val tileLink: StacLink = StacLink(
s"$apiHost/collections/$encodedCollectionId/items/$encodedItemId/tiles",
StacLinkType.VendorLinkType("tiles"),
Some(`application/json`),
Some("Tile URLs for Item")
)
tileLink :: item.links
}
case _ => item.links
}
(item.copy(links = updatedLinks))
}
}
implicit class UpdatedStacLink(link: StacLink) {
def addServerHost(apiConfig: ApiConfig) = {
link.href.startsWith("/") match {
case true => link.copy(href = s"${apiConfig.apiHost}${link.href}")
case _ => link
}
}
}
implicit class StacCollectionWithTiles(collection: StacCollection) {
def addTilesLink(apiHost: String): StacCollection = {
val encodedCollectionId = URLEncoder.encode(collection.id, StandardCharsets.UTF_8.toString)
val tileLink = StacLink(
s"$apiHost/collections/$encodedCollectionId/tiles",
StacLinkType.VendorLinkType("tiles"),
Some(`application/json`),
Some("Tile URLs for Collection")
)
collection.copy(links = tileLink :: collection.links)
}
def maybeAddTilesLink(enableTiles: Boolean, apiHost: String) =
if (enableTiles) addTilesLink(apiHost) else collection
def updateLinksWithHost(apiConfig: ApiConfig) = {
val updatedLinks = collection.links.map(_.addServerHost(apiConfig))
collection.copy(links = updatedLinks)
}
}
}
Example 3
| Source File: OAuth.scala From iot-demo with Apache License 2.0 | 5 votes |
|
package core
import java.net.URLEncoder
import java.nio.charset.Charset
import javax.crypto
import org.parboiled.common.Base64
import org.slf4j.LoggerFactory
import spray.http.HttpHeaders.RawHeader
import spray.http.{ContentType, HttpEntity, HttpRequest, MediaTypes}
import scala.collection.immutable.TreeMap
object OAuth {
private[this] val log = LoggerFactory.getLogger(getClass)
def oAuthAuthorizer(consumer: Consumer, token: Token): HttpRequest => HttpRequest = {
// construct the key and cryptographic entity
val SHA1 = "HmacSHA1"
val keyString = percentEncode(consumer.secret :: token.secret :: Nil)
val key = new crypto.spec.SecretKeySpec(bytes(keyString), SHA1)
val mac = crypto.Mac.getInstance(SHA1)
{ httpRequest: HttpRequest =>
val timestamp = (System.currentTimeMillis / 1000).toString
// nonce is unique enough for our purposes here
val nonce = System.nanoTime.toString
// pick out x-www-form-urlencoded body
val (requestParams, newEntity) = httpRequest.entity match {
case [email protected](ContentType(MediaTypes.`application/x-www-form-urlencoded`, _), data) =>
log.info("request {}", request)
val params = data.asString.split("&")
val pairs = params.map { param =>
val p = param.split("=")
p(0) -> percentEncode(p(1))
}
(pairs.toMap, HttpEntity(ContentType(MediaTypes.`application/x-www-form-urlencoded`), "%s=%s" format(pairs(0)._1, pairs(0)._2)))
case e => (Map(), e)
}
// prepare the OAuth parameters
val oauthParams = Map(
"oauth_consumer_key" -> consumer.key,
"oauth_signature_method" -> "HMAC-SHA1",
"oauth_timestamp" -> timestamp,
"oauth_nonce" -> nonce,
"oauth_token" -> token.value,
"oauth_version" -> "1.0"
)
// construct parts of the signature base string
val encodedOrderedParams = (TreeMap[String, String]() ++ oauthParams ++ requestParams) map { case (k, v) => k + "=" + v } mkString "&"
val url = httpRequest.uri.toString()
// construct the signature base string
val signatureBaseString = percentEncode(httpRequest.method.toString() :: url :: encodedOrderedParams :: Nil)
mac.init(key)
val sig = Base64.rfc2045().encodeToString(mac.doFinal(bytes(signatureBaseString)), false)
mac.reset()
val oauth = TreeMap[String, String]() ++ (oauthParams + ("oauth_signature" -> percentEncode(sig))) map { case (k, v) => "%s=\"%s\"" format(k, v) } mkString ", "
// return the signed request
httpRequest.withHeaders(List(RawHeader("Authorization", "OAuth " + oauth))).withEntity(newEntity)
}
}
private def percentEncode(str: String): String = URLEncoder.encode(str, "UTF-8") replace("+", "%20") replace("%7E", "~")
private def percentEncode(s: Seq[String]): String = s map percentEncode mkString "&"
private def bytes(str: String) = str.getBytes(Charset.forName("UTF-8"))
case class Consumer(key: String, secret: String)
case class Token(value: String, secret: String)
}
Example 4
| Source File: PoolTable.scala From BigDatalog with Apache License 2.0 | 5 votes |
|
package org.apache.spark.ui.jobs
import java.net.URLEncoder
import scala.collection.mutable.HashMap
import scala.xml.Node
import org.apache.spark.scheduler.{Schedulable, StageInfo}
import org.apache.spark.ui.UIUtils
private[ui] class PoolTable(pools: Seq[Schedulable], parent: StagesTab) {
private val listener = parent.progressListener
def toNodeSeq: Seq[Node] = {
listener.synchronized {
poolTable(poolRow, pools)
}
}
private def poolTable(
makeRow: (Schedulable, HashMap[String, HashMap[Int, StageInfo]]) => Seq[Node],
rows: Seq[Schedulable]): Seq[Node] = {
<table class="table table-bordered table-striped table-condensed sortable table-fixed">
<thead>
<th>Pool Name</th>
<th>Minimum Share</th>
<th>Pool Weight</th>
<th>Active Stages</th>
<th>Running Tasks</th>
<th>SchedulingMode</th>
</thead>
<tbody>
{rows.map(r => makeRow(r, listener.poolToActiveStages))}
</tbody>
</table>
}
private def poolRow(
p: Schedulable,
poolToActiveStages: HashMap[String, HashMap[Int, StageInfo]]): Seq[Node] = {
val activeStages = poolToActiveStages.get(p.name) match {
case Some(stages) => stages.size
case None => 0
}
val href = "%s/stages/pool?poolname=%s"
.format(UIUtils.prependBaseUri(parent.basePath), URLEncoder.encode(p.name, "UTF-8"))
<tr>
<td>
<a href={href}>{p.name}</a>
</td>
<td>{p.minShare}</td>
<td>{p.weight}</td>
<td>{activeStages}</td>
<td>{p.runningTasks}</td>
<td>{p.schedulingMode}</td>
</tr>
}
}
Example 5
| Source File: SnapshotRemoteImpl.scala From c4proto with Apache License 2.0 | 5 votes |
|
package ee.cone.c4actor
import java.net.{URLDecoder, URLEncoder}
import java.nio.file.{Files, Paths}
import java.nio.charset.StandardCharsets.UTF_8
import ee.cone.c4di.c4
@c4("ConfigSimpleSignerApp") final class SimpleSignerImpl(
config: Config, idGenUtil : IdGenUtil
)(
fileName: String = config.get("C4AUTH_KEY_FILE")
)(
val salt: String = new String(Files.readAllBytes(Paths.get(fileName)),UTF_8)
) extends SimpleSigner {
def sign(data: List[String], until: Long): String = {
val uData = until.toString :: data
val hash = idGenUtil.srcIdFromStrings(salt :: uData:_*)
(hash :: uData).map(URLEncoder.encode(_,"UTF-8")).mkString("=")
}
def retrieve(check: Boolean): Option[String]=>Option[List[String]] = _.flatMap{ signed =>
val hash :: untilStr :: data = signed.split("=").map(URLDecoder.decode(_,"UTF-8")).toList
val until = untilStr.toLong
if(!check) Option(data)
else if(until < System.currentTimeMillis) None
else if(sign(data,until) == signed) Option(data)
else None
}
}
@c4("TaskSignerApp") final class SnapshotTaskSignerImpl(inner: SimpleSigner)(
val url: String = "/need-snapshot"
) extends SnapshotTaskSigner {
def sign(task: SnapshotTask, until: Long): String = inner.sign(List(url,task.name) ++ task.offsetOpt, until)
def retrieve(check: Boolean): Option[String]=>Option[SnapshotTask] =
signed => inner.retrieve(check)(signed) match {
case Some(Seq(`url`,"next")) => Option(NextSnapshotTask(None))
case Some(Seq(`url`,"next", offset)) => Option(NextSnapshotTask(Option(offset)))
case Some(Seq(`url`,"debug", offset)) => Option(DebugSnapshotTask(offset))
case _ => None
}
}
Example 6
| Source File: PoolTable.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.ui.jobs
import java.net.URLEncoder
import scala.xml.Node
import org.apache.spark.scheduler.Schedulable
import org.apache.spark.status.PoolData
import org.apache.spark.ui.UIUtils
private[ui] class PoolTable(pools: Map[Schedulable, PoolData], parent: StagesTab) {
def toNodeSeq: Seq[Node] = {
<table class="table table-bordered table-striped table-condensed sortable table-fixed">
<thead>
<th>Pool Name</th>
<th>Minimum Share</th>
<th>Pool Weight</th>
<th>Active Stages</th>
<th>Running Tasks</th>
<th>SchedulingMode</th>
</thead>
<tbody>
{pools.map { case (s, p) => poolRow(s, p) }}
</tbody>
</table>
}
private def poolRow(s: Schedulable, p: PoolData): Seq[Node] = {
val activeStages = p.stageIds.size
val href = "%s/stages/pool?poolname=%s"
.format(UIUtils.prependBaseUri(parent.basePath), URLEncoder.encode(p.name, "UTF-8"))
<tr>
<td>
<a href={href}>{p.name}</a>
</td>
<td>{s.minShare}</td>
<td>{s.weight}</td>
<td>{activeStages}</td>
<td>{s.runningTasks}</td>
<td>{s.schedulingMode}</td>
</tr>
}
}
Example 7
| Source File: DistributedProcessing.scala From aecor with MIT License | 5 votes |
|
package aecor.distributedprocessing
import java.net.URLEncoder
import java.nio.charset.StandardCharsets
import aecor.distributedprocessing.DistributedProcessing.{ KillSwitch, Process }
import aecor.distributedprocessing.DistributedProcessingWorker.KeepRunning
import aecor.util.effect._
import akka.actor.ActorSystem
import akka.cluster.sharding.{ ClusterSharding, ClusterShardingSettings }
import akka.pattern.{ BackoffOpts, BackoffSupervisor, ask }
import akka.util.Timeout
import cats.effect.Effect
import cats.implicits._
import scala.concurrent.duration.{ FiniteDuration, _ }
final class DistributedProcessing private (system: ActorSystem) {
def start[F[_]: Effect](name: String,
processes: List[Process[F]],
settings: DistributedProcessingSettings =
DistributedProcessingSettings.default(system)): F[KillSwitch[F]] =
Effect[F].delay {
val opts = BackoffOpts
.onFailure(
DistributedProcessingWorker.props(processes, name),
"worker",
settings.minBackoff,
settings.maxBackoff,
settings.randomFactor
)
val props = BackoffSupervisor.props(opts)
val region = ClusterSharding(system).start(
typeName = name,
entityProps = props,
settings = settings.clusterShardingSettings,
extractEntityId = {
case c @ KeepRunning(workerId) => (workerId.toString, c)
},
extractShardId = {
case KeepRunning(workerId) => (workerId % settings.numberOfShards).toString
case other => throw new IllegalArgumentException(s"Unexpected message [$other]")
}
)
val regionSupervisor = system.actorOf(
DistributedProcessingSupervisor
.props(processes.size, region, settings.heartbeatInterval),
"DistributedProcessingSupervisor-" + URLEncoder
.encode(name, StandardCharsets.UTF_8.name())
)
implicit val timeout = Timeout(settings.shutdownTimeout)
KillSwitch {
Effect[F].fromFuture {
regionSupervisor ? DistributedProcessingSupervisor.GracefulShutdown
}.void
}
}
}
object DistributedProcessing {
def apply(system: ActorSystem): DistributedProcessing = new DistributedProcessing(system)
final case class KillSwitch[F[_]](shutdown: F[Unit]) extends AnyVal
final case class RunningProcess[F[_]](watchTermination: F[Unit], shutdown: F[Unit])
final case class Process[F[_]](run: F[RunningProcess[F]]) extends AnyVal
}
final case class DistributedProcessingSettings(minBackoff: FiniteDuration,
maxBackoff: FiniteDuration,
randomFactor: Double,
shutdownTimeout: FiniteDuration,
numberOfShards: Int,
heartbeatInterval: FiniteDuration,
clusterShardingSettings: ClusterShardingSettings)
object DistributedProcessingSettings {
def default(clusterShardingSettings: ClusterShardingSettings): DistributedProcessingSettings =
DistributedProcessingSettings(
minBackoff = 3.seconds,
maxBackoff = 10.seconds,
randomFactor = 0.2,
shutdownTimeout = 10.seconds,
numberOfShards = 100,
heartbeatInterval = 2.seconds,
clusterShardingSettings = clusterShardingSettings
)
def default(system: ActorSystem): DistributedProcessingSettings =
default(ClusterShardingSettings(system))
}
Example 8
| Source File: RealmDirectivesSpec.scala From nexus-iam with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.iam.directives
import java.net.URLEncoder
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server.Route
import akka.http.scaladsl.testkit.ScalatestRouteTest
import ch.epfl.bluebrain.nexus.commons.http.JsonLdCirceSupport._
import ch.epfl.bluebrain.nexus.commons.test.EitherValues
import ch.epfl.bluebrain.nexus.iam.directives.RealmDirectives._
import ch.epfl.bluebrain.nexus.iam.routes.SearchParams
import ch.epfl.bluebrain.nexus.rdf.Iri.AbsoluteIri
import ch.epfl.bluebrain.nexus.rdf.implicits._
import io.circe.generic.auto._
import org.mockito.IdiomaticMockito
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpecLike
class RealmDirectivesSpec
extends AnyWordSpecLike
with ScalatestRouteTest
with Matchers
with ScalaFutures
with EitherValues
with IdiomaticMockito {
private def route: Route = {
(get & searchParams) { params =>
complete(params)
}
}
private def encode(s: AbsoluteIri) = URLEncoder.encode(s.asString, "UTF-8")
"Realm directives" should {
"return query params" in {
val createdBy: AbsoluteIri = url"http://example.com/created"
val updatedBy: AbsoluteIri = url"http://example.com/updated"
val tpe: AbsoluteIri = url"http://example.com/tpe"
val tpe2: AbsoluteIri = url"http://example.com/tpe2"
Get(
s"/?rev=2&deprecated=true&type=${encode(tpe)}&type=${encode(tpe2)}&createdBy=${encode(createdBy)}&updatedBy=${encode(updatedBy)}"
) ~> route ~> check {
responseAs[SearchParams] shouldEqual SearchParams(
Some(true),
Some(2L),
Some(createdBy),
Some(updatedBy),
Set(tpe, tpe2)
)
}
}
}
}
Example 9
| Source File: RExecutor.scala From seahorse with Apache License 2.0 | 5 votes |
|
package ai.deepsense.workflowexecutor.executor
import java.net.URLEncoder
import scala.sys.process._
import ai.deepsense.commons.utils.Logging
import ai.deepsense.deeplang.CustomCodeExecutor
import ai.deepsense.workflowexecutor.customcode.CustomCodeEntryPoint
case class RExecutor(rBackendPort: Int,
entryPointId: String,
customCodeEntryPoint: CustomCodeEntryPoint,
rExecutorScript: String)
extends CustomCodeExecutor with Logging {
def isValid(code: String): Boolean = true
val RExecutable = "Rscript"
def run(workflowId: String, nodeId: String, code: String): Unit = {
val command = s"""$RExecutable $rExecutorScript """ +
s""" $rBackendPort """ +
s""" $workflowId """ +
s""" $nodeId """ +
s""" $entryPointId """ +
s""" ${URLEncoder.encode(code, "UTF-8").replace("+", "%20")} """
logger.info(s"Starting a new RExecutor process: $command")
val log = new StringBuffer()
def logMethod(logmethod: String => Unit)(s: String) : Unit = {
log.append(s)
logmethod(s)
}
val rLogger = ProcessLogger(fout = logMethod(logger.debug), ferr = logMethod(logger.error))
val errorCode = command ! rLogger
// if RScript is successful then r_executor.R should take care of setting execution status
if (errorCode != 0) {
customCodeEntryPoint.executionFailed(workflowId, nodeId, log.toString())
}
}
}
Example 10
| Source File: PoolTable.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.ui.jobs
import java.net.URLEncoder
import scala.collection.mutable.HashMap
import scala.xml.Node
import org.apache.spark.scheduler.{Schedulable, StageInfo}
import org.apache.spark.ui.UIUtils
private[ui] class PoolTable(pools: Seq[Schedulable], parent: StagesTab) {
private val listener = parent.progressListener
def toNodeSeq: Seq[Node] = {
listener.synchronized {
poolTable(poolRow, pools)
}
}
private def poolTable(
makeRow: (Schedulable, HashMap[String, HashMap[Int, StageInfo]]) => Seq[Node],
rows: Seq[Schedulable]): Seq[Node] = {
<table class="table table-bordered table-striped table-condensed sortable table-fixed">
<thead>
<th>Pool Name</th>
<th>Minimum Share</th>
<th>Pool Weight</th>
<th>Active Stages</th>
<th>Running Tasks</th>
<th>SchedulingMode</th>
</thead>
<tbody>
{rows.map(r => makeRow(r, listener.poolToActiveStages))}
</tbody>
</table>
}
private def poolRow(
p: Schedulable,
poolToActiveStages: HashMap[String, HashMap[Int, StageInfo]]): Seq[Node] = {
val activeStages = poolToActiveStages.get(p.name) match {
case Some(stages) => stages.size
case None => 0
}
val href = "%s/stages/pool?poolname=%s"
.format(UIUtils.prependBaseUri(parent.basePath, sparkUser = parent.sparkUser), URLEncoder.encode(p.name, "UTF-8"))
<tr>
<td>
<a href={href}>{p.name}</a>
</td>
<td>{p.minShare}</td>
<td>{p.weight}</td>
<td>{activeStages}</td>
<td>{p.runningTasks}</td>
<td>{p.schedulingMode}</td>
</tr>
}
}
Example 11
| Source File: SparqlUtil.scala From CM-Well with Apache License 2.0 | 5 votes |
|
package cmwell.tools.neptune.export
import java.io.ByteArrayOutputStream
import java.net.URLEncoder
import org.apache.jena.graph.Graph
import org.apache.jena.riot.{Lang, RDFDataMgr}
object SparqlUtil {
def extractSubjectFromTriple(triple: String):String = {
triple.split(" ")(0)
}
def getTriplesOfSubGraph(subGraph:Graph):String = {
val tempOs = new ByteArrayOutputStream
RDFDataMgr.write(tempOs, subGraph, Lang.NTRIPLES)
new String(tempOs.toByteArray, "UTF-8")
}
def generateSparqlCmdForDefaultGraph(triplesPerGraph: Iterable[SubjectGraphTriple] ):String = {
triplesPerGraph.map(subGraphTriple => encode(subGraphTriple.triple)).mkString
}
def generateSparqlCmdForNamedGraph(graph:String, triplesPerGraph: Iterable[SubjectGraphTriple] ):String = {
" GRAPH <" + encode(graph) + "> { " + triplesPerGraph.map(trio => encode(trio.triple)).mkString + "}"
}
def buildGroupedSparqlCmd(subjects: Iterable[String], allSubjGraphTriples: Iterable[List[SubjectGraphTriple]], updateMode: Boolean): String = {
var sparqlCmd = "update="
val deleteSubj = if (updateMode) Some("DELETE { ?s ?p ?o . } WHERE { VALUES ?s { " + subjects.map(subject => encode(subject) + " ").mkString + "} ?s ?p ?o };") else None
val insertDefaultGraphSparqlCmd = "INSERT DATA {" + allSubjGraphTriples.flatten.filterNot(trio => predicateContainsMeta(trio)).groupBy(trio => trio.graph).map(graphWithTriples => graphWithTriples._1.fold(generateSparqlCmdForDefaultGraph(graphWithTriples._2))(graph => "")).mkString + "}"
val insertNamedGraphSparqlCmd = "INSERT DATA {" + allSubjGraphTriples.flatten.filterNot(trio => predicateContainsMeta(trio)).groupBy(trio => trio.graph).map(graphWithTriples => graphWithTriples._1.fold("")(graphName => generateSparqlCmdForNamedGraph(graphName, graphWithTriples._2))).mkString + "}"
sparqlCmd + deleteSubj.getOrElse("") + insertDefaultGraphSparqlCmd + ";" + insertNamedGraphSparqlCmd
}
def encode(str: String):String = {
URLEncoder.encode(str, "UTF-8")
}
def predicateContainsMeta(trio: SubjectGraphTriple): Boolean = {
trio.triple.contains("meta/sys")
}
}
Example 12
| Source File: CmWellConsumeHandler.scala From CM-Well with Apache License 2.0 | 5 votes |
|
package cmwell.tools.neptune.export
import java.net.{URL, URLDecoder, URLEncoder}
import java.time.Instant
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.DefaultHttpClient
import org.apache.http.util.EntityUtils
import org.slf4j.LoggerFactory
object CmWellConsumeHandler {
protected lazy val logger = LoggerFactory.getLogger("cm_well_consumer")
val maxRetry = 5
private val sleepTimeout = 10000
def bulkConsume(cluster: String, position: String, format: String, updateMode:Boolean, retryCount:Int= 0): CloseableHttpResponse = {
val withMeta = if(updateMode) "&with-meta" else ""
val url = "http://" + cluster + "/_bulk-consume?position=" + position + "&format=" + format + withMeta
val client = new DefaultHttpClient
client.setHttpRequestRetryHandler(new CustomHttpClientRetryHandler())
val get = new HttpGet(url)
logger.info("Going to bulk consume,url= " + url)
val response = client.execute(get)
val statusCode = response.getStatusLine.getStatusCode
if (statusCode != 200 && statusCode != 204) {
if(statusCode == 503) {
logger.error("Failed to bulk consume, error status code=" + statusCode + "response entity=" + EntityUtils.toString(response.getEntity) + ".Going to retry...")
Thread.sleep(sleepTimeout)
bulkConsume(cluster, position, format, updateMode)
}
else{
if (retryCount < maxRetry) {
logger.error("Failed to bulk consume, error status code=" + statusCode + "response entity=" + EntityUtils.toString(response.getEntity) + ".Going to retry...,retry count=" + retryCount)
Thread.sleep(sleepTimeout)
bulkConsume(cluster, position, format, updateMode, retryCount + 1)
} else {
throw new Throwable("Failed to consume from cm-well, error code status=" + statusCode + ", response entity=" + EntityUtils.toString(response.getEntity))
}
}
}
response
}
def retrivePositionFromCreateConsumer(cluster: String, lengthHint: Int, qp: Option[String], updateMode:Boolean, automaticUpdateMode:Boolean, toolStartTime:Instant, retryCount:Int = 0): String = {
val withDeletedParam = if(updateMode || automaticUpdateMode) "&with-deleted" else ""
//initial mode
val qpTillStartTime = if(!updateMode && !automaticUpdateMode) URLEncoder.encode(",system.lastModified<") + toolStartTime.toString else ""
//automatic update mode
val qpAfterStartTime = if(!updateMode && automaticUpdateMode) URLEncoder.encode(",system.lastModified>>" )+ toolStartTime.toString else ""
val createConsumerUrl = "http://" + cluster + "/?op=create-consumer&qp=-system.parent.parent_hierarchy:/meta/" + qp.getOrElse("") + qpTillStartTime + qpAfterStartTime + "&recursive&length-hint=" + lengthHint + withDeletedParam
logger.info("create-consumer-url=" + createConsumerUrl)
val get = new HttpGet(createConsumerUrl)
val client = new DefaultHttpClient
client.setHttpRequestRetryHandler(new CustomHttpClientRetryHandler())
val response = client.execute(get)
val res = response.getAllHeaders.find(_.getName == "X-CM-WELL-POSITION").map(_.getValue).getOrElse("")
logger.info("create-Consumer http status=" + response.getStatusLine.getStatusCode)
val statusCode = response.getStatusLine.getStatusCode
if (statusCode != 200) {
if(statusCode == 503){
logger.error("Failed to retrieve position via create-consumer api,error status code=" + statusCode + ", response entity=" + EntityUtils.toString(response.getEntity) + ".Going to retry...")
Thread.sleep(sleepTimeout)
retrivePositionFromCreateConsumer(cluster, lengthHint, qp, updateMode, automaticUpdateMode, toolStartTime)
}else {
if (retryCount < maxRetry) {
logger.error("Failed to retrieve position via create-consumer api,error status code=" + statusCode + ", response entity=" + EntityUtils.toString(response.getEntity) + ".Going to retry..., retry count=" + retryCount)
Thread.sleep(sleepTimeout)
retrivePositionFromCreateConsumer(cluster, lengthHint, qp, updateMode, automaticUpdateMode, toolStartTime, retryCount+1)
}
else {
throw new Throwable("Failed to consume from cm-well, error code status=" + statusCode + ", response entity=" + EntityUtils.toString(response.getEntity))
}
}
}
res
}
}
Example 13
| Source File: QueryStringBindablesSpec.scala From gospeak with Apache License 2.0 | 5 votes |
|
package gospeak.web.utils
import java.net.URLEncoder
import java.time.{LocalDate, LocalDateTime}
import gospeak.core.domain.UserRequest
import gospeak.libs.scala.domain.{Page, Url}
import gospeak.web.testingutils.BaseSpec
import gospeak.web.utils.QueryStringBindables._
class QueryStringBindablesSpec extends BaseSpec {
describe("QueryStringBindables") {
describe("LocalDateTime") {
it("should parse and format dates") {
val ldt = LocalDateTime.of(2019, 9, 21, 19, 12)
val date = "21/09/2019"
val dateTime = s"$date 19:12"
val dateEncoded = URLEncoder.encode(date, "UTF-8")
val dateTimeEncoded = URLEncoder.encode(dateTime, "UTF-8")
LocalDateTime.parse(dateTime, dtf1) shouldBe ldt
LocalDateTime.parse(dateTimeEncoded, dtf2) shouldBe ldt
LocalDate.parse(date, df1).atTime(19, 12) shouldBe ldt
LocalDate.parse(dateEncoded, df2).atTime(19, 12) shouldBe ldt
ldt.format(df1) shouldBe date
}
it("should bind & unbind a LocalDateTime when no params") {
val ldt = LocalDateTime.of(2019, 9, 21, 0, 0)
val date = "21/09/2019"
val dateTimeEncoded = URLEncoder.encode(date + " 00:00", "UTF-8")
localDateTimeQueryStringBindable.bind("key", Map("key" -> Seq(date))) shouldBe Some(Right(ldt))
localDateTimeQueryStringBindable.unbind("key", ldt) shouldBe s"key=$dateTimeEncoded"
}
}
describe("Page.Params") {
it("should bind & unbind a Page.Params when no params") {
val params = Page.Params()
pageParamsQueryStringBindable.bind("", Map()) shouldBe Some(Right(params))
pageParamsQueryStringBindable.unbind("", params) shouldBe ""
}
it("should bind & unbind a Page.Params when all params") {
val params = buildParams(2, 30, "test", "name")
pageParamsQueryStringBindable.bind("", Map(
Page.No.key -> Seq("2"),
Page.Size.key -> Seq("30"),
Page.Search.key -> Seq("test"),
Page.OrderBy.key -> Seq("name"))) shouldBe Some(Right(params))
pageParamsQueryStringBindable.unbind("", params) shouldBe s"${Page.No.key}=2&${Page.Size.key}=30&${Page.Search.key}=test&${Page.OrderBy.key}=name"
}
it("should bind & unbind filters") {
val params = Page.Params.defaults.toggleFilter("f1").withFilter("f2", "v2")
pageParamsQueryStringBindable.bind("", Map(
"f1" -> Seq("true"),
"f2" -> Seq("v2"))) shouldBe Some(Right(params))
pageParamsQueryStringBindable.unbind("", params) shouldBe s"f1=true&f2=v2"
}
}
it("should bind & unbind a Url") {
val url = Url.from("http://youtube.com").right.get
urlQueryStringBindable.bind("key", Map("key" -> Seq("http://youtube.com"))) shouldBe Some(Right(url))
urlQueryStringBindable.unbind("key", url) shouldBe s"key=http%3A%2F%2Fyoutube.com"
}
it("should bind & unbind a UserRequest.Id") {
val id = UserRequest.Id.generate()
userRequestIdQueryStringBindable.bind("key", Map("key" -> Seq(id.value))) shouldBe Some(Right(id))
userRequestIdQueryStringBindable.unbind("key", id) shouldBe s"key=${id.value}"
}
}
private def buildParams(no: Int, size: Int, search: String, order: String) =
Page.Params(Page.No(no), Page.Size(size), Some(Page.Search(search)), Some(Page.OrderBy(order)))
}
Example 14
| Source File: WolframServiceImpl.scala From lagom-on-kube with Apache License 2.0 | 5 votes |
|
package me.alexray.wolfram.impl
import java.net.URLEncoder
import akka.NotUsed
import akka.actor.ActorSystem
import akka.http.scaladsl.Http
import akka.http.scaladsl.model.{HttpRequest, Uri}
import akka.http.scaladsl.unmarshalling.Unmarshal
import akka.stream.Materializer
import akka.util.ByteString
import com.lightbend.lagom.scaladsl.api.ServiceCall
import me.alexray.wolfram.api.WolframService
import play.api.Configuration
import scala.concurrent.{ExecutionContext, Future}
class WolframServiceImpl(config: Configuration)
(implicit system: ActorSystem, mat: Materializer, ec: ExecutionContext)
extends WolframService
{
val appID = config.underlying.getString("wolfram.appid")
val apiUrl = s"http://api.wolframalpha.com/v2/"
override def query(q: String): ServiceCall[NotUsed, String] = ServiceCall { _ =>
val url = apiUrl + s"query?appid=$appID&input=" + URLEncoder.encode(q, "UTF-8")
for {
response <- Http().singleRequest(HttpRequest(uri = Uri(url)))
if response.status.isSuccess()
data <- Unmarshal(response).to[String]
} yield data
}
override def simple(q: String): ServiceCall[NotUsed, Array[Byte]] = ServiceCall { _ =>
println(s"quetions = '$q'")
val url = apiUrl + s"simple?appid=$appID&input=" + URLEncoder.encode(q, "UTF-8").replace("+", "%20")
println(s"url = '$url'")
for {
response <- Http().singleRequest(HttpRequest(uri = Uri(url)))
if response.status.isSuccess()
bytes <- Unmarshal(response).to[ByteString]
} yield {
println(s"received image ${bytes.size} bytes long")
bytes.toArray
}
}
}
Example 15
| Source File: httpserverplugin_staticfile.scala From scalabpe with Apache License 2.0 | 5 votes |
|
package scalabpe.plugin.http
import java.io.File
import java.net.URLEncoder
import java.text.SimpleDateFormat
import java.util.Calendar
import java.util.GregorianCalendar
import java.util.Locale
import java.util.TimeZone
import scala.collection.mutable.HashMap
import org.jboss.netty.handler.codec.http.HttpHeaders
import scalabpe.core.HashMapStringAny
class StaticFilePlugin extends HttpServerPlugin with HttpServerStaticFilePlugin {
val ETAG_TAG = "etag"
val EXPIRE_TAG = "expire"
val ATTACHMENT = "attachment"
val FILENAME = "filename"
val HTTP_DATE_FORMAT = "EEE, dd MMM yyyy HH:mm:ss zzz";
val HTTP_DATE_GMT_TIMEZONE = "GMT";
val df_tl = new ThreadLocal[SimpleDateFormat]() {
override def initialValue(): SimpleDateFormat = {
val df = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US)
df.setTimeZone(TimeZone.getTimeZone(HTTP_DATE_GMT_TIMEZONE));
df
}
}
def generateStaticFile(serviceId: Int, msgId: Int, errorCode: Int, errorMessage: String, body: HashMapStringAny, pluginParam: String, headers: HashMap[String, String]): String = {
if (body.ns(FILENAME) == "") {
return null
}
val filename = body.ns(FILENAME)
if (!new File(filename).exists()) {
return null
}
if (body.ns(ETAG_TAG) != "") {
headers.put("ETag", body.ns(ETAG_TAG))
}
if (body.ns(EXPIRE_TAG) != "") {
body.i(EXPIRE_TAG) match {
case 0 | -1 =>
headers.put(HttpHeaders.Names.CACHE_CONTROL, "no-cache")
case n => // seconds
val time = new GregorianCalendar();
time.add(Calendar.SECOND, n);
headers.put(HttpHeaders.Names.EXPIRES, df_tl.get.format(time.getTime()));
headers.put(HttpHeaders.Names.CACHE_CONTROL, "max-age=" + n);
}
}
val ext = parseExt(filename)
if (ext != "")
body.put("__file_ext__", ext)
if (body.ns(ATTACHMENT, "1") == "1") {
val filename = body.ns(FILENAME)
val v = "attachment; filename=\"%s\"".format(URLEncoder.encode(parseFilename(filename), "UTF-8"))
headers.put("Content-Disposition", v)
}
filename
}
def parseFilename(name: String): String = {
val p = name.lastIndexOf("/")
if (p < 0) return name
name.substring(p + 1)
}
def parseExt(name: String): String = {
val p = name.lastIndexOf(".")
if (p < 0) return ""
name.substring(p + 1).toLowerCase()
}
}
Example 16
| Source File: PctString.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.rdf
import java.net.URLEncoder
import java.nio.charset.StandardCharsets.UTF_8
import org.parboiled2.CharPredicate
object PctString {
private val UTF8 = UTF_8.displayName()
private[rdf] def pctEncodedIgnore(s: String, toIgnore: CharPredicate): String = {
val (_, encoded) = s.foldLeft((0, new StringBuilder())) {
case ((forceIgnore, sb), b) if forceIgnore > 0 => (forceIgnore - 1, sb.append(b))
case ((_, sb), b) if toIgnore.matchesAny(b.toString) => (0, sb.append(b))
case ((_, sb), b) if b == '%' => (2, sb.append(b))
case ((_, sb), b) => (0, pctEncode(sb, b))
}
encoded.toString()
}
private def pctEncode(sb: StringBuilder, char: Char): StringBuilder =
if (char == ' ') sb.append("%20")
else sb.append(URLEncoder.encode(char.toString, UTF8))
implicit final class StringEncodingOpts(private val value: String) extends AnyVal {
def pctEncodeIgnore(toIgnore: CharPredicate): String = pctEncodedIgnore(value, toIgnore)
}
}
Example 17
| Source File: RealmDirectivesSpec.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.iam.directives
import java.net.URLEncoder
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server.Route
import akka.http.scaladsl.testkit.ScalatestRouteTest
import ch.epfl.bluebrain.nexus.commons.http.JsonLdCirceSupport._
import ch.epfl.bluebrain.nexus.iam.directives.RealmDirectives._
import ch.epfl.bluebrain.nexus.iam.routes.SearchParams
import ch.epfl.bluebrain.nexus.rdf.Iri.AbsoluteIri
import ch.epfl.bluebrain.nexus.rdf.implicits._
import ch.epfl.bluebrain.nexus.util.EitherValues
import io.circe.generic.auto._
import org.mockito.IdiomaticMockito
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpecLike
class RealmDirectivesSpec
extends AnyWordSpecLike
with ScalatestRouteTest
with Matchers
with ScalaFutures
with EitherValues
with IdiomaticMockito {
private def route: Route = {
(get & searchParams) { params => complete(params) }
}
private def encode(s: AbsoluteIri) = URLEncoder.encode(s.asString, "UTF-8")
"Realm directives" should {
"return query params" in {
val createdBy: AbsoluteIri = url"http://example.com/created"
val updatedBy: AbsoluteIri = url"http://example.com/updated"
val tpe: AbsoluteIri = url"http://example.com/tpe"
val tpe2: AbsoluteIri = url"http://example.com/tpe2"
Get(
s"/?rev=2&deprecated=true&type=${encode(tpe)}&type=${encode(tpe2)}&createdBy=${encode(createdBy)}&updatedBy=${encode(updatedBy)}"
) ~> route ~> check {
responseAs[SearchParams] shouldEqual SearchParams(
Some(true),
Some(2L),
Some(createdBy),
Some(updatedBy),
Set(tpe, tpe2)
)
}
}
}
}
Example 18
| Source File: QueryDirectivesSpec.scala From nexus with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus.admin.directives
import java.net.URLEncoder
import java.util.UUID
import akka.http.scaladsl.model.StatusCodes
import akka.http.scaladsl.server.Directives.{complete, get}
import akka.http.scaladsl.server.Route
import akka.http.scaladsl.testkit.ScalatestRouteTest
import ch.epfl.bluebrain.nexus.admin.routes.SearchParams
import ch.epfl.bluebrain.nexus.admin.routes.SearchParams.Field
import ch.epfl.bluebrain.nexus.commons.http.JsonLdCirceSupport._
import ch.epfl.bluebrain.nexus.rdf.Iri.AbsoluteIri
import ch.epfl.bluebrain.nexus.rdf.implicits._
import ch.epfl.bluebrain.nexus.service.config.ServiceConfig.HttpConfig
import ch.epfl.bluebrain.nexus.service.config.Settings
import ch.epfl.bluebrain.nexus.service.routes.Routes
import ch.epfl.bluebrain.nexus.util.EitherValues
import io.circe.generic.auto._
import org.mockito.IdiomaticMockito
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpecLike
class QueryDirectivesSpec
extends AnyWordSpecLike
with ScalatestRouteTest
with Matchers
with ScalaFutures
with EitherValues
with IdiomaticMockito {
private def genIri: AbsoluteIri = url"http://nexus.example.com/${UUID.randomUUID()}"
private def encode(url: AbsoluteIri): String = URLEncoder.encode(url.asString, "UTF-8")
private val config = Settings(system).serviceConfig
implicit private val http: HttpConfig = config.http
private def routes(inner: Route): Route =
Routes.wrap(inner)
"Query directives" should {
"handle query params" in {
val createdBy = genIri
val updatedBy = genIri
val type1 = genIri
val type2 = genIri
def projectParams =
Routes.wrap(
(get & QueryDirectives.searchParamsProjects) { params => complete(StatusCodes.OK -> params) }
)
Get("/") ~> routes(projectParams) ~> check {
responseAs[SearchParams] shouldEqual SearchParams.empty
}
Get(
s"/?rev=1&deprecated=true&label=myLabel&type=${encode(type1)}&type=${encode(type2)}&createdBy=${encode(createdBy)}&updatedBy=${encode(updatedBy)}"
) ~> routes(projectParams) ~> check {
responseAs[SearchParams] shouldEqual
SearchParams(
rev = Some(1L),
deprecated = Some(true),
projectLabel = Some(Field("myLabel", exactMatch = false)),
types = Set(type1, type2),
createdBy = Some(createdBy),
updatedBy = Some(updatedBy)
)
}
}
}
}
Example 19
| Source File: HttpGet.scala From http-verbs with Apache License 2.0 | 5 votes |
|
package uk.gov.hmrc.http
import java.net.URLEncoder
import uk.gov.hmrc.http.HttpVerbs.{GET => GET_VERB}
import uk.gov.hmrc.http.hooks.HttpHooks
import uk.gov.hmrc.http.logging.ConnectionTracing
import scala.concurrent.{ExecutionContext, Future}
trait HttpGet extends CoreGet with GetHttpTransport with HttpVerb with ConnectionTracing with HttpHooks with Retries {
override def GET[A](
url: String,
queryParams: Seq[(String, String)],
headers: Seq[(String, String)])(
implicit rds: HttpReads[A],
hc: HeaderCarrier,
ec: ExecutionContext): Future[A] = {
if (queryParams.nonEmpty && url.contains("?")) {
throw new UrlValidationException(
url,
s"${this.getClass}.GET(url, queryParams)",
"Query parameters should be provided in either url or as a Seq of tuples")
}
val urlWithQuery = url + makeQueryString(queryParams)
withTracing(GET_VERB, urlWithQuery) {
val httpResponse = retry(GET_VERB, urlWithQuery)(doGet(urlWithQuery, headers = headers))
executeHooks(url, GET_VERB, None, httpResponse)
mapErrors(GET_VERB, urlWithQuery, httpResponse).map(response => rds.read(GET_VERB, urlWithQuery, response))
}
}
private def makeQueryString(queryParams: Seq[(String, String)]) = {
val paramPairs = queryParams.map { case (k, v) => s"$k=${URLEncoder.encode(v, "utf-8")}" }
if (paramPairs.isEmpty) "" else paramPairs.mkString("?", "&", "")
}
}
Example 20
| Source File: security_api_yaml.scala From api-first-hand with MIT License | 5 votes |
|
package security.api.yaml
import de.zalando.play.controllers._
import org.scalacheck._
import org.scalacheck.Arbitrary._
import org.scalacheck.Prop._
import org.scalacheck.Test._
import org.specs2.mutable._
import org.specs2.execute._
import play.api.test.Helpers._
import play.api.test._
import play.api.mvc.MultipartFormData.FilePart
import play.api.mvc._
import org.junit.runner.RunWith
import java.net.URLEncoder
import com.fasterxml.jackson.databind.ObjectMapper
import play.api.http.Writeable
import play.api.libs.Files.TemporaryFile
import play.api.test.Helpers.{status => requestStatusCode_}
import play.api.test.Helpers.{contentAsString => requestContentAsString_}
import play.api.test.Helpers.{contentType => requestContentType_}
import org.scalatest.{OptionValues, WordSpec}
import org.scalatestplus.play.{OneAppPerTest, WsScalaTestClient}
import Generators._
import de.zalando.play.controllers.ArrayWrapper
//noinspection ScalaStyle
class Security_api_yamlSpec extends WordSpec with OptionValues with WsScalaTestClient with OneAppPerTest {
def toPath[T](value: T)(implicit binder: PathBindable[T]): String = Option(binder.unbind("", value)).getOrElse("")
def toQuery[T](key: String, value: T)(implicit binder: QueryStringBindable[T]): String = Option(binder.unbind(key, value)).getOrElse("")
def toHeader[T](value: T)(implicit binder: QueryStringBindable[T]): String = Option(binder.unbind("", value)).getOrElse("")
def checkResult(props: Prop): org.specs2.execute.Result =
Test.check(Test.Parameters.default, props).status match {
case Failed(args, labels) =>
val failureMsg = labels.mkString("\n") + " given args: " + args.map(_.arg).mkString("'", "', '","'")
org.specs2.execute.Failure(failureMsg)
case Proved(_) | Exhausted | Passed => org.specs2.execute.Success()
case PropException(_, e: IllegalStateException, _) => org.specs2.execute.Error(e.getMessage)
case PropException(_, e, labels) =>
val error = if (labels.isEmpty) e.getLocalizedMessage else labels.mkString("\n")
org.specs2.execute.Failure(error)
}
private def parserConstructor(mimeType: String) = PlayBodyParsing.jacksonMapper(mimeType)
def parseResponseContent[T](mapper: ObjectMapper, content: String, mimeType: Option[String], expectedType: Class[T]) =
if (expectedType.getCanonicalName == "scala.runtime.Null$") null else mapper.readValue(content, expectedType)
}
Example 21
| Source File: SearchPluginId.scala From sbt-idea-plugin with Apache License 2.0 | 5 votes |
|
package org.jetbrains.sbtidea.tasks
import java.net.URLEncoder
import java.nio.file.Path
import java.util.regex.Pattern
import org.jetbrains.sbtidea.PluginLogger
import org.jetbrains.sbtidea.download.BuildInfo
import org.jetbrains.sbtidea.download.plugin.LocalPluginRegistry
import com.eclipsesource.json._
import scalaj.http.Http
import scala.collection.JavaConverters._
class SearchPluginId(ideaRoot: Path, buildInfo: BuildInfo, useBundled: Boolean = true, useRemote: Boolean = true) {
private val REPO_QUERY = "https://plugins.jetbrains.com/api/search/plugins?search=%s&build=%s"
// true if plugin was found in the remote repo
def apply(query: String): Map[String, (String, Boolean)] = {
val local = if (useBundled) searchPluginIdLocal(query) else Map.empty
val remote = if (useRemote) searchPluginIdRemote(query) else Map.empty
remote ++ local
}
private def searchPluginIdLocal(query: String): Map[String, (String, Boolean)] = {
val pattern = Pattern.compile(query)
val registry = new LocalPluginRegistry(ideaRoot)
val allDescriptors = registry.getAllDescriptors
allDescriptors
.filter(descriptor => pattern.matcher(descriptor.name).find() || pattern.matcher(descriptor.id).find())
.map(descriptor => descriptor.id -> (descriptor.name, false))
.toMap
}
// Apparently we can't use json4s when cross-compiling for sbt because there are BOTH no shared versions AND binary compatibility
private def searchPluginIdRemote(query: String): Map[String, (String, Boolean)] = {
try {
val param = URLEncoder.encode(query, "UTF-8")
val url = REPO_QUERY.format(param, s"${buildInfo.edition.edition}-${buildInfo.getActualIdeaBuild(ideaRoot)}")
val data = Http(url).asString.body
val json = Json.parse(data)
val values = json.asArray().values().asScala.map(_.asObject())
val names = values.map(_.getString("name", "") -> true)
val ids = values.map(_.getString("xmlId", ""))
ids.zip(names).toMap
} catch {
case ex: Throwable =>
PluginLogger.warn(s"Failed to query IJ plugin repo: $ex")
Map.empty
}
}
}
Example 22
| Source File: ChuckNorris.scala From sumobot with Apache License 2.0 | 5 votes |
|
package com.sumologic.sumobot.plugins.chuck
import java.net.URLEncoder
import com.sumologic.sumobot.core.model.{IncomingMessage, OutgoingMessage, UserSender}
import com.sumologic.sumobot.plugins.BotPlugin
import org.apache.http.HttpResponse
import org.apache.http.util.EntityUtils
import play.api.libs.json.Json
import slack.models.User
class ChuckNorris extends BotPlugin {
override protected def help: String =
s"""I can Chuck Norris someone.
|
|chuck norris - Generally Chuck Norris.
|chuck norris me - Chuck Norris yourself.
|chuck norris <person> - Chuck Norris them.
""".stripMargin
private val ChuckNorrisAtMention = matchText(s"chuck norris $UserId.*")
private val ChuckNorrisMe = matchText(s"chuck norris me")
private val ChuckNorris = matchText(s"chuck norris")
private val BaseUrl = "http://api.icndb.com/jokes/random"
override protected def receiveIncomingMessage: ReceiveIncomingMessage = {
case msg@IncomingMessage(ChuckNorris(), _, _, _, _, _, _) =>
msg.httpGet(BaseUrl)(convertResponse)
case msg@IncomingMessage(ChuckNorrisMe(), _, _, _, _, _, sentByUser: UserSender) =>
msg.httpGet(url(sentByUser.slackUser))(convertResponse)
case msg@IncomingMessage(ChuckNorrisAtMention(userId), _, _, _, _, _, _) =>
userById(userId).foreach {
user =>
msg.httpGet(url(user))(convertResponse)
}
}
private def convertResponse(message: IncomingMessage, response: HttpResponse): OutgoingMessage = {
if (response.getStatusLine.getStatusCode == 200) {
val json = Json.parse(EntityUtils.toString(response.getEntity))
val joke = json \ "value" \ "joke"
val cleanJoke = joke.as[String].
replaceAllLiterally(""", "\"")
message.message(cleanJoke)
} else {
message.message("Chuck Norris is busy right now.")
}
}
private def url(user: User): String =
user.profile match {
case Some(profile) =>
s"$BaseUrl?firstName=${URLEncoder.encode(profile.first_name.getOrElse(user.name), "utf-8")}&lastName=${URLEncoder.encode(profile.last_name.getOrElse(""), "utf-8")}"
case None =>
s"$BaseUrl?firstName=${URLEncoder.encode(user.name, "utf-8")}&lastName="
}
}
Example 23
| Source File: RExecutor.scala From seahorse-workflow-executor with Apache License 2.0 | 5 votes |
|
package io.deepsense.workflowexecutor.executor
import java.net.URLEncoder
import scala.sys.process._
import io.deepsense.commons.utils.Logging
import io.deepsense.deeplang.CustomCodeExecutor
import io.deepsense.workflowexecutor.customcode.CustomCodeEntryPoint
case class RExecutor(rBackendPort: Int,
entryPointId: String,
customCodeEntryPoint: CustomCodeEntryPoint,
rExecutorScript: String)
extends CustomCodeExecutor with Logging {
def isValid(code: String): Boolean = true
val RExecutable = "Rscript"
def run(workflowId: String, nodeId: String, code: String): Unit = {
val command = s"""$RExecutable $rExecutorScript """ +
s""" $rBackendPort """ +
s""" $workflowId """ +
s""" $nodeId """ +
s""" $entryPointId """ +
s""" ${URLEncoder.encode(code, "UTF-8").replace("+", "%20")} """
logger.info(s"Starting a new RExecutor process: $command")
val log = new StringBuffer()
def logMethod(logmethod: String => Unit)(s: String) : Unit = {
log.append(s)
logmethod(s)
}
val rLogger = ProcessLogger(fout = logMethod(logger.debug), ferr = logMethod(logger.error))
val errorCode = command ! rLogger
// if RScript is successful then r_executor.R should take care of setting execution status
if (errorCode != 0) {
customCodeEntryPoint.executionFailed(workflowId, nodeId, log.toString())
}
}
}
Example 24
| Source File: UrlencodedData.scala From tapir with Apache License 2.0 | 5 votes |
|
package sttp.tapir.internal
import java.net.{URLDecoder, URLEncoder}
import java.nio.charset.Charset
private[tapir] object UrlencodedData {
def decode(s: String, charset: Charset): Seq[(String, String)] = {
s.split("&")
.toList
.flatMap(kv =>
kv.split("=", 2) match {
case Array(k, v) =>
Some((URLDecoder.decode(k, charset.toString), URLDecoder.decode(v, charset.toString)))
case _ => None
}
)
}
def encode(s: Seq[(String, String)], charset: Charset): String = {
s.map {
case (k, v) =>
s"${URLEncoder.encode(k, charset.toString)}=${URLEncoder.encode(v, charset.toString)}"
}
.mkString("&")
}
}
Example 25
| Source File: package.scala From twitter4s with Apache License 2.0 | 5 votes |
|
package com.danielasfregola.twitter4s
import java.net.URLEncoder
import scala.util.matching.Regex
package object http {
implicit class RichString(val value: String) extends AnyVal {
def urlEncoded = {
val urlEncodePattern = """\+|\*|%7E""".r
val replacer: Regex.Match => String = m =>
m.group(0) match {
case "+" => "%20"
case "*" => "%2A"
case "%7E" => "~"
}
val encodedValue = URLEncoder.encode(value, "UTF-8")
urlEncodePattern.replaceAllIn(encodedValue, replacer)
}
}
}
Example 26
| Source File: ExecutorService.scala From sparkplug with MIT License | 5 votes |
|
package springnz.sparkplug.executor
import java.net.{ URLDecoder, URLEncoder }
import java.time.LocalDate
import akka.actor._
import com.typesafe.config.ConfigFactory
import springnz.sparkplug.core._
import springnz.sparkplug.util.Logging
import scala.util.{ Properties, Try }
object Constants {
val defaultAkkaRemoteConfigSection = "akkaRemote"
val actorSystemName = "sparkplugExecutorSystem"
val brokerActorName = "sparkplugRequestBroker"
}
object ExecutorService extends Logging {
import Constants._
lazy val defaultRemoteAkkaConfig = ConfigFactory.load.getConfig(s"sparkplug.$defaultAkkaRemoteConfigSection")
// TODO: proper command line parsing to allow richer config options
def main(args: Array[String]): Unit = {
if (args.length < 4)
throw new IllegalArgumentException(s"Expected at least 4 arguments to ExecutorService. Args = : ${args.toList}")
val appName = args(1)
val sparkClientPath = args(3)
log.info(s"Starting Sparkplug ExecutorService: SparkClient = $sparkClientPath: ${LocalDate.now()}")
val remoteConfig = if (args.length == 6) {
val urlEncodedConfig = args(5)
val configString = URLDecoder.decode(urlEncodedConfig, "UTF-8")
val config = ConfigFactory.parseString(configString)
log.info(s"Using akka remote config:\n$configString")
config
} else {
log.info(s"Using default akka remote config from config section 'sparkplug.$defaultAkkaRemoteConfigSection'")
defaultRemoteAkkaConfig
}
import scala.collection.JavaConversions._
def env = System.getenv().toMap
log.debug(s"Environment:\n $env")
val system = ActorSystem(actorSystemName, remoteConfig)
val executorService = new ExecutorService(appName)
executorService.start(system, sparkClientPath)
log.info("Terminating the remote application.")
}
}
class ExecutorService(appName: String, brokerName: String = Constants.brokerActorName) extends LongLivedExecutor with Logging {
// Note that the SparkConf inherits all its settings from spark-submit
override val configurer: Configurer = new LocalConfigurer(appName, Properties.envOrNone("SPARK_MASTER"), None)
def start(system: ActorSystem, sparkClientPath: String): Try[Unit] = {
val actorOperation = SparkOperation[Unit] { implicit sparkContext ⇒
def postStopAction() = {
log.info("Cancelling any jobs (if any are running).")
sparkContext.cancelAllJobs()
log.info("Stopping Spark context.")
sparkContext.stop()
}
log.info("Creating requestBroker for ExecutorService.")
system.actorOf(Props(new RequestBroker(sparkClientPath, postStopAction)), name = brokerName)
}
log.info("Executing container operation (everything happens inside this method).")
val result = execute(actorOperation)
log.info("Finished executing container operation (everything happens inside this method).")
result
}
}
Example 27
| Source File: WikiServiceImpl.scala From BacklogMigration-Redmine with MIT License | 5 votes |
|
package com.nulabinc.backlog.r2b.redmine.service
import java.net.URLEncoder
import javax.inject.Inject
import com.nulabinc.backlog.migration.common.utils.Logging
import com.nulabinc.backlog.r2b.redmine.conf.RedmineApiConfiguration
import com.taskadapter.redmineapi.{NotFoundException, RedmineFormatException, RedmineInternalError, RedmineManager}
import com.taskadapter.redmineapi.bean.{WikiPage, WikiPageDetail}
import scala.jdk.CollectionConverters._
import scala.util.Try
class WikiServiceImpl @Inject()(apiConfig: RedmineApiConfiguration, redmine: RedmineManager) extends WikiService with Logging {
override def allWikis(): Seq[WikiPage] =
try {
redmine.getWikiManager.getWikiPagesByProject(apiConfig.projectKey).asScala.toSeq
} catch {
case e: Throwable =>
logger.warn(e.getMessage, e)
Seq.empty[WikiPage]
}
override def optWikiDetail(pageTitle: String): Option[WikiPageDetail] = {
logger.debug("Get a wiki Title: " + pageTitle)
try {
val wiki = redmine.getWikiManager.getWikiPageDetailByProjectAndTitle(apiConfig.projectKey, pageTitle)
Some(wiki)
} catch {
case e: RedmineInternalError if e.getMessage.contains("URISyntaxException") =>
logger.warn(s"Failed to get wiki details. URISyntaxException: ${e.getMessage} Title: $pageTitle")
None
case e: NotFoundException =>
logger.warn(s"Failed to get wiki details. NotFoundException: ${e.getMessage} Title: $pageTitle")
None
case e: RedmineFormatException =>
val url = s"${apiConfig.url}/projects/${apiConfig.projectKey}/wiki/${encode(pageTitle)}.json?include=attachments&key=${apiConfig.key}"
Try {
scala.io.Source.fromURL(url, "UTF-8").mkString
}.recover {
case e: Throwable =>
e.getMessage
}.map { res =>
logger.warn(s"Failed to get wiki details. RedmineFormatException: ${e.getMessage} Title: $pageTitle Raw response: $res")
}
None
case e: Throwable =>
throw e
}
}
private def encode(str: String): String =
URLEncoder.encode(str, "UTF-8")
}
Example 28
| Source File: IFrame.scala From almond with BSD 3-Clause "New" or "Revised" License | 5 votes |
|
package almond.display
import java.net.URLEncoder
final class IFrame private(
val src: String,
val width: Option[String],
val height: Option[String],
val params: Seq[(String, String)],
val displayId: String
) extends UpdatableDisplay {
private def copy(
src: String = src,
width: Option[String] = width,
height: Option[String] = height,
params: Seq[(String, String)] = params
): IFrame =
new IFrame(src, width, height, params, displayId)
def withSrc(src: String): IFrame =
copy(src = src)
def withWidth(width: Int): IFrame =
copy(width = Some(width.toString))
def withWidth(width: String): IFrame =
copy(width = Some(width))
def withWidth(widthOpt: Option[String]): IFrame =
copy(width = widthOpt)
def withHeight(height: Int): IFrame =
copy(height = Some(height.toString))
def withHeight(height: String): IFrame =
copy(height = Some(height))
def withHeight(heightOpt: Option[String]): IFrame =
copy(height = heightOpt)
private def html: String = {
val widthPart = width.fold("")(w => s"""width="$w"""")
val heightPart = height.fold("")(h => s"""height="$h"""")
val url = src + {
if (params.isEmpty) ""
else {
def encode(s: String): String =
URLEncoder.encode(s, "UTF-8")
.replaceAll("\\+", "%20")
params
.map { case (k, v) => s"${encode(k)}=${encode(v)}" }
.mkString("?", "&", "")
}
}
// FIXME Escaping of width / height
s"""<iframe
| $widthPart
| $heightPart
| src="$url"
|>
|</iframe>
""".stripMargin
}
def data(): Map[String, String] =
Map(Html.mimeType -> html)
}
object IFrame extends {
def apply(src: String): IFrame =
new IFrame(src, None, None, Nil, UpdatableDisplay.generateId())
}
Example 29
| Source File: ConfigController.scala From teamcity-slack with MIT License | 5 votes |
|
package com.fpd.teamcity.slack.controllers
import java.net.URLEncoder
import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
import com.fpd.teamcity.slack._
import com.fpd.teamcity.slack.Strings.ConfigController._
import jetbrains.buildServer.web.openapi.WebControllerManager
import org.springframework.web.servlet.ModelAndView
import scala.util.{Failure, Success}
class ConfigController(
configManager: ConfigManager,
controllerManager: WebControllerManager,
val permissionManager: PermissionManager,
slackGateway: SlackGateway
)
extends SlackController {
import ConfigController._
import Helpers.Implicits._
controllerManager.registerController(Resources.configPage.controllerUrl, this)
override def handle(request: HttpServletRequest, response: HttpServletResponse): ModelAndView = {
val result = for {
oauthKey ← request.param("oauthKey")
} yield {
val newConfig = ConfigManager.Config(oauthKey)
val publicUrl = request.param("publicUrl").getOrElse("")
val senderName = request.param("senderName").getOrElse("")
slackGateway.sessionByConfig(newConfig).map { _ ⇒
configManager.update(
oauthKey,
publicUrl,
request.param("personalEnabled").isDefined,
request.param("enabled").isDefined,
senderName,
request.param("sendAsAttachment").isDefined
)
} match {
case Success(true) ⇒ Left(true)
case Success(_) ⇒ Right(oauthTokenUpdateFailed)
case Failure(e) ⇒ Right(sessionByConfigError(e.getMessage))
}
}
val either = result.getOrElse(Right(oauthKeyParamMissing))
redirectTo(createRedirect(either, request.getContextPath), response)
}
}
object ConfigController {
private def createRedirect(either: Either[Boolean, String], context: String): String = either match {
case Left(_) ⇒ s"$context/admin/admin.html?item=${Strings.tabId}"
case Right(error) ⇒ s"$context/admin/admin.html?item=${Strings.tabId}&error=${URLEncoder.encode(error, "UTF-8")}"
}
}
Example 30
| Source File: instagram_api_yaml.scala From api-first-hand with MIT License | 5 votes |
|
package instagram.api.yaml
import de.zalando.play.controllers._
import org.scalacheck._
import org.scalacheck.Arbitrary._
import org.scalacheck.Prop._
import org.scalacheck.Test._
import org.specs2.mutable._
import org.specs2.execute._
import play.api.test.Helpers._
import play.api.test._
import play.api.mvc.MultipartFormData.FilePart
import play.api.mvc._
import org.junit.runner.RunWith
import java.net.URLEncoder
import com.fasterxml.jackson.databind.ObjectMapper
import play.api.http.Writeable
import play.api.libs.Files.TemporaryFile
import play.api.test.Helpers.{status => requestStatusCode_}
import play.api.test.Helpers.{contentAsString => requestContentAsString_}
import play.api.test.Helpers.{contentType => requestContentType_}
import org.scalatest.{OptionValues, WordSpec}
import org.scalatestplus.play.{OneAppPerTest, WsScalaTestClient}
import Generators._
import scala.math.BigInt
import scala.math.BigDecimal
//noinspection ScalaStyle
class Instagram_api_yamlSpec extends WordSpec with OptionValues with WsScalaTestClient with OneAppPerTest {
def toPath[T](value: T)(implicit binder: PathBindable[T]): String = Option(binder.unbind("", value)).getOrElse("")
def toQuery[T](key: String, value: T)(implicit binder: QueryStringBindable[T]): String = Option(binder.unbind(key, value)).getOrElse("")
def toHeader[T](value: T)(implicit binder: QueryStringBindable[T]): String = Option(binder.unbind("", value)).getOrElse("")
def checkResult(props: Prop): org.specs2.execute.Result =
Test.check(Test.Parameters.default, props).status match {
case Failed(args, labels) =>
val failureMsg = labels.mkString("\n") + " given args: " + args.map(_.arg).mkString("'", "', '","'")
org.specs2.execute.Failure(failureMsg)
case Proved(_) | Exhausted | Passed => org.specs2.execute.Success()
case PropException(_, e: IllegalStateException, _) => org.specs2.execute.Error(e.getMessage)
case PropException(_, e, labels) =>
val error = if (labels.isEmpty) e.getLocalizedMessage else labels.mkString("\n")
org.specs2.execute.Failure(error)
}
private def parserConstructor(mimeType: String) = PlayBodyParsing.jacksonMapper(mimeType)
def parseResponseContent[T](mapper: ObjectMapper, content: String, mimeType: Option[String], expectedType: Class[T]) =
if (expectedType.getCanonicalName == "scala.runtime.Null$") null else mapper.readValue(content, expectedType)
}
Example 31
| Source File: package.scala From nexus-kg with Apache License 2.0 | 5 votes |
|
package ch.epfl.bluebrain.nexus
import java.net.URLEncoder
import java.util.UUID
import akka.persistence.query.Offset
import cats.Show
import cats.implicits._
import ch.epfl.bluebrain.nexus.iam.client.types.Identity
import ch.epfl.bluebrain.nexus.iam.client.types.Identity.{Anonymous, Authenticated, Group, User}
import ch.epfl.bluebrain.nexus.kg.config.Vocabulary.nxv
import ch.epfl.bluebrain.nexus.kg.indexing.Statistics.ViewStatistics
import ch.epfl.bluebrain.nexus.kg.indexing.IdentifiedProgress
import ch.epfl.bluebrain.nexus.kg.resources.Rejection.InvalidResourceFormat
import ch.epfl.bluebrain.nexus.kg.resources.{Rejection, ResId}
import ch.epfl.bluebrain.nexus.rdf.Iri.AbsoluteIri
import ch.epfl.bluebrain.nexus.rdf.Node.BNode
import ch.epfl.bluebrain.nexus.rdf.Vocabulary.rdf
import ch.epfl.bluebrain.nexus.rdf.{Cursor, DecodingError, Graph, GraphDecoder, GraphEncoder}
import scala.util.Try
package object kg {
type IdStats = IdentifiedProgress[ViewStatistics]
type IdOffset = IdentifiedProgress[Offset]
def urlEncodeOrElse(s: String)(default: => String): String =
Try(URLEncoder.encode(s, "UTF-8").replaceAll("\\+", "%20")).getOrElse(default)
def uuid(): String =
UUID.randomUUID().toString.toLowerCase
def identities(resId: ResId, cursors: Set[Cursor]): Either[Rejection, Set[Identity]] = {
import alleycats.std.set._
cursors.foldM(Set.empty[Identity]) { (acc, cursor) =>
identity(resId, cursor).map(i => acc + i)
}
}
def anonDecoder: GraphDecoder[Anonymous] = GraphDecoder.instance { c =>
val tpe = c.down(rdf.tpe)
tpe.as[AbsoluteIri].flatMap {
case nxv.Anonymous.value => Right(Anonymous)
case other =>
val msg = s"Type does not match expected '${nxv.Anonymous.value.asUri}', but '${other.asUri}'"
Left(DecodingError(msg, tpe.history))
}
}
def userDecoder: GraphDecoder[User] = GraphDecoder.instance { c =>
(c.down(nxv.subject.value).as[String], c.down(nxv.realm.value).as[String]).mapN(User.apply)
}
def groupDecoder: GraphDecoder[Group] = GraphDecoder.instance { c =>
(c.down(nxv.group.value).as[String], c.down(nxv.realm.value).as[String]).mapN(Group.apply)
}
def authenticatedDecoder: GraphDecoder[Authenticated] = GraphDecoder.instance { c =>
c.down(nxv.realm).as[String].map(realm => Authenticated(realm))
}
implicit final def identityDecoder: GraphDecoder[Identity] =
anonDecoder.asInstanceOf[GraphDecoder[Identity]] or
userDecoder.asInstanceOf[GraphDecoder[Identity]] or
groupDecoder.asInstanceOf[GraphDecoder[Identity]] or
authenticatedDecoder.asInstanceOf[GraphDecoder[Identity]]
def identity(resId: ResId, c: Cursor): Either[Rejection, Identity] = {
identityDecoder(c).leftMap { _ =>
InvalidResourceFormat(
resId.ref,
"The provided payload could not be mapped to a resolver because the identity format is wrong"
)
}
}
implicit final val identityEncoder: GraphEncoder[Identity] = GraphEncoder.instance {
case Anonymous =>
Graph(BNode())
.append(rdf.tpe, nxv.Anonymous)
case User(subject, realm) =>
Graph(BNode())
.append(rdf.tpe, nxv.User)
.append(nxv.subject, subject)
.append(nxv.realm, realm)
case Group(group, realm) =>
Graph(BNode())
.append(rdf.tpe, nxv.Group)
.append(nxv.group, group)
.append(nxv.realm, realm)
case Authenticated(realm) =>
Graph(BNode())
.append(rdf.tpe, nxv.Authenticated)
.append(nxv.realm, realm)
}
}
Example 32
| Source File: package.scala From squbs with Apache License 2.0 | 5 votes |
|
package org.squbs
import java.net.{URLDecoder, URLEncoder}
import java.nio.ByteBuffer
import java.nio.charset.Charset
import akka.actor.{Address, AddressFromURIString}
import akka.util.ByteString
import com.typesafe.scalalogging.Logger
import org.apache.curator.framework.CuratorFramework
import org.apache.zookeeper.CreateMode
import org.apache.zookeeper.KeeperException.NodeExistsException
import scala.language.implicitConversions
import scala.util.Try
import scala.util.control.NonFatal
import scala.collection.JavaConverters._
package object cluster {
trait SegmentationLogic {
val segmentsSize:Int
def segmentation(partitionKey:ByteString): String = s"segment-${Math.abs(partitionKey.hashCode()) % segmentsSize}"
def partitionZkPath(partitionKey:ByteString): String = s"/segments/${segmentation(partitionKey)}/${keyToPath(partitionKey)}"
def sizeOfParZkPath(partitionKey:ByteString): String = s"${partitionZkPath(partitionKey)}/$$size"
def servantsOfParZkPath(partitionKey:ByteString): String = s"${partitionZkPath(partitionKey)}/servants"
}
case class DefaultSegmentationLogic(segmentsSize:Int) extends SegmentationLogic
def guarantee(path:String, data:Option[Array[Byte]], mode:CreateMode = CreateMode.EPHEMERAL)
(implicit zkClient:CuratorFramework, logger:Logger):String = {
try{
data match {
case None => zkClient.create.withMode(mode).forPath(path)
case Some(bytes) => zkClient.create.withMode(mode).forPath(path, bytes)
}
}
catch{
case e: NodeExistsException =>
if(data.nonEmpty && data.get.length > 0){
zkClient.setData().forPath(path, data.get)
}
path
case NonFatal(e) =>
logger.info("leader znode creation failed due to %s\n", e)
path
}
}
def safelyDiscard(path:String, recursive: Boolean = true)(implicit zkClient: CuratorFramework): String = Try {
if(recursive) zkClient.getChildren.forPath(path).asScala.foreach(child => safelyDiscard(s"$path/$child", recursive))
zkClient.delete.forPath(path)
path
} getOrElse path
def keyToPath(name:String):String = URLEncoder.encode(name, "utf-8")
def pathToKey(name:String):String = URLDecoder.decode(name, "utf-8")
private[cluster] val BYTES_OF_INT = Integer.SIZE / java.lang.Byte.SIZE
implicit def intToBytes(integer:Int):Array[Byte] = {
val buf = ByteBuffer.allocate(BYTES_OF_INT)
buf.putInt(integer)
buf.rewind
buf.array()
}
val UTF_8 = Charset.forName("utf-8")
implicit class ByteConversions(val bytes: Array[Byte]) extends AnyVal {
def toAddress: Option[Address] =
Option(bytes) flatMap (b => if (b.length <= 0) None else Some(AddressFromURIString(new String(b, UTF_8))))
def toInt: Int = ByteBuffer.wrap(bytes).getInt
def toUtf8: String = new String(bytes, UTF_8)
def toByteString: ByteString = ByteString(bytes)
def toAddressSet: Set[Address] = Try {
new String(bytes, UTF_8).split("[,]").map(seg => AddressFromURIString(seg.trim)).toSet
} getOrElse Set.empty
}
implicit def byteStringToUtf8(bs:ByteString):String = new String(bs.toArray, UTF_8)
implicit def addressToBytes(address:Address):Array[Byte] = {
address.toString.getBytes(UTF_8)
}
implicit def addressSetToBytes(members: Set[Address]): Array[Byte] = {
members.mkString(",").getBytes(UTF_8)
}
}
Example 33
| Source File: AdminSvc.scala From squbs with Apache License 2.0 | 5 votes |
|
package org.squbs.admin
import java.net.URLEncoder
import akka.http.scaladsl.model.Uri.Path
import akka.http.scaladsl.model._
import org.json4s.JsonAST.{JObject, JString}
import org.json4s.jackson.JsonMethods._
import org.squbs.unicomplex.{RouteDefinition, WebContext}
import org.squbs.util.ConfigUtil._
class AdminSvc extends RouteDefinition with WebContext {
val prefix = if (webContext == "") "/bean" else s"/$webContext/bean"
val exclusions = context.system.settings.config.get[Seq[String]]("squbs.admin.exclusions", Seq.empty[String]).toSet
val (exBeans, exFieldSet) = exclusions partition { !_.contains("::") }
val exFields = exFieldSet map { fieldSpec =>
val fields = fieldSpec split "::"
fields(0) -> fields(1)
} groupBy (_._1) mapValues { _.map(_._2) }
val route =
get {
pathEndOrSingleSlash {
extractUri { uri =>
complete {
val kv = MBeanUtil.allObjectNames collect {
case name if !(exBeans contains name) =>
val resource = Path(s"$prefix/${URLEncoder.encode(name.replace('=', '~'), "UTF-8")}")
name -> JString(uri.withPath(resource).toString())
}
HttpResponse(entity = HttpEntity(ContentTypes.`application/json`, pretty(render(JObject(kv)))))
}
}
} ~
path("bean" / Segment) { encName =>
complete {
val name = encName.replace('~', '=').replace('%', '/')
val response: HttpResponse =
if (exBeans contains name) HttpResponse(StatusCodes.NotFound, entity = StatusCodes.NotFound.defaultMessage)
else MBeanUtil.asJSON(name, exFields getOrElse (name, Set.empty))
.map { json => HttpResponse(entity = json) }
.getOrElse (HttpResponse(StatusCodes.NotFound, entity = StatusCodes.NotFound.defaultMessage))
response
}
}
}
}
Example 34
| Source File: UriUtils.scala From asura with MIT License | 5 votes |
|
package asura.core.http
import java.net.{URLDecoder, URLEncoder}
import java.nio.charset.StandardCharsets
import akka.http.scaladsl.model.Uri
import asura.common.exceptions.InvalidStatusException
import asura.common.util.StringUtils
import asura.core.es.model.HttpCaseRequest
import asura.core.protocols.Protocols
import asura.core.runtime.RuntimeContext
import asura.core.util.StringTemplate
object UriUtils {
val UTF8 = StandardCharsets.UTF_8.name()
def toUri(cs: HttpCaseRequest, context: RuntimeContext): Uri = {
Uri.from(
scheme = StringUtils.notEmptyElse(cs.request.protocol, Protocols.HTTP),
host = context.renderSingleMacroAsString(URLDecoder.decode(cs.request.host, UTF8)),
port = if (cs.request.port < 0 || cs.request.port > 65535) 80 else cs.request.port,
path = renderPath(URLDecoder.decode(cs.request.urlPath, UTF8), cs, context),
queryString = buildQueryString(cs, context)
)
}
def mapToQueryString(map: Map[String, Any], context: RuntimeContext = null): String = {
val sb = StringBuilder.newBuilder
for ((k, v) <- map) {
v match {
case v: String =>
val renderedValue = if (null != context) context.renderSingleMacroAsString(v) else v
sb.append(k).append("=").append(URLEncoder.encode(renderedValue, UTF8)).append("&")
case v: List[_] =>
v.foreach(i => {
val value = i.toString
val renderedValue = if (null != context) context.renderSingleMacroAsString(value) else value
sb.append(k).append("=").append(URLEncoder.encode(renderedValue, UTF8)).append("&")
})
}
}
if (sb.nonEmpty) {
sb.deleteCharAt(sb.length - 1)
}
sb.toString
}
@throws[InvalidStatusException]("if path template variable not in cs")
def renderPath(tpl: String, cs: HttpCaseRequest, context: RuntimeContext): String = {
if (null != cs.request) {
val params = cs.request.path
if (null != params && params.nonEmpty) {
val ctx = params.map(param => param.key -> context.renderSingleMacroAsString(param.value)).toMap
StringTemplate.uriPathParse(tpl, ctx)
} else {
tpl
}
} else {
tpl
}
}
def buildQueryString(cs: HttpCaseRequest, context: RuntimeContext): Option[String] = {
if (null != cs.request) {
val params = cs.request.query
if (null != params && params.nonEmpty) {
val sb = StringBuilder.newBuilder
for (param <- params if param.enabled) {
val key = if (StringUtils.isNotEmpty(param.key)) {
URLEncoder.encode(param.key, UTF8)
} else {
StringUtils.EMPTY
}
val value = if (StringUtils.isNotEmpty(param.value)) {
URLEncoder.encode(context.renderSingleMacroAsString(param.value), UTF8)
} else {
StringUtils.EMPTY
}
sb.append(key).append("=").append(value).append("&")
}
if (sb.nonEmpty) {
sb.deleteCharAt(sb.length - 1)
}
Some(sb.toString)
} else {
None
}
} else {
None
}
}
}
Example 35
| Source File: EntityUtils.scala From asura with MIT License | 5 votes |
|
package asura.core.http
import java.net.URLEncoder
import akka.http.scaladsl.model.{ContentType, ContentTypes, HttpEntity, RequestEntity}
import akka.util.ByteString
import asura.common.util.{LogUtils, StringUtils}
import asura.core.es.model.{HttpCaseRequest, KeyValueObject}
import asura.core.http.UriUtils.UTF8
import asura.core.runtime.RuntimeContext
import asura.core.util.JacksonSupport
import com.fasterxml.jackson.core.`type`.TypeReference
import com.typesafe.scalalogging.Logger
object EntityUtils {
val logger = Logger("EntityUtils")
def toEntity(cs: HttpCaseRequest, context: RuntimeContext): RequestEntity = {
val request = cs.request
var contentType: ContentType = ContentTypes.`text/plain(UTF-8)`
var byteString: ByteString = ByteString.empty
if (StringUtils.isNotEmpty(request.contentType) && null != request.body && request.body.nonEmpty) {
request.contentType match {
case HttpContentTypes.JSON =>
contentType = ContentTypes.`application/json`
val body = request.body.find(_.contentType == HttpContentTypes.JSON)
if (body.nonEmpty) {
byteString = ByteString(context.renderBodyAsString(body.get.data))
}
case HttpContentTypes.X_WWW_FORM_URLENCODED =>
contentType = ContentTypes.`application/x-www-form-urlencoded`
val body = request.body.find(_.contentType == HttpContentTypes.X_WWW_FORM_URLENCODED)
if (body.nonEmpty) {
var bodyStr: String = null
try {
val sb = StringBuilder.newBuilder
val params = JacksonSupport.parse(body.get.data, new TypeReference[Seq[KeyValueObject]]() {})
for (pair <- params if (pair.enabled && StringUtils.isNotEmpty(pair.key))) {
val rendered = context.renderBodyAsString(pair.value)
sb.append(pair.key).append("=").append(URLEncoder.encode(rendered, UTF8)).append("&")
}
if (sb.nonEmpty) {
sb.deleteCharAt(sb.length - 1)
}
bodyStr = sb.toString
} catch {
case t: Throwable =>
val errLog = LogUtils.stackTraceToString(t)
logger.warn(errLog)
bodyStr = errLog
}
byteString = ByteString(bodyStr)
}
case HttpContentTypes.TEXT_PLAIN =>
contentType = ContentTypes.`text/plain(UTF-8)`
val body = request.body.find(_.contentType == HttpContentTypes.TEXT_PLAIN)
if (body.nonEmpty) {
byteString = ByteString(context.renderBodyAsString(body.get.data))
}
case _ =>
}
}
HttpEntity(contentType, byteString)
}
}
Example 36
| Source File: HttpClient.scala From gospeak with Apache License 2.0 | 5 votes |
|
package gospeak.libs.http
import java.net.URLEncoder
import java.nio.charset.StandardCharsets
import cats.effect.IO
import gospeak.libs.scala.domain.CustomException
import hammock.apache.ApacheInterpreter
import hammock.{Encoder, Entity, Hammock, HttpResponse, InterpTrans, Method, Uri}
object HttpClient {
final case class Request(url: String,
query: Map[String, String] = Map(),
headers: Map[String, String] = Map())
final case class Response(status: Int,
body: String,
headers: Map[String, String]) {
def isOk: Boolean = 200 <= status && status < 400
}
object Response {
def from(res: HttpResponse): Response =
Response(
status = res.status.code,
headers = res.headers,
body = res.entity match {
case Entity.EmptyEntity => ""
case e: Entity.StringEntity => e.body
case e: Entity.ByteArrayEntity => new String(e.body, StandardCharsets.UTF_8)
})
}
private implicit val interpreter: InterpTrans[IO] = ApacheInterpreter.instance[IO]
private implicit val stringEncoder: Encoder[String] = (value: String) => Entity.StringEntity(value)
def get(url: String, query: Map[String, String] = Map(), headers: Map[String, String] = Map()): IO[Response] =
buildUri(url, query)(request(Method.GET, _, headers))
def postJson(url: String, body: String, query: Map[String, String] = Map(), headers: Map[String, String] = Map()): IO[Response] =
sendJson(method = Method.POST, url = url, body = body, query = query, headers = headers)
def putJson(url: String, body: String, query: Map[String, String] = Map(), headers: Map[String, String] = Map()): IO[Response] =
sendJson(method = Method.PUT, url = url, body = body, query = query, headers = headers)
def patchJson(url: String, body: String, query: Map[String, String] = Map(), headers: Map[String, String] = Map()): IO[Response] =
sendJson(method = Method.PATCH, url = url, body = body, query = query, headers = headers)
def deleteJson(url: String, body: String, query: Map[String, String] = Map(), headers: Map[String, String] = Map()): IO[Response] =
sendJson(method = Method.DELETE, url = url, body = body, query = query, headers = headers)
def postForm(url: String, body: Map[String, String], query: Map[String, String] = Map(), headers: Map[String, String] = Map()): IO[Response] =
sendForm(method = Method.POST, url = url, body = body, query = query, headers = headers)
def putForm(url: String, body: Map[String, String], query: Map[String, String] = Map(), headers: Map[String, String] = Map()): IO[Response] =
sendForm(method = Method.PUT, url = url, body = body, query = query, headers = headers)
def patchForm(url: String, body: Map[String, String], query: Map[String, String] = Map(), headers: Map[String, String] = Map()): IO[Response] =
sendForm(method = Method.PATCH, url = url, body = body, query = query, headers = headers)
def deleteForm(url: String, body: Map[String, String], query: Map[String, String] = Map(), headers: Map[String, String] = Map()): IO[Response] =
sendForm(method = Method.DELETE, url = url, body = body, query = query, headers = headers)
private def sendJson(method: Method, url: String, body: String, query: Map[String, String] = Map(), headers: Map[String, String] = Map()): IO[Response] =
buildUri(url, query)(request(method, _, headers + ("Content-Type" -> "application/json"), Some(body)))
private def sendForm(method: Method, url: String, body: Map[String, String], query: Map[String, String], headers: Map[String, String]): IO[Response] =
buildUri(url, query)(request(method, _, headers + ("Content-Type" -> "application/x-www-form-urlencoded"), Some(buildParams(body))))
private def buildUri(url: String, query: Map[String, String])(callback: Uri => IO[Response]): IO[Response] =
Uri.fromString(buildUrl(url, query)).map(callback).getOrElse(IO.raiseError(CustomException(s"Invalid URI '${buildUrl(url, query)}'")))
private def request(method: Method,
uri: Uri,
headers: Map[String, String],
body: Option[String] = None): IO[Response] = {
// def requestInfo: String = s"HttpClient.$method ${uri.path}\n headers: $headers" + body.map(b => s"\n body: $b").getOrElse("")
Hammock.request(method, uri, headers, body).exec[IO].map(Response.from)
// .map { r => println(s"\n$requestInfo\n response: ${r.status} ${r.body}"); r }
// .recoverWith { case e => println(s"\n$requestInfo\n response: ${e.getClass.getSimpleName}: ${e.getMessage}"); IO.raiseError(e) }
}
def buildUrl(url: String, query: Map[String, String]): String = {
if (query.isEmpty) {
url
} else if (url.contains("?")) {
url + "&" + buildParams(query)
} else {
url + "?" + buildParams(query)
}
}
private def buildParams(params: Map[String, String]): String =
params.map { case (key, value) => s"$key=${URLEncoder.encode(value, "UTF8")}" }.mkString("&")
}
Example 37
| Source File: DefaultBodyWritables.scala From play-ws with Apache License 2.0 | 5 votes |
|
package play.api.libs.ws
import java.io.File
import java.nio.ByteBuffer
import java.util.function.Supplier
import akka.stream.scaladsl.StreamConverters.fromInputStream
import akka.stream.scaladsl.FileIO
import akka.stream.scaladsl.Source
import akka.util.ByteString
import scala.compat.java8.FunctionConverters.asScalaFromSupplier
implicit val writeableOf_urlEncodedForm: BodyWritable[Map[String, Seq[String]]] = {
import java.net.URLEncoder
BodyWritable(
formData =>
InMemoryBody(
ByteString.fromString(
formData.flatMap(item => item._2.map(c => s"${item._1}=${URLEncoder.encode(c, "UTF-8")}")).mkString("&")
)
),
"application/x-www-form-urlencoded"
)
}
implicit val writeableOf_urlEncodedSimpleForm: BodyWritable[Map[String, String]] = {
writeableOf_urlEncodedForm.map[Map[String, String]](_.map(kv => kv._1 -> Seq(kv._2)))
}
}
object DefaultBodyWritables extends DefaultBodyWritables
Example 38
| Source File: LivyConnectionSpec.scala From incubator-livy with Apache License 2.0 | 5 votes |
|
package org.apache.livy.client.http
import java.io.IOException
import java.net.URLEncoder
import java.nio.charset.StandardCharsets.UTF_8
import org.apache.http.client.utils.URIBuilder
import org.eclipse.jetty.security._
import org.eclipse.jetty.security.authentication.BasicAuthenticator
import org.eclipse.jetty.util.security._
import org.scalatest.{BeforeAndAfterAll, FunSpecLike}
import org.scalatest.Matchers._
import org.scalatra.servlet.ScalatraListener
import org.apache.livy.{LivyBaseUnitTestSuite, LivyConf}
import org.apache.livy.server.WebServer
class LivyConnectionSpec extends FunSpecLike with BeforeAndAfterAll with LivyBaseUnitTestSuite {
describe("LivyConnection") {
def basicAuth(username: String, password: String, realm: String): SecurityHandler = {
val roles = Array("user")
val l = new HashLoginService()
l.putUser(username, Credential.getCredential(password), roles)
l.setName(realm)
val constraint = new Constraint()
constraint.setName(Constraint.__BASIC_AUTH)
constraint.setRoles(roles)
constraint.setAuthenticate(true)
val cm = new ConstraintMapping()
cm.setConstraint(constraint)
cm.setPathSpec("/*")
val csh = new ConstraintSecurityHandler()
csh.setAuthenticator(new BasicAuthenticator())
csh.setRealmName(realm)
csh.addConstraintMapping(cm)
csh.setLoginService(l)
csh
}
def test(password: String, livyConf: LivyConf = new LivyConf()): Unit = {
val username = "user name"
val server = new WebServer(livyConf, "0.0.0.0", 0)
server.context.setSecurityHandler(basicAuth(username, password, "realm"))
server.context.setResourceBase("src/main/org/apache/livy/server")
server.context.setInitParameter(ScalatraListener.LifeCycleKey,
classOf[HttpClientTestBootstrap].getCanonicalName)
server.context.addEventListener(new ScalatraListener)
server.start()
val utf8Name = UTF_8.name()
val uri = new URIBuilder()
.setScheme(server.protocol)
.setHost(server.host)
.setPort(server.port)
.setUserInfo(URLEncoder.encode(username, utf8Name), URLEncoder.encode(password, utf8Name))
.build()
info(uri.toString)
val conn = new LivyConnection(uri, new HttpConf(null))
try {
conn.get(classOf[Object], "/") should not be (null)
} finally {
conn.close()
}
server.stop()
server.join()
}
it("should support HTTP auth with password") {
test("pass:word")
}
it("should support HTTP auth with empty password") {
test("")
}
it("should be failed with large header size") {
val livyConf = new LivyConf()
.set(LivyConf.REQUEST_HEADER_SIZE, 1024)
.set(LivyConf.RESPONSE_HEADER_SIZE, 1024)
val pwd = "test-password" * 100
val exception = intercept[IOException](test(pwd, livyConf))
exception.getMessage.contains("Request Header Fields Too Large") should be(true)
}
it("should be succeeded with configured header size") {
val livyConf = new LivyConf()
.set(LivyConf.REQUEST_HEADER_SIZE, 2048)
.set(LivyConf.RESPONSE_HEADER_SIZE, 2048)
val pwd = "test-password" * 100
test(pwd, livyConf)
}
}
}
Example 39
| Source File: JobReceptionist.scala From 006877 with MIT License | 5 votes |
|
package aia.cluster
package words
import java.net.URLEncoder
import akka.actor._
import akka.actor.Terminated
object JobReceptionist {
def props = Props(new JobReceptionist)
def name = "receptionist"
case class JobRequest(name: String, text: List[String])
sealed trait Response
case class JobSuccess(name: String, map: Map[String, Int]) extends Response
case class JobFailure(name: String) extends Response
case class WordCount(name: String, map: Map[String, Int])
case class Job(name: String, text: List[String], respondTo: ActorRef, jobMaster: ActorRef)
}
class JobReceptionist extends Actor
with ActorLogging
with CreateMaster {
import JobReceptionist._
import JobMaster.StartJob
import context._
override def supervisorStrategy: SupervisorStrategy =
SupervisorStrategy.stoppingStrategy
var jobs = Set[Job]()
var retries = Map[String, Int]()
val maxRetries = 3
def receive = {
case JobRequest(name, text) =>
log.info(s"Received job $name")
val masterName = "master-"+URLEncoder.encode(name, "UTF8")
val jobMaster = createMaster(masterName)
val job = Job(name, text, sender, jobMaster)
jobs = jobs + job
jobMaster ! StartJob(name, text)
watch(jobMaster)
case WordCount(jobName, map) =>
log.info(s"Job $jobName complete.")
log.info(s"result:${map}")
jobs.find(_.name == jobName).foreach { job =>
job.respondTo ! JobSuccess(jobName, map)
stop(job.jobMaster)
jobs = jobs - job
}
case Terminated(jobMaster) =>
jobs.find(_.jobMaster == jobMaster).foreach { failedJob =>
log.error(s"Job Master $jobMaster terminated before finishing job.")
val name = failedJob.name
log.error(s"Job ${name} failed.")
val nrOfRetries = retries.getOrElse(name, 0)
if(maxRetries > nrOfRetries) {
if(nrOfRetries == maxRetries -1) {
// Simulating that the Job worker will work just before max retries
val text = failedJob.text.filterNot(_.contains("FAIL"))
self.tell(JobRequest(name, text), failedJob.respondTo)
} else self.tell(JobRequest(name, failedJob.text), failedJob.respondTo)
retries = retries + retries.get(name).map(r=> name -> (r + 1)).getOrElse(name -> 1)
}
}
}
}
trait CreateMaster {
def context: ActorContext
def createMaster(name: String) = context.actorOf(JobMaster.props, name)
}
Example 40
| Source File: PathBuilderTest.scala From wix-http-testkit with MIT License | 5 votes |
|
package com.wix.e2e.http.client.internals
import java.net.URLEncoder
import com.wix.e2e.http.client.drivers.PathBuilderTestSupport
import com.wix.e2e.http.client.drivers.UrlBuilderMatchers._
import org.specs2.mutable.SpecWithJUnit
import org.specs2.specification.Scope
class PathBuilderTest extends SpecWithJUnit {
trait ctx extends Scope
with PathBuilderTestSupport
"Url building" should {
"handle context path with single path" in new ctx {
baseUri.copy(contextRoot = Some(contextRoot)).asUri must beUrl(s"http://${baseUri.host}:${baseUri.port}$contextRoot")
}
"handle empty context root" in new ctx {
baseUri.copy(contextRoot = None).asUri must beUrl(s"http://${baseUri.host}:${baseUri.port}")
baseUri.copy(contextRoot = Some("")).asUri must beUrl(s"http://${baseUri.host}:${baseUri.port}")
baseUri.copy(contextRoot = Some(" ")).asUri must beUrl(s"http://${baseUri.host}:${baseUri.port}")
}
"handle context path with more than one path" in new ctx {
baseUri.copy(contextRoot = Some(contextRootWithMultiplePaths)).asUri must beUrl(s"http://${baseUri.host}:${baseUri.port}$contextRootWithMultiplePaths")
}
"handle no context and empty relative path" in new ctx {
baseUri.copy(contextRoot = None).asUriWith("/") must beUrl(s"http://${baseUri.host}:${baseUri.port}")
baseUri.copy(contextRoot = None).asUriWith("") must beUrl(s"http://${baseUri.host}:${baseUri.port}")
baseUri.copy(contextRoot = None).asUriWith(" ") must beUrl(s"http://${baseUri.host}:${baseUri.port}")
}
"ignore cases in which path and context root are single slash" in new ctx {
baseUri.copy(contextRoot = Some("/")).asUriWith("/") must beUrl(s"http://${baseUri.host}:${baseUri.port}")
baseUri.copy(contextRoot = None).asUriWith("/") must beUrl(s"http://${baseUri.host}:${baseUri.port}")
}
"allow to append relative path" in new ctx {
baseUri.copy(contextRoot = None).asUriWith(relativePath) must beUrl(s"http://${baseUri.host}:${baseUri.port}$relativePath")
baseUri.copy(contextRoot = Some("")).asUriWith(relativePath) must beUrl(s"http://${baseUri.host}:${baseUri.port}$relativePath")
baseUri.copy(contextRoot = Some("/")).asUriWith(relativePath) must beUrl(s"http://${baseUri.host}:${baseUri.port}$relativePath")
}
"allow to append relative path with multiple parts" in new ctx {
baseUri.copy(contextRoot = None).asUriWith(relativePathWithMultipleParts) must beUrl(s"http://${baseUri.host}:${baseUri.port}$relativePathWithMultipleParts")
}
"properly combine context root and relative path" in new ctx {
baseUri.copy(contextRoot = Some(contextRoot)).asUriWith(relativePath) must beUrl(s"http://${baseUri.host}:${baseUri.port}$contextRoot$relativePath")
baseUri.copy(contextRoot = Some(contextRootWithMultiplePaths)).asUriWith(relativePathWithMultipleParts) must beUrl(s"http://${baseUri.host}:${baseUri.port}$contextRootWithMultiplePaths$relativePathWithMultipleParts")
}
"support context root that doesn't start with /" in new ctx {
baseUri.copy(contextRoot = Some(contextRoot.stripPrefix("/"))).asUri must beUrl(s"http://${baseUri.host}:${baseUri.port}$contextRoot")
baseUri.copy(contextRoot = None).asUriWith(relativePath.stripPrefix("/")) must beUrl(s"http://${baseUri.host}:${baseUri.port}$relativePath")
}
"support relative path with explicit request params" in new ctx {
baseUri.copy(contextRoot = None).asUriWith(s"$relativePath?key=val") must beUrl(s"http://${baseUri.host}:${baseUri.port}$relativePath?key=val")
}
"support relative path with explicit request escaped params" in new ctx {
baseUri.copy(contextRoot = None).asUriWith(s"$relativePath?key=val&encoded=$escapedCharacters") must beUrl(s"http://${baseUri.host}:${baseUri.port}$relativePath?key=val&encoded=${URLEncoder.encode(escapedCharacters, "UTF-8")}")
}
"support relative path with explicit request params without value" in new ctx {
baseUri.copy(contextRoot = None).asUriWith(s"$relativePath?key") must beUrl(s"http://${baseUri.host}:${baseUri.port}$relativePath?key")
}
}
}
Example 41
| Source File: package.scala From wix-http-testkit with MIT License | 5 votes |
|
package com.wix.e2e.http.client
import java.net.URLEncoder
import akka.http.scaladsl.model.Uri
import akka.http.scaladsl.model.Uri.Path
import com.wix.e2e.http.BaseUri
package object internals {
implicit class `BaseUri --> akka.Uri`(private val u: BaseUri) extends AnyVal {
def asUri: Uri = asUriWith("")
def asUriWith(relativeUrl: String): Uri =
if (relativeUrl.contains('?'))
urlWithoutParams(relativeUrl).withRawQueryString( extractParamsFrom(relativeUrl) )
else urlWithoutParams(relativeUrl)
private def fixPath(url: Option[String]) = {
url.map( _.trim )
.map( u => s"/${u.stripPrefix("/")}" )
.filterNot( _.equals("/") )
.map( Path(_) )
.getOrElse( Path.Empty )
}
private def buildPath(context: Option[String], relativePath: Option[String]) = {
val c = fixPath(context)
val r = fixPath(relativePath)
c ++ r
}
private def urlWithoutParams(relativeUrl: String) =
Uri(scheme = "http").withHost(u.host)
.withPort(u.port)
.withPath( buildPath(u.contextRoot, Option(extractPathFrom(relativeUrl))) )
private def extractPathFrom(relativeUrl: String) = relativeUrl.split('?').head
private def extractParamsFrom(relativeUrl: String) =
rebuildAndEscapeParams(relativeUrl.substring(relativeUrl.indexOf('?') + 1))
private def rebuildAndEscapeParams(p: String) =
p.split("&")
.map( _.split("=") )
.map( { case Array(k, v) => s"$k=${URLEncoder.encode(v, "UTF-8")}"
case Array(k) => k
} )
.mkString("&")
}
}
Example 42
| Source File: PlayAPISpec.scala From playsonify with MIT License | 5 votes |
|
package com.alexitc.playsonify.test
import java.net.URLEncoder
import org.scalatest.concurrent.ScalaFutures
import org.scalatestplus.play.PlaySpec
import play.api.inject.guice.GuiceApplicationBuilder
import play.api.mvc.Result
import play.api.test.FakeRequest
import play.api.test.Helpers._
import play.api.{Application, Mode}
import scala.concurrent.Future
def GET(url: String, extraHeaders: (String, String)*): Future[Result] = {
val headers = JsonHeader :: extraHeaders.toList
val request = FakeRequest("GET", url)
.withHeaders(headers: _*)
val response = route(application, request).get
log(request, response)
response
}
def POST(url: String, extraHeaders: (String, String)*): Future[Result] = {
POST(url, None, extraHeaders: _*)
}
def POST(url: String, jsonBody: Option[String], extraHeaders: (String, String)*): Future[Result] = {
val headers = JsonHeader :: extraHeaders.toList
val json = jsonBody.getOrElse(EmptyJson)
val request = FakeRequest("POST", url)
.withHeaders(headers: _*)
.withBody(json)
val response = route(application, request).get
log(request, response)
response
}
def PUT(url: String, extraHeaders: (String, String)*): Future[Result] = {
PUT(url, None, extraHeaders: _*)
}
def PUT(url: String, jsonBody: Option[String], extraHeaders: (String, String)*): Future[Result] = {
val headers = JsonHeader :: extraHeaders.toList
val json = jsonBody.getOrElse(EmptyJson)
val request = FakeRequest("PUT", url)
.withHeaders(headers: _*)
.withBody(json)
val response = route(application, request).get
log(request, response)
response
}
def DELETE(url: String, extraHeaders: (String, String)*): Future[Result] = {
val headers = JsonHeader :: extraHeaders.toList
val request = FakeRequest("DELETE", url)
.withHeaders(headers: _*)
val response = route(application, request).get
log(request, response)
response
}
}
object PlayAPISpec {
object Implicits {
implicit class HttpExt(val params: List[(String, String)]) extends AnyVal {
def toQueryString: String = {
params
.map { case (key, value) =>
val encodedKey = URLEncoder.encode(key, "UTF-8")
val encodedValue = URLEncoder.encode(value, "UTF-8")
List(encodedKey, encodedValue).mkString("=")
}
.mkString("&")
}
}
implicit class StringUrlExt(val url: String) extends AnyVal {
def withQueryParams(params: (String, String)*): String = {
List(url, params.toList.toQueryString).mkString("?")
}
}
}
}
Example 43
| Source File: Config.scala From dependency with MIT License | 5 votes |
|
package io.flow.dependency.www.lib
import io.flow.dependency.v0.models.Scms
import io.flow.play.util.{Config => FlowConfig}
import java.net.URLEncoder
import javax.inject.{Inject, Singleton}
@Singleton
class GitHubConfig @Inject()(config: FlowConfig) {
private val Scopes = Seq("user:email", "repo", "read:repo_hook", "write:repo_hook")
private val OauthUrl = "https://github.com/login/oauth/authorize"
private val githubClientId = config.requiredString("github.dependency.client.id")
private val dependencyWwwHost = config.requiredString("dependency.www.host")
private val githubBaseUrl = s"$dependencyWwwHost/login/github"
def githubOauthUrl(returnUrl: Option[String]): String = {
val returnUrlParam = returnUrl
.map { encoded =>
s"$githubBaseUrl?return_url=$encoded"
}
val params: Map[String, String] = Seq(
Some("scope" -> Scopes.mkString(",")),
Some("client_id" -> githubClientId),
returnUrlParam.map("redirect_uri" -> _)
).flatten.toMap
val queryParams =
params
.view
.mapValues(URLEncoder.encode(_, "UTF-8"))
.map {
case (key, value) => s"$key=$value"
}
OauthUrl + "?" + queryParams.mkString("&")
}
}
object Config {
val VersionsPerPage = 5
def scmsUrl(scms: Scms, uri: String, path: String): String = {
val separator = if (path.startsWith("/")) "" else "/"
val pathSep = separator + path
scms match {
case Scms.Github =>
s"$uri/blob/master$pathSep"
case Scms.UNDEFINED(_) =>
uri + pathSep
}
}
}
Example 44
| Source File: AttachmentCtrl.scala From Cortex with GNU Affero General Public License v3.0 | 5 votes |
|
package org.thp.cortex.controllers
import java.net.URLEncoder
import java.nio.file.Files
import javax.inject.{Inject, Singleton}
import play.api.http.HttpEntity
import play.api.libs.Files.DefaultTemporaryFileCreator
import play.api.mvc._
import play.api.{mvc, Configuration}
import akka.stream.scaladsl.FileIO
import net.lingala.zip4j.core.ZipFile
import net.lingala.zip4j.model.ZipParameters
import net.lingala.zip4j.util.Zip4jConstants
import org.thp.cortex.models.Roles
import org.elastic4play.Timed
import org.elastic4play.controllers.{Authenticated, Renderer}
import org.elastic4play.models.AttachmentAttributeFormat
import org.elastic4play.services.AttachmentSrv
@Timed("controllers.AttachmentCtrl.downloadZip")
def downloadZip(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { _ ⇒
if (!name.getOrElse("").intersect(AttachmentAttributeFormat.forbiddenChar).isEmpty)
BadRequest("File name is invalid")
else {
val f = tempFileCreator.create("zip", hash).path
Files.delete(f)
val zipFile = new ZipFile(f.toFile)
val zipParams = new ZipParameters
zipParams.setCompressionLevel(Zip4jConstants.DEFLATE_LEVEL_FASTEST)
zipParams.setEncryptFiles(true)
zipParams.setEncryptionMethod(Zip4jConstants.ENC_METHOD_STANDARD)
zipParams.setPassword(password)
zipParams.setFileNameInZip(name.getOrElse(hash))
zipParams.setSourceExternalStream(true)
zipFile.addStream(attachmentSrv.stream(hash), zipParams)
Result(
header = ResponseHeader(
200,
Map(
"Content-Disposition" → s"""attachment; filename="${URLEncoder.encode(name.getOrElse(hash), "utf-8")}.zip"""",
"Content-Type" → "application/zip",
"Content-Transfer-Encoding" → "binary",
"Content-Length" → Files.size(f).toString
)
),
body = HttpEntity.Streamed(FileIO.fromPath(f), Some(Files.size(f)), Some("application/zip"))
)
}
}
}
Example 45
| Source File: VatReturnsConnectorSpec.scala From vat-api with Apache License 2.0 | 5 votes |
|
package uk.gov.hmrc.vatapi.connectors
import java.net.URLEncoder
import org.joda.time.DateTime
import org.scalatestplus.play.guice.GuiceOneAppPerSuite
import play.api.http.Status._
import play.api.libs.json.Json
import uk.gov.hmrc.domain.Vrn
import uk.gov.hmrc.http.{HeaderCarrier, HttpResponse}
import uk.gov.hmrc.vatapi.UnitSpec
import uk.gov.hmrc.vatapi.assets.TestConstants.VatReturn._
import uk.gov.hmrc.vatapi.mocks.MockHttp
import uk.gov.hmrc.vatapi.mocks.config.MockAppContext
import uk.gov.hmrc.vatapi.models.des.{DesError, DesErrorCode}
import uk.gov.hmrc.vatapi.resources.wrappers.VatReturnResponse
import scala.concurrent.ExecutionContext.Implicits.global
class VatReturnsConnectorSpec extends UnitSpec with GuiceOneAppPerSuite
with MockHttp
with MockAppContext {
class Test {
val connector = new VatReturnsConnector (mockAppContext, mockHttp)
val testDesUrl = "test"
MockAppContext.desUrl.returns(testDesUrl)
}
implicit val hc: HeaderCarrier = HeaderCarrier()
val testVrn: Vrn = Vrn("123456789")
val testDateTime: DateTime = DateTime.now()
val successResponse = HttpResponse(OK, responseJson = None)
val invalidPayloadResponse =
HttpResponse(
BAD_REQUEST,
responseJson = Some(Json.toJson(DesError(DesErrorCode.INVALID_PAYLOAD, "Submission has not passed validation. Invalid parameter Payload.")))
)
"VatReturnsConnector.post" should {
"return a VatReturnsResponse model with correct body in case of success" when {
"pointing to the vat normal" in new Test {
val testUrl: String = s"$testDesUrl/enterprise/return/vat/$testVrn"
setupMockHttpPostString(testUrl, desVatReturnDeclaration(testDateTime).toJsonString)(successResponse)
await(connector.post(testVrn, desVatReturnDeclaration(testDateTime))) shouldBe VatReturnResponse(successResponse)
val headers: Map[String, String] = MockHttp.fetchHeaderCarrier.headers.toMap
headers("Accept") shouldBe "application/json"
headers("OriginatorID") shouldBe "MDTP"
}
}
"return a VatReturnsResponse with the correct error body when an error is retrieved from DES" when {
"pointing to the vat normal" in new Test {
val testUrl: String = s"$testDesUrl/enterprise/return/vat/$testVrn"
setupMockHttpPostString(testUrl, desVatReturnDeclaration(testDateTime).toJsonString)(invalidPayloadResponse)
await(connector.post(testVrn, desVatReturnDeclaration(testDateTime))) shouldBe VatReturnResponse(invalidPayloadResponse)
}
}
"VatReturnsConnector.query" should {
val periodKey = "test"
"return a VatReturnsResponse model in case of success" when {
"pointing to the vat normal" in new Test {
val testUrl: String = s"$testDesUrl/vat/returns/vrn/$testVrn?period-key=${URLEncoder.encode(periodKey, "UTF-8")}"
setupMockHttpGet(testUrl)(successResponse)
await(connector.query(testVrn, periodKey)) shouldBe VatReturnResponse(successResponse)
}
}
"return a VatReturnsResponse with the correct error body when an error is retrieved from DES" when {
"pointing to the vat normal" in new Test {
val testUrl: String = s"$testDesUrl/vat/returns/vrn/$testVrn?period-key=${URLEncoder.encode(periodKey, "UTF-8")}"
setupMockHttpGet(testUrl)(invalidPayloadResponse)
await(connector.query(testVrn, periodKey)) shouldBe VatReturnResponse(invalidPayloadResponse)
}
}
}
}
}
Example 46
| Source File: ReadSideImpl.scala From lagom with Apache License 2.0 | 5 votes |
|
package com.lightbend.lagom.internal.scaladsl.persistence
import java.net.URLEncoder
import akka.actor.ActorSystem
import akka.actor.Props
import akka.stream.Materializer
import com.lightbend.lagom.internal.persistence.ReadSideConfig
import com.lightbend.lagom.internal.persistence.cluster.ClusterStartupTask
import com.lightbend.lagom.internal.projection.ProjectionRegistry
import com.lightbend.lagom.internal.projection.ProjectionRegistryActor.WorkerCoordinates
import com.lightbend.lagom.scaladsl.persistence._
import scala.concurrent.ExecutionContext
private[lagom] class ReadSideImpl(
system: ActorSystem,
config: ReadSideConfig,
persistentEntityRegistry: PersistentEntityRegistry,
projectionRegistry: ProjectionRegistry,
name: Option[String]
)(implicit ec: ExecutionContext, mat: Materializer)
extends ReadSide {
override def register[Event <: AggregateEvent[Event]](processorFactory: => ReadSideProcessor[Event]): Unit =
registerFactory(() => processorFactory)
private[lagom] def registerFactory[Event <: AggregateEvent[Event]](
processorFactory: () => ReadSideProcessor[Event]
) = {
val readSideProcessor = processorFactory()
val readSideName = name.fold("")(_ + "-") + readSideProcessor.readSideName
val tags = readSideProcessor.aggregateTags
val entityIds = tags.map(_.tag)
// try to create one instance to fail fast
val eventClass = tags.headOption match {
case Some(tag) => tag.eventType
case None =>
throw new IllegalArgumentException(s"ReadSideProcessor ${readSideProcessor.getClass.getName} returned 0 tags")
}
val encodedReadSideName = URLEncoder.encode(readSideName, "utf-8")
val globalPrepareTask: ClusterStartupTask =
ClusterStartupTask(
system,
s"readSideGlobalPrepare-$encodedReadSideName",
() => processorFactory().buildHandler().globalPrepare(),
config.globalPrepareTimeout,
config.role,
config.minBackoff,
config.maxBackoff,
config.randomBackoffFactor
)
val projectionName = readSideName
val readSidePropsFactory: WorkerCoordinates => Props = (coordinates) =>
ReadSideActor.props(
coordinates,
config,
eventClass,
globalPrepareTask,
persistentEntityRegistry.eventStream[Event],
processorFactory
)
projectionRegistry.registerProjection(
projectionName,
entityIds,
readSidePropsFactory,
config.role
)
}
}
Example 47
| Source File: FilesystemSnapshotStore.scala From eventuate with Apache License 2.0 | 5 votes |
|
package com.rbmhtechnology.eventuate.snapshot.filesystem
import java.io._
import java.net.URLEncoder
import akka.event.{ LogSource, Logging }
import com.rbmhtechnology.eventuate._
import com.rbmhtechnology.eventuate.snapshot.SnapshotStore
import org.apache.commons.io.IOUtils
import scala.concurrent.Future
import scala.collection.immutable.Seq
import scala.util._
object FilesystemSnapshotStore {
implicit val logSource = LogSource.fromAnyClass[FilesystemSnapshotStore]
}
class FilesystemSnapshotStore(settings: FilesystemSnapshotStoreSettings, logId: String) extends SnapshotStore {
private val log = Logging(settings.system, classOf[FilesystemSnapshotStore])
private val rootDir = new File(settings.rootDir, URLEncoder.encode(logId, "UTF-8"))
rootDir.mkdirs()
override def deleteAsync(lowerSequenceNr: Long): Future[Unit] = {
import settings.writeDispatcher
Future(delete(lowerSequenceNr))
}
override def saveAsync(snapshot: Snapshot): Future[Unit] = {
import settings.writeDispatcher
Future(withOutputStream(dstDir(snapshot.metadata.emitterId), snapshot.metadata.sequenceNr)(serialize(_, snapshot)))
}
override def loadAsync(emitterId: String): Future[Option[Snapshot]] = {
import settings.readDispatcher
Future(load(dstDir(emitterId)))
}
def delete(lowerSequenceNr: Long): Unit = for {
emitterId <- rootDir.listFiles
emitterDir = dstDir(emitterId.getName)
sequenceNr <- decreasingSequenceNrs(emitterDir) if sequenceNr >= lowerSequenceNr
} dstFile(emitterDir, sequenceNr).delete()
def load(dir: File): Option[Snapshot] = {
@annotation.tailrec
def go(snrs: Seq[Long]): Option[Snapshot] = snrs.headOption match {
case None => None
case Some(snr) => Try(withInputStream(dir, snr)(deserialize)) match {
case Success(s) => Some(s)
case Failure(e) =>
log.error(e, s"error loading snapshot ${dstFile(dir, snr)}")
go(snrs.tail)
}
}
go(decreasingSequenceNrs(dir))
}
private def serialize(outputStream: OutputStream, snapshot: Snapshot): Unit =
outputStream.write(settings.serialization.serialize(snapshot).get)
private def deserialize(inputStream: InputStream): Snapshot =
settings.serialization.deserialize(IOUtils.toByteArray(inputStream), classOf[Snapshot]).get
private def withOutputStream(dir: File, snr: Long)(body: OutputStream => Unit): Unit = {
val dst = dstFile(dir, snr)
val tmp = tmpFile(dir, snr)
dir.mkdirs()
withStream(new BufferedOutputStream(new FileOutputStream(tmp)), body)
tmp.renameTo(dst)
// do not keep more than the configured maximum number of snapshot files
decreasingSequenceNrs(dir).drop(settings.snapshotsPerEmitterMax).foreach { snr =>
dstFile(dir, snr).delete()
}
}
private def withInputStream[A](dir: File, snr: Long)(body: InputStream => A): A =
withStream(new BufferedInputStream(new FileInputStream(dstFile(dir, snr))), body)
private def withStream[A <: Closeable, B](stream: A, p: A => B): B =
try { p(stream) } finally { stream.close() }
private val DstFilenamePattern =
"""^snr-(\d+)""".r
private[eventuate] def dstDir(emitterId: String): File =
new File(rootDir, URLEncoder.encode(emitterId, "UTF-8"))
private[eventuate] def dstFile(dstDir: File, sequenceNr: Long): File =
new File(dstDir, s"snr-${sequenceNr}")
private[eventuate] def tmpFile(dstDir: File, sequenceNr: Long): File =
new File(dstDir, s"tmp-${sequenceNr}")
private[eventuate] def decreasingSequenceNrs(dir: File): Seq[Long] =
if (!dir.exists) Nil else dir.listFiles.map(_.getName).collect { case DstFilenamePattern(snr) => snr.toLong }.toList.sorted.reverse
}
Example 48
| Source File: stringUtils.scala From Converter with GNU General Public License v3.0 | 5 votes |
|
package org.scalablytyped.converter.internal
import java.net.URLEncoder
import scala.annotation.switch
object stringUtils {
val Quote = '"'
val QuoteStr = Quote.toString
def quote(s: String): String =
s"$Quote${EscapeStrings.java(s)}$Quote"
def escapeUnicodeEscapes(s: String): String =
s.replaceAll("\\\\u", "\\\\\\\\u")
def unCapitalize(str: String): String =
if (str.length == 0) ""
else if (str.charAt(0).isLower) str
else {
val chars = str.toCharArray
chars(0) = chars(0).toLower
new String(chars)
}
def joinCamelCase(strings: List[String]): String =
strings
.filterNot(_.isEmpty)
.zipWithIndex
.map {
case (x, 0) if x.length > 2 && x(0).isUpper && x(1).isUpper => x.toLowerCase //avoid things like dOM...
case (x, 0) => stringUtils.unCapitalize(x)
case (x, _) => x.capitalize
}
.mkString
def toCamelCase(str: String): String =
joinCamelCase(str.split("[_-]").toList)
// https://stackoverflow.com/a/611117
def encodeURIComponent(s: String): String =
URLEncoder
.encode(s, "UTF-8")
.replaceAll("\\+", "%20")
.replaceAll("\\%21", "!")
.replaceAll("\\%27", "'")
.replaceAll("\\%28", "(")
.replaceAll("\\%29", ")")
.replaceAll("\\%7E", "~")
}
Example 49
| Source File: GetAction.scala From Linkis with Apache License 2.0 | 5 votes |
|
abstract class GetAction extends HttpAction {
private val queryParams: util.Map[String, Any] = new util.HashMap[String, Any]
def setParameter(key: String, value: Any): Unit = this.queryParams.put(key, value)
def getParameters: util.Map[String, Any] = queryParams
override def getRequestBody: String = {
val queryString = new StringBuilder
JavaConversions.mapAsScalaMap(queryParams).foreach { case (k, v) =>
queryString.append(URLEncoder.encode(k, Configuration.BDP_ENCODING.getValue)).append("=")
.append(URLEncoder.encode(v.toString, Configuration.BDP_ENCODING.getValue)).append("&")
}
if(!queryParams.isEmpty) queryString.deleteCharAt(queryString.length - 1)
queryString.toString
}
}
Example 50
| Source File: WikipediaHeldoutCorpus.scala From dbpedia-spotlight-model with Apache License 2.0 | 5 votes |
|
package org.dbpedia.spotlight.io
import org.dbpedia.spotlight.model.{Text, DBpediaResourceOccurrence, AnnotatedParagraph}
import org.dbpedia.spotlight.db.{DBCandidateSearcher, WikipediaToDBpediaClosure}
import java.io.File
import io.Source
import org.dbpedia.spotlight.exceptions.NotADBpediaResourceException
import org.dbpedia.extraction.util.WikiUtil
import java.net.URLEncoder
class WikipediaHeldoutCorpus(val lines: Seq[String],
val wikiToDBpediaClosure: Option[WikipediaToDBpediaClosure],
val candidateSearcher: Option[DBCandidateSearcher]) extends AnnotatedTextSource {
override def foreach[U](f : AnnotatedParagraph => U) {
WikiOccurrenceSource.fromPigHeldoutFile(lines.iterator).groupBy(_.context).foreach {
m: (Text, Traversable[DBpediaResourceOccurrence]) =>
f(new AnnotatedParagraph(m._1, resolveRedirectsAndFilter(m._2)))
}
}
def resolveRedirectsAndFilter(occs: Traversable[DBpediaResourceOccurrence]): List[DBpediaResourceOccurrence] =
if(wikiToDBpediaClosure.isEmpty && candidateSearcher.isEmpty)
occs.toList
else
occs.flatMap({ occ: DBpediaResourceOccurrence =>
try {
occ.resource.uri = WikiUtil.wikiEncode(wikiToDBpediaClosure.get.wikipediaToDBpediaURI(occ.resource.uri))
if (candidateSearcher.get.getAmbiguity(occ.surfaceForm) > 1)
Some(occ)
else
None
} catch {
case e: NotADBpediaResourceException => None
}
}).toList
}
object WikipediaHeldoutCorpus {
def fromFile(corpus: File, wikiToDBpediaClosure: WikipediaToDBpediaClosure, candidateSearcher: DBCandidateSearcher): WikipediaHeldoutCorpus = {
new WikipediaHeldoutCorpus(Source.fromFile(corpus).getLines().toSeq, Option(wikiToDBpediaClosure), Option(candidateSearcher))
}
def fromFile(corpus: File): WikipediaHeldoutCorpus = fromFile(corpus, null, null)
}
Example 51
| Source File: DirManager.scala From daf with BSD 3-Clause "New" or "Revised" License | 5 votes |
|
package it.gov.daf.catalogmanager.listeners
import java.net.URLEncoder
import akka.actor.ActorSystem
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.{ FileIO, Source }
import net.caoticode.dirwatcher.FSListener
import play.api.libs.ws.WSClient
import play.api.libs.ws.ahc.AhcWSClient
import play.api.mvc.MultipartFormData.FilePart
import play.Logger
import scala.concurrent.Future
class DirManager() extends FSListener {
import java.nio.file.Path
import scala.concurrent.ExecutionContext.Implicits.global
val logger = Logger.underlying()
override def onCreate(ref: Path): Unit = {
implicit val system = ActorSystem()
implicit val materializer = ActorMaterializer()
val wsClient = AhcWSClient()
val name = ref.getParent.getFileName.toString
println(name)
val uri: Option[String] = IngestionUtils.datasetsNameUri.get(name)
val logicalUri = URLEncoder.encode(uri.get, "UTF-8")
logger.debug("logicalUri: " + logicalUri)
call(wsClient)
.andThen { case _ => wsClient.close() }
.andThen { case _ => system.terminate() }
def call(wsClient: WSClient): Future[Unit] = {
wsClient.url("http://localhost:9001/ingestion-manager/v1/add-datasets/" + logicalUri)
//.withHeaders("content-type" -> "multipart/form-data")
.post(
Source(FilePart("upfile", name, None, FileIO.fromPath(ref)) :: List())).map { response =>
val statusText: String = response.statusText
logger.debug(s"Got a response $statusText")
}
}
logger.debug(s"created $ref")
}
override def onDelete(ref: Path): Unit = println(s"deleted $ref")
override def onModify(ref: Path): Unit = println(s"modified $ref")
}
Example 52
| Source File: CatalogManagerClient.scala From daf with BSD 3-Clause "New" or "Revised" License | 5 votes |
|
package daf.catalogmanager
import java.net.URLEncoder
import java.security.AccessControlException
import it.gov.daf.common.config.Read
import json._
import org.slf4j.LoggerFactory
import play.api.Configuration
import play.api.libs.json.Json
import scalaj.http.{ Http, HttpResponse }
import scala.util.{ Failure, Try, Success => TrySuccess }
class CatalogManagerClient(serviceUrl: String) {
val logger = LoggerFactory.getLogger("it.gov.daf.CatalogManager")
private def callService(authorization: String, catalogId: String) = Try {
Http(s"$serviceUrl/catalog-manager/v1/catalog-ds/get/${URLEncoder.encode(catalogId,"UTF-8")}")
.header("Authorization", authorization)
.asString
}
private def parseCatalog(response: HttpResponse[String]) =
if (response.code == 401) Failure { new AccessControlException("Unauthorized") }
else if (response.isError) Failure { new RuntimeException(s"Error retrieving catalog data: [${response.code}] with body [${response.body}]") }
else Try { Json.parse(response.body).as[MetaCatalog] }
def getById(authorization: String, catalogId: String): Try[MetaCatalog] = for {
response <- callService(authorization, catalogId)
catalog <- parseCatalog(response)
} yield catalog
}
object CatalogManagerClient {
def fromConfig(config: Configuration) = Read.string { "daf.catalog_url" }.!.read(config) match {
case TrySuccess(baseUrl) => new CatalogManagerClient(baseUrl)
case Failure(error) => throw new RuntimeException("Unable to create catalog-manager client", error)
}
}
Example 53
| Source File: ExcludedAuthActionImpl.scala From nisp-frontend with Apache License 2.0 | 5 votes |
|
package uk.gov.hmrc.nisp.controllers.auth
import java.net.{URI, URLEncoder}
import com.google.inject.{ImplementedBy, Inject}
import play.api.mvc.Results.Redirect
import play.api.mvc.{ActionBuilder, ActionFunction, Request, Result}
import uk.gov.hmrc.auth.core._
import uk.gov.hmrc.auth.core.retrieve.v2.Retrievals
import uk.gov.hmrc.auth.core.retrieve.~
import uk.gov.hmrc.domain.Nino
import uk.gov.hmrc.http.HeaderCarrier
import uk.gov.hmrc.nisp.config.ApplicationConfig
import uk.gov.hmrc.play.HeaderCarrierConverter
import scala.concurrent.{ExecutionContext, Future}
class ExcludedAuthActionImpl @Inject()(override val authConnector: PlayAuthConnector)
(implicit ec: ExecutionContext) extends ExcludedAuthAction with AuthorisedFunctions {
override def invokeBlock[A](request: Request[A], block: ExcludedAuthenticatedRequest[A] => Future[Result]): Future[Result] = {
implicit val hc: HeaderCarrier = HeaderCarrierConverter.fromHeadersAndSession(request.headers, Some(request.session))
authorised(ConfidenceLevel.L200)
.retrieve(Retrievals.nino and Retrievals.confidenceLevel and Retrievals.credentials and Retrievals.loginTimes) {
case Some(nino) ~ confidenceLevel ~ credentials ~ loginTimes =>
block(ExcludedAuthenticatedRequest(request,
Nino(nino),
AuthDetails(confidenceLevel, credentials.map(creds => creds.providerType), loginTimes))
)
case _ => throw new RuntimeException("Can't find credentials for user")
} recover {
case _: NoActiveSession => Redirect(ApplicationConfig.ggSignInUrl, Map("continue" -> Seq(ApplicationConfig.postSignInRedirectUrl),
"origin" -> Seq("nisp-frontend"), "accountType" -> Seq("individual")))
case _: InsufficientConfidenceLevel => Redirect(ivUpliftURI.toURL.toString)
}
}
private val ivUpliftURI: URI =
new URI(s"${ApplicationConfig.ivUpliftUrl}?origin=NISP&" +
s"completionURL=${URLEncoder.encode(ApplicationConfig.postSignInRedirectUrl, "UTF-8")}&" +
s"failureURL=${URLEncoder.encode(ApplicationConfig.notAuthorisedRedirectUrl, "UTF-8")}" +
s"&confidenceLevel=200")
}
@ImplementedBy(classOf[ExcludedAuthActionImpl])
trait ExcludedAuthAction extends ActionBuilder[ExcludedAuthenticatedRequest] with ActionFunction[Request, ExcludedAuthenticatedRequest]
Example 54
| Source File: WaybackSpec.scala From ArchiveSpark with MIT License | 5 votes |
|
package org.archive.archivespark.specific.warc.specs
import java.net.URLEncoder
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import org.archive.archivespark.dataspecs.DataSpec
import org.archive.archivespark.sparkling.Sparkling
import org.archive.archivespark.sparkling.cdx.CdxRecord
import org.archive.archivespark.sparkling.util.{IteratorUtil, RddUtil, StringUtil}
import org.archive.archivespark.specific.warc.WaybackRecord
import scala.io.Source
class WaybackSpec (cdxServerUrl: String, pages: Int, maxPartitions: Int) extends DataSpec[String, WaybackRecord] {
override def load(sc: SparkContext, minPartitions: Int): RDD[String] = {
RddUtil.parallelize(pages, if (maxPartitions == 0) minPartitions else maxPartitions.min(minPartitions)).flatMap{page =>
try {
val source = Source.fromURL(cdxServerUrl + "&page=" + page)(StringUtil.codec(Sparkling.DefaultCharset))
IteratorUtil.cleanup(source.getLines, source.close)
} catch {
case e: Exception =>
e.printStackTrace()
Iterator.empty
}
}.cache
}
override def parse(data: String): Option[WaybackRecord] = CdxRecord.fromString(data).map(cdx => new WaybackRecord(cdx))
}
object WaybackSpec {
def apply(url: String, matchPrefix: Boolean = false, from: Long = 0, to: Long = 0, blocksPerPage: Int = 5, pages: Int = 50, maxPartitions: Int = 0): WaybackSpec = {
var cdxServerUrl = "http://web.archive.org/cdx/search/cdx?url=$url&matchType=$prefix&pageSize=$blocks"
cdxServerUrl = cdxServerUrl.replace("$url", URLEncoder.encode(url, "UTF-8"))
cdxServerUrl = cdxServerUrl.replace("$prefix", if (matchPrefix) "prefix" else "exact")
cdxServerUrl = cdxServerUrl.replace("$blocks", blocksPerPage.toString)
if (from > 0) cdxServerUrl += "&from=" + from
if (to > 0) cdxServerUrl += "&to=" + to
new WaybackSpec(cdxServerUrl, pages, maxPartitions)
}
}
Example 55
| Source File: package.scala From chronicler with Apache License 2.0 | 5 votes |
|
package com.github.fsanaulla.chronicler.akka.query
import java.net.URLEncoder
import com.github.fsanaulla.chronicler.core.model.InfluxCredentials
package object unit {
implicit class StringRich(val str: String) extends AnyVal {
def encode: String = URLEncoder.encode(str, "UTF-8")
}
def queryTesterAuth(query: String)(credentials: InfluxCredentials): String =
s"http://localhost:8086/query?q=${query.encode}&p=${credentials.password.encode}&u=${credentials.username.encode}"
def queryTesterAuth(db: String, query: String)(credentials: InfluxCredentials): String =
s"http://localhost:8086/query?q=${query.encode}&p=${credentials.password.encode}&db=${db.encode}&u=${credentials.username.encode}"
def queryTester(query: String): String =
s"http://localhost:8086/query?q=${query.encode}"
def queryTester(db: String, query: String): String =
s"http://localhost:8086/query?q=${query.encode}&db=${db.encode}"
def queryTester(path: String, queryParams: List[(String, String)]): String = {
val s = queryParams
.map {
case (k, v) => s"$k=${v.encode}"
}
.mkString("&")
s"http://localhost:8086$path?$s"
}
}
Example 56
| Source File: PoolTable.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.ui.jobs
import java.net.URLEncoder
import scala.collection.mutable.HashMap
import scala.xml.Node
import org.apache.spark.scheduler.{Schedulable, StageInfo}
import org.apache.spark.ui.UIUtils
private[ui] class PoolTable(pools: Seq[Schedulable], parent: StagesTab) {
private val listener = parent.progressListener
def toNodeSeq: Seq[Node] = {
listener.synchronized {
poolTable(poolRow, pools)
}
}
private def poolTable(
makeRow: (Schedulable, HashMap[String, HashMap[Int, StageInfo]]) => Seq[Node],
rows: Seq[Schedulable]): Seq[Node] = {
<table class="table table-bordered table-striped table-condensed sortable table-fixed">
<thead>
<th>Pool Name</th>
<th>Minimum Share</th>
<th>Pool Weight</th>
<th>Active Stages</th>
<th>Running Tasks</th>
<th>SchedulingMode</th>
</thead>
<tbody>
{rows.map(r => makeRow(r, listener.poolToActiveStages))}
</tbody>
</table>
}
private def poolRow(
p: Schedulable,
poolToActiveStages: HashMap[String, HashMap[Int, StageInfo]]): Seq[Node] = {
val activeStages = poolToActiveStages.get(p.name) match {
case Some(stages) => stages.size
case None => 0
}
val href = "%s/stages/pool?poolname=%s"
.format(UIUtils.prependBaseUri(parent.basePath), URLEncoder.encode(p.name, "UTF-8"))
<tr>
<td>
<a href={href}>{p.name}</a>
</td>
<td>{p.minShare}</td>
<td>{p.weight}</td>
<td>{activeStages}</td>
<td>{p.runningTasks}</td>
<td>{p.schedulingMode}</td>
</tr>
}
}
Example 57
| Source File: TestClient.scala From franklin with Apache License 2.0 | 5 votes |
|
package com.azavea.franklin.api
import cats.effect.Resource
import cats.effect.Sync
import cats.implicits._
import com.azavea.franklin.api.services.{CollectionItemsService, CollectionsService}
import com.azavea.stac4s.{StacCollection, StacItem}
import eu.timepit.refined.auto._
import io.circe.syntax._
import org.http4s.circe.CirceEntityDecoder._
import org.http4s.circe.CirceEntityEncoder._
import org.http4s.implicits._
import org.http4s.{Method, Request, Uri}
import java.net.URLEncoder
import java.nio.charset.StandardCharsets
class TestClient[F[_]: Sync](
collectionsService: CollectionsService[F],
collectionItemsService: CollectionItemsService[F]
) {
private def createCollection(collection: StacCollection): F[StacCollection] =
collectionsService.routes.orNotFound.run(
Request(
method = Method.POST,
uri = Uri.unsafeFromString("/collections")
).withEntity(collection.asJson)
) flatMap { _.as[StacCollection] }
private def deleteCollection(collection: StacCollection): F[Unit] = {
val encodedCollectionId = URLEncoder.encode(collection.id, StandardCharsets.UTF_8.toString)
collectionsService.routes.orNotFound
.run(
Request(
method = Method.DELETE,
uri = Uri.unsafeFromString(s"/collections/$encodedCollectionId")
)
)
.void
}
private def createItemInCollection(collection: StacCollection, item: StacItem): F[StacItem] = {
val encodedCollectionId = URLEncoder.encode(collection.id, StandardCharsets.UTF_8.toString)
collectionItemsService.routes.orNotFound.run(
Request(
method = Method.POST,
uri = Uri.unsafeFromString(s"/collections/$encodedCollectionId/items")
).withEntity(item)
) flatMap { _.as[StacItem] }
}
private def deleteItemInCollection(collection: StacCollection, item: StacItem): F[Unit] = {
val encodedCollectionId = URLEncoder.encode(collection.id, StandardCharsets.UTF_8.toString)
val encodedItemId = URLEncoder.encode(item.id, StandardCharsets.UTF_8.toString)
collectionItemsService.routes.orNotFound
.run(
Request(
method = Method.DELETE,
uri = Uri.unsafeFromString(s"/collections/$encodedCollectionId/items/$encodedItemId")
)
)
.void
}
def getItemResource(collection: StacCollection, item: StacItem): Resource[F, StacItem] =
Resource.make(createItemInCollection(collection, item.copy(collection = Some(collection.id))))(
item => deleteItemInCollection(collection, item)
)
def getCollectionResource(collection: StacCollection): Resource[F, StacCollection] =
Resource.make(createCollection(collection))(collection => deleteCollection(collection))
def getCollectionItemResource(
item: StacItem,
collection: StacCollection
): Resource[F, (StacItem, StacCollection)] =
(getItemResource(collection, item), getCollectionResource(collection)).tupled
}
Example 58
| Source File: CollectionsServiceSpec.scala From franklin with Apache License 2.0 | 5 votes |
|
package com.azavea.franklin.api.services
import cats.data.OptionT
import cats.effect.IO
import cats.implicits._
import com.azavea.franklin.Generators
import com.azavea.franklin.api.{TestClient, TestServices}
import com.azavea.franklin.database.TestDatabaseSpec
import com.azavea.franklin.datamodel.CollectionsResponse
import com.azavea.stac4s.StacCollection
import com.azavea.stac4s.testing._
import org.http4s.circe.CirceEntityDecoder._
import org.http4s.{Method, Request, Uri}
import org.specs2.{ScalaCheck, Specification}
import java.net.URLEncoder
import java.nio.charset.StandardCharsets
class CollectionsServiceSpec
extends Specification
with ScalaCheck
with TestDatabaseSpec
with Generators {
def is = s2"""
This specification verifies that the collections service can run without crashing
The collections service should:
- create and delete collections $createDeleteCollectionExpectation
- list collections $listCollectionsExpectation
- get collections by id $getCollectionsExpectation
"""
val testServices: TestServices[IO] = new TestServices[IO](transactor)
val testClient: TestClient[IO] =
new TestClient[IO](testServices.collectionsService, testServices.collectionItemsService)
def listCollectionsExpectation = prop {
(stacCollectionA: StacCollection, stacCollectionB: StacCollection) =>
{
val listIO = (
testClient.getCollectionResource(stacCollectionA),
testClient.getCollectionResource(stacCollectionB)
).tupled use { _ =>
val request = Request[IO](method = Method.GET, Uri.unsafeFromString(s"/collections"))
(for {
resp <- testServices.collectionsService.routes.run(request)
decoded <- OptionT.liftF { resp.as[CollectionsResponse] }
} yield decoded).value
}
val result = listIO.unsafeRunSync.get.collections map { _.id }
(result must contain(stacCollectionA.id)) and (result must contain(stacCollectionB.id))
}
}
def getCollectionsExpectation = prop { (stacCollection: StacCollection) =>
val fetchIO =
testClient.getCollectionResource(stacCollection) use { collection =>
val encodedId = URLEncoder.encode(collection.id, StandardCharsets.UTF_8.toString)
val request =
Request[IO](method = Method.GET, Uri.unsafeFromString(s"/collections/$encodedId"))
(for {
resp <- testServices.collectionsService.routes.run(request)
decoded <- OptionT.liftF { resp.as[StacCollection] }
} yield (decoded, collection)).value
}
val (fetched, inserted) = fetchIO.unsafeRunSync.get
fetched must beTypedEqualTo(inserted)
}
// since creation / deletion is a part of the collection resource, and accurate creation is checked
// in getCollectionsExpectation, this test just makes sure that if other tests are failing, it's
// not because create/delete are broken
def createDeleteCollectionExpectation = prop { (stacCollection: StacCollection) =>
(testClient
.getCollectionResource(stacCollection) use { _ => IO.unit }).unsafeRunSync must beTypedEqualTo(
()
)
}
}
Example 59
| Source File: FeatureExtractor.scala From franklin with Apache License 2.0 | 5 votes |
|
package com.azavea.franklin.crawler
import com.azavea.stac4s.StacLink
import com.azavea.stac4s.TwoDimBbox
import com.azavea.stac4s._
import geotrellis.vector.methods.Implicits._
import geotrellis.vector.{Feature, Geometry}
import io.circe.JsonObject
import java.net.URLEncoder
import java.nio.charset.StandardCharsets
import java.util.UUID
object FeatureExtractor {
def toItem(
feature: Feature[Geometry, JsonObject],
forItem: StacItem,
forItemCollection: String,
inCollection: StacCollection
): StacItem = {
val collectionHref =
s"/collections/${URLEncoder.encode(inCollection.id, StandardCharsets.UTF_8.toString)}"
val encodedSourceItemCollectionId =
URLEncoder.encode(forItemCollection, StandardCharsets.UTF_8.toString)
val sourceItemHref =
s"/collections/$encodedSourceItemCollectionId/items/${URLEncoder.encode(forItem.id, StandardCharsets.UTF_8.toString)}"
val collectionLink = StacLink(
collectionHref,
StacLinkType.Collection,
Some(`application/json`),
title = Some("Source item's original collection")
)
val sourceItemLink = StacLink(
sourceItemHref,
StacLinkType.VendorLinkType("derived_from"),
Some(`application/json`),
None
)
val featureExtent = feature.geom.extent
StacItem(
s"${UUID.randomUUID}",
"0.9.0",
Nil,
"Feature",
feature.geom,
TwoDimBbox(featureExtent.xmin, featureExtent.ymin, featureExtent.xmax, featureExtent.ymax),
links = List(collectionLink, sourceItemLink),
assets = Map.empty,
collection = Some(inCollection.id),
properties = feature.data
)
}
}
Example 60
| Source File: TileInfo.scala From franklin with Apache License 2.0 | 5 votes |
|
package com.azavea.franklin.datamodel
import cats.implicits._
import com.azavea.stac4s._
import io.circe.generic.JsonCodec
import java.net.URLEncoder
import java.nio.charset.StandardCharsets
@JsonCodec
case class TileInfo(
extent: StacExtent,
title: Option[String],
description: Option[String],
tileMatrixSetLinks: List[TileMatrixSetLink],
links: List[TileSetLink]
)
object TileInfo {
val webMercatorQuadLink = TileMatrixSetLink(
"WebMercatorQuad",
"http://schemas.opengis.net/tms/1.0/json/examples/WebMercatorQuad.json"
)
def fromStacItem(host: String, collectionId: String, item: StacItem): Option[TileInfo] = {
val spatialExtent = SpatialExtent(List(item.bbox))
val stacExtent = StacExtent(spatialExtent, Interval(List.empty))
val cogTileLinks = item.assets.collect {
case (key, asset) if asset._type === Some(`image/cog`) =>
val encodedItemId = URLEncoder.encode(item.id, StandardCharsets.UTF_8.toString)
val encodedKey = URLEncoder.encode(key, StandardCharsets.UTF_8.toString)
val href =
s"$host/tiles/collections/$collectionId/items/$encodedItemId/{tileMatrixSetId}/{tileMatrix}/{tileCol}/{tileRow}/?asset=$encodedKey"
val mediaType = Some(`image/png`)
TileSetLink(href, StacLinkType.Item, mediaType, None, Some(true))
}
cogTileLinks.isEmpty match {
case false =>
Some(TileInfo(stacExtent, None, None, List(webMercatorQuadLink), cogTileLinks.toList))
case _ => None
}
}
def fromStacCollection(host: String, collection: StacCollection): TileInfo = {
val mvtHref =
s"$host/tiles/collections/${collection.id}/footprint/{tileMatrixSetId}/{tileMatrix}/{tileCol}/{tileRow}"
val tileEndpointLink = TileSetLink(
mvtHref,
StacLinkType.VendorLinkType("tiles"),
Some(VendorMediaType("application/vnd.mapbox-vector-tile")),
Some(s"${collection.id} -- Footprints"),
Some(true)
)
val tileJsonHref =
s"$host/tiles/collections/${collection.id}/footprint/tile-json"
val tileJsonLink = TileSetLink(
tileJsonHref,
StacLinkType.VendorLinkType("tile-json"),
Some(`application/json`),
Some(s"${collection.id} -- Footprints TileJSON"),
Some(false)
)
TileInfo(
collection.extent,
collection.title map { title => s"$title - MVT" },
Some("Mapbox Vector Tile representation of item footprints for this collection"),
List(webMercatorQuadLink),
List(
tileEndpointLink,
tileJsonLink
)
)
}
}
Example 61
| Source File: SearchFilters.scala From franklin with Apache License 2.0 | 5 votes |
|
package com.azavea.franklin.database
import cats.implicits._
import com.azavea.franklin.api.schemas.bboxToString
import com.azavea.franklin.datamodel.{PaginationToken, Query}
import com.azavea.stac4s.{Bbox, TemporalExtent}
import eu.timepit.refined.types.numeric.NonNegInt
import geotrellis.vector.Geometry
import geotrellis.vector.{io => _, _}
import io.circe.generic.semiauto._
import io.circe.refined._
import io.circe.{Decoder, HCursor}
import java.net.URLEncoder
import java.nio.charset.StandardCharsets
final case class SearchFilters(
bbox: Option[Bbox],
datetime: Option[TemporalExtent],
intersects: Option[Geometry],
collections: List[String],
items: List[String],
limit: Option[NonNegInt],
query: Map[String, List[Query]],
next: Option[PaginationToken]
) {
def asQueryParameters: String = {
val bboxQP =
bbox map { box => s"bbox=${bboxToString(box)}" }
val datetimeQP =
datetime map { tempExtent =>
s"datetime=${SearchFilters.encodeString(temporalExtentToString(tempExtent))}"
}
val collectionsQP = collections.toNel map { _ =>
s"""collections=${SearchFilters.encodeString(collections.mkString(","))}"""
}
val itemsQP = items.toNel map { _ =>
s"""ids=${SearchFilters.encodeString(items.mkString(","))}"""
}
List(bboxQP, datetimeQP, collectionsQP, itemsQP).flatten.mkString("&")
}
}
object SearchFilters {
def encodeString(s: String): String = URLEncoder.encode(s, StandardCharsets.UTF_8.toString)
implicit val searchFilterDecoder = new Decoder[SearchFilters] {
final def apply(c: HCursor): Decoder.Result[SearchFilters] =
for {
bbox <- c.downField("bbox").as[Option[Bbox]]
datetime <- c.downField("datetime").as[Option[TemporalExtent]]
intersects <- c.downField("intersects").as[Option[Geometry]]
collectionsOption <- c.downField("collections").as[Option[List[String]]]
itemsOption <- c.downField("items").as[Option[List[String]]]
limit <- c.downField("limit").as[Option[NonNegInt]]
query <- c.get[Option[Map[String, List[Query]]]]("query")
paginationToken <- c.get[Option[PaginationToken]]("next")
} yield {
SearchFilters(
bbox,
datetime,
intersects,
collectionsOption.getOrElse(List.empty),
itemsOption.getOrElse(List.empty),
limit,
query getOrElse Map.empty,
paginationToken
)
}
}
implicit val searchFilterEncoder = deriveEncoder[SearchFilters]
}
Example 62
| Source File: security_api_yaml.scala From play-swagger with MIT License | 5 votes |
|
package security.api.yaml
import de.zalando.play.controllers._
import org.scalacheck._
import org.scalacheck.Arbitrary._
import org.scalacheck.Prop._
import org.scalacheck.Test._
import org.specs2.mutable._
import play.api.test.Helpers._
import play.api.test._
import play.api.mvc.MultipartFormData.FilePart
import play.api.mvc._
import org.junit.runner.RunWith
import org.specs2.runner.JUnitRunner
import java.net.URLEncoder
import com.fasterxml.jackson.databind.ObjectMapper
import play.api.http.Writeable
import play.api.libs.Files.TemporaryFile
import play.api.test.Helpers.{status => requestStatusCode_}
import play.api.test.Helpers.{contentAsString => requestContentAsString_}
import play.api.test.Helpers.{contentType => requestContentType_}
import de.zalando.play.controllers.ArrayWrapper
import Generators._
@RunWith(classOf[JUnitRunner])
class Security_api_yamlSpec extends Specification {
def toPath[T](value: T)(implicit binder: PathBindable[T]): String = Option(binder.unbind("", value)).getOrElse("")
def toQuery[T](key: String, value: T)(implicit binder: QueryStringBindable[T]): String = Option(binder.unbind(key, value)).getOrElse("")
def toHeader[T](value: T)(implicit binder: PathBindable[T]): String = Option(binder.unbind("", value)).getOrElse("")
def checkResult(props: Prop) =
Test.check(Test.Parameters.default, props).status match {
case Failed(args, labels) =>
val failureMsg = labels.mkString("\n") + " given args: " + args.map(_.arg).mkString("'", "', '","'")
failure(failureMsg)
case Proved(_) | Exhausted | Passed => success
case PropException(_, e, labels) =>
val error = if (labels.isEmpty) e.getLocalizedMessage() else labels.mkString("\n")
failure(error)
}
private def parserConstructor(mimeType: String) = PlayBodyParsing.jacksonMapper(mimeType)
def parseResponseContent[T](mapper: ObjectMapper, content: String, mimeType: Option[String], expectedType: Class[T]) =
mapper.readValue(content, expectedType)
}
Example 63
| Source File: instagram_api_yaml.scala From play-swagger with MIT License | 5 votes |
|
package instagram.api.yaml
import de.zalando.play.controllers._
import org.scalacheck._
import org.scalacheck.Arbitrary._
import org.scalacheck.Prop._
import org.scalacheck.Test._
import org.specs2.mutable._
import play.api.test.Helpers._
import play.api.test._
import play.api.mvc.MultipartFormData.FilePart
import play.api.mvc._
import org.junit.runner.RunWith
import org.specs2.runner.JUnitRunner
import java.net.URLEncoder
import com.fasterxml.jackson.databind.ObjectMapper
import play.api.http.Writeable
import play.api.libs.Files.TemporaryFile
import play.api.test.Helpers.{status => requestStatusCode_}
import play.api.test.Helpers.{contentAsString => requestContentAsString_}
import play.api.test.Helpers.{contentType => requestContentType_}
import scala.math.BigInt
import scala.math.BigDecimal
import Generators._
@RunWith(classOf[JUnitRunner])
class Instagram_api_yamlSpec extends Specification {
def toPath[T](value: T)(implicit binder: PathBindable[T]): String = Option(binder.unbind("", value)).getOrElse("")
def toQuery[T](key: String, value: T)(implicit binder: QueryStringBindable[T]): String = Option(binder.unbind(key, value)).getOrElse("")
def toHeader[T](value: T)(implicit binder: PathBindable[T]): String = Option(binder.unbind("", value)).getOrElse("")
def checkResult(props: Prop) =
Test.check(Test.Parameters.default, props).status match {
case Failed(args, labels) =>
val failureMsg = labels.mkString("\n") + " given args: " + args.map(_.arg).mkString("'", "', '","'")
failure(failureMsg)
case Proved(_) | Exhausted | Passed => success
case PropException(_, e, labels) =>
val error = if (labels.isEmpty) e.getLocalizedMessage() else labels.mkString("\n")
failure(error)
}
private def parserConstructor(mimeType: String) = PlayBodyParsing.jacksonMapper(mimeType)
def parseResponseContent[T](mapper: ObjectMapper, content: String, mimeType: Option[String], expectedType: Class[T]) =
mapper.readValue(content, expectedType)
}
Example 64
| Source File: UrlEncodedFormBody.scala From fintrospect with Apache License 2.0 | 5 votes |
|
package io.fintrospect.parameters
import java.net.{URLDecoder, URLEncoder}
import com.twitter.finagle.http._
import io.fintrospect.ContentTypes.APPLICATION_FORM_URLENCODED
import io.fintrospect.util.{Extraction, ExtractionError, ExtractionFailed, Extractor}
import scala.util.{Failure, Success, Try}
case class UrlEncodedFormBody(formContents: Seq[FormField[_] with Extractor[Form, _]],
validator: FormValidator, extractor: FormFieldExtractor)
extends Body[Form] {
override val contentType = APPLICATION_FORM_URLENCODED
override def iterator = formContents.iterator
private def decodeFields(content: String): Map[String, Seq[String]] = {
content
.split("&")
.filter(_.contains("="))
.map(nvp => {
val parts = nvp.split("=")
(URLDecoder.decode(parts(0), "UTF-8"), if (parts.length > 1) URLDecoder.decode(parts(1), "UTF-8") else "")
})
.groupBy(_._1)
.mapValues(_.map(_._2))
}
private def encode(form: Form): String = form.fields.flatMap {
case (name, values) => values.map(value => URLEncoder.encode(name, "UTF-8") + "=" + URLEncoder.encode(value, "UTF-8"))
}.mkString("&")
override def -->(value: Form): Seq[RequestBinding] =
Seq(new RequestBinding(null, req => {
val contentString = encode(value)
req.headerMap.add("Content-type", contentType.value)
req.headerMap.add("Content-length", contentString.length.toString)
req.contentString = contentString
req
})) ++ formContents.map(f => new FormFieldBinding(f, ""))
override def <--?(message: Message): Extraction[Form] =
Try(validator(formContents, new Form(decodeFields(message.contentString), Map.empty, Nil))) match {
case Success(form) => extractor(formContents, form)
case Failure(_) => ExtractionFailed(formContents.filter(_.required).map(param => ExtractionError(param, "Could not parse")))
}
}
Example 65
| Source File: package.scala From chronicler with Apache License 2.0 | 5 votes |
|
package com.github.fsanaulla.chronicler.ahc.io
import java.net.URLEncoder
import com.github.fsanaulla.chronicler.core.model.InfluxCredentials
package object unit {
implicit class StringRich(val str: String) extends AnyVal {
def encode: String = URLEncoder.encode(str, "UTF-8")
}
def queryTesterAuth(query: String)(credentials: InfluxCredentials): String =
s"http://localhost:8086/query?q=${query.encode}&p=${credentials.password.encode}&u=${credentials.username.encode}"
def queryTesterAuth(db: String, query: String)(credentials: InfluxCredentials): String =
s"http://localhost:8086/query?q=${query.encode}&p=${credentials.password.encode}&db=${db}&u=${credentials.username.encode}"
def queryTester(query: String): String =
s"http://localhost:8086/query?q=${query.encode}"
def queryTester(db: String, query: String): String =
s"http://localhost:8086/query?q=${query.encode}&db=${db}"
def queryTester(path: String, mp: List[(String, String)]): String = {
val queries = mp
.map { case (k, v) => s"$k=${v.encode}" }
.mkString("&")
s"http://localhost:8086$path?$queries"
}
}
Example 66
| Source File: package.scala From chronicler with Apache License 2.0 | 5 votes |
|
package com.github.fsanaulla.chronicler.ahc
import java.net.URLEncoder
import com.github.fsanaulla.chronicler.core.model.InfluxCredentials
package object management {
implicit class StringRich(val str: String) extends AnyVal {
def encode: String = URLEncoder.encode(str, "UTF-8")
}
def queryTesterAuth(query: String)(credentials: InfluxCredentials): String =
s"http://localhost:8086/query?u=${credentials.username.encode}&p=${credentials.password.encode}&q=${query.encode}"
def queryTesterAuth(db: String, query: String)(credentials: InfluxCredentials): String =
s"http://localhost:8086/query?db=${db}&u=${credentials.username.encode}&p=${credentials.password.encode}&q=${query.encode}"
def queryTester(query: String): String =
s"http://localhost:8086/query?q=${query.encode}"
def queryTester(db: String, query: String): String =
s"http://localhost:8086/query?db=${db}&q=${query.encode}"
def queryTester(path: String, queryPrms: List[(String, String)]): String = {
val s = queryPrms
.map { case (k, v) => s"$k=${v.encode}" }
.mkString("&")
s"http://localhost:8086$path?$s"
}
}
Example 67
| Source File: ApplicationController.scala From scuruto with MIT License | 5 votes |
|
package controller
import java.net.URLEncoder
import skinny._
import skinny.controller.feature.ScaldiFeature
import skinny.filter._
case _ => redirect302("/?ref=" + getRef)
}
}
set("loginUser" -> loginUser)
}
protected def getRef: String = {
val uri = request.getRequestURI
val query = request.getQueryString
val ref = {
if (query != null) {
uri + "?" + query
} else {
uri
}
}
if (ref == null) return ""
URLEncoder.encode(ref, "UTF-8")
}
}
Example 68
| Source File: WebhookBot.scala From telegram with Apache License 2.0 | 5 votes |
|
import java.net.URLEncoder
import akka.http.scaladsl.Http
import akka.http.scaladsl.model.{HttpRequest, Uri}
import akka.http.scaladsl.unmarshalling.Unmarshal
import com.bot4s.telegram.api.Webhook
import com.bot4s.telegram.methods._
import com.bot4s.telegram.models.Message
import scala.concurrent.Future
class WebhookBot(token: String) extends AkkaExampleBot(token) with Webhook {
val port = 8080
val webhookUrl = "https://88c444ab.ngrok.io"
val baseUrl = "http://api.mathjs.org/v1/?expr="
override def receiveMessage(msg: Message): Future[Unit] = {
msg.text.fold(Future.successful(())) { text =>
val url = baseUrl + URLEncoder.encode(text, "UTF-8")
for {
res <- Http().singleRequest(HttpRequest(uri = Uri(url)))
if res.status.isSuccess()
result <- Unmarshal(res).to[String]
_ <- request(SendMessage(msg.source, result))
} yield ()
}
}
}
Example 69
| Source File: QrCodesBot.scala From telegram with Apache License 2.0 | 5 votes |
|
import java.net.URLEncoder
import akka.http.scaladsl.Http
import akka.http.scaladsl.model.{HttpRequest, Uri}
import akka.http.scaladsl.unmarshalling.Unmarshal
import akka.util.ByteString
import com.bot4s.telegram.api.declarative.Commands
import com.bot4s.telegram.api._
import com.bot4s.telegram.future.Polling
import com.bot4s.telegram.methods._
import com.bot4s.telegram.models.AkkaInputFile
import scala.concurrent.Future
class QrCodesBot(token: String) extends AkkaExampleBot(token)
with Polling
with Commands[Future]
with ChatActions[Future] {
// Multiple variants
onCommand('qr | 'qrcode | 'qr_code) { implicit msg =>
withArgs { args =>
val url = "https://api.qrserver.com/v1/create-qr-code/?data=" +
URLEncoder.encode(args mkString " ", "UTF-8")
for {
response <- Http().singleRequest(HttpRequest(uri = Uri(url)))
if response.status.isSuccess()
bytes <- Unmarshal(response).to[ByteString]
photo = AkkaInputFile("qrcode.png", bytes)
_ <- uploadingPhoto // Hint the user
_ <- request(SendPhoto(msg.source, photo))
} yield ()
}
}
}
Example 70
| Source File: PoolTable.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.ui.jobs
import java.net.URLEncoder
import scala.collection.mutable.HashMap
import scala.xml.Node
import org.apache.spark.scheduler.{Schedulable, StageInfo}
import org.apache.spark.ui.UIUtils
private[ui] class PoolTable(pools: Seq[Schedulable], parent: StagesTab) {
private val listener = parent.progressListener
def toNodeSeq: Seq[Node] = {
listener.synchronized {
poolTable(poolRow, pools)
}
}
private def poolTable(
makeRow: (Schedulable, HashMap[String, HashMap[Int, StageInfo]]) => Seq[Node],
rows: Seq[Schedulable]): Seq[Node] = {
<table class="table table-bordered table-striped table-condensed sortable table-fixed">
<thead>
<th>Pool Name</th>
<th>Minimum Share</th>
<th>Pool Weight</th>
<th>Active Stages</th>
<th>Running Tasks</th>
<th>SchedulingMode</th>
</thead>
<tbody>
{rows.map(r => makeRow(r, listener.poolToActiveStages))}
</tbody>
</table>
}
private def poolRow(
p: Schedulable,
poolToActiveStages: HashMap[String, HashMap[Int, StageInfo]]): Seq[Node] = {
val activeStages = poolToActiveStages.get(p.name) match {
case Some(stages) => stages.size
case None => 0
}
val href = "%s/stages/pool?poolname=%s"
.format(UIUtils.prependBaseUri(parent.basePath), URLEncoder.encode(p.name, "UTF-8"))
<tr>
<td>
<a href={href}>{p.name}</a>
</td>
<td>{p.minShare}</td>
<td>{p.weight}</td>
<td>{activeStages}</td>
<td>{p.runningTasks}</td>
<td>{p.schedulingMode}</td>
</tr>
}
}
Example 71
| Source File: SessionSerializer.scala From akka-http-session with Apache License 2.0 | 5 votes |
|
package com.softwaremill.session
import java.net.{URLDecoder, URLEncoder}
import scala.util.Try
trait SessionSerializer[T, R] {
def serialize(t: T): R
def deserialize(r: R): Try[T]
def deserializeV0_5_2(r: R): Try[T] = deserialize(r)
}
class SingleValueSessionSerializer[T, V](toValue: T => V, fromValue: V => Try[T])(
implicit
valueSerializer: SessionSerializer[V, String])
extends SessionSerializer[T, String] {
override def serialize(t: T) = valueSerializer.serialize(toValue(t))
override def deserialize(r: String) = valueSerializer.deserialize(r).flatMap(fromValue)
}
class MultiValueSessionSerializer[T](toMap: T => Map[String, String], fromMap: Map[String, String] => Try[T])
extends SessionSerializer[T, String] {
import SessionSerializer._
override def serialize(t: T) =
toMap(t)
.map { case (k, v) => urlEncode(k) + "~" + urlEncode(v) }
.mkString("&")
override def deserialize(s: String) = {
Try {
if (s == "") Map.empty[String, String]
else {
s.split("&")
.map(_.split("~", 2))
.map(p => urlDecode(p(0)) -> urlDecode(p(1)))
.toMap
}
}.flatMap(fromMap)
}
override def deserializeV0_5_2(s: String) = {
Try {
if (s == "") Map.empty[String, String]
else {
s.split("&")
.map(_.split("=", 2))
.map(p => urlDecode(p(0)) -> urlDecode(p(1)))
.toMap
}
}.flatMap(fromMap)
}
}
object SessionSerializer {
implicit def stringToStringSessionSerializer: SessionSerializer[String, String] =
new SessionSerializer[String, String] {
override def serialize(t: String) = urlEncode(t)
override def deserialize(s: String) = Try(urlDecode(s))
}
implicit def intToStringSessionSerializer: SessionSerializer[Int, String] = new SessionSerializer[Int, String] {
override def serialize(t: Int) = urlEncode(t.toString)
override def deserialize(s: String) = Try(urlDecode(s).toInt)
}
implicit def longToStringSessionSerializer: SessionSerializer[Long, String] = new SessionSerializer[Long, String] {
override def serialize(t: Long) = urlEncode(t.toString)
override def deserialize(s: String) = Try(urlDecode(s).toLong)
}
implicit def floatToStringSessionSerializer: SessionSerializer[Float, String] = new SessionSerializer[Float, String] {
override def serialize(t: Float) = urlEncode(t.toString)
override def deserialize(s: String) = Try(urlDecode(s).toFloat)
}
implicit def doubleToStringSessionSerializer: SessionSerializer[Double, String] =
new SessionSerializer[Double, String] {
override def serialize(t: Double) = urlEncode(t.toString)
override def deserialize(s: String) = Try(urlDecode(s).toDouble)
}
implicit def mapToStringSessionSerializer: SessionSerializer[Map[String, String], String] =
new MultiValueSessionSerializer[Map[String, String]](identity, Try(_))
private[session] def urlEncode(s: String): String = URLEncoder.encode(s, "UTF-8")
private[session] def urlDecode(s: String): String = URLDecoder.decode(s, "UTF-8")
}
