java.net.URL Scala Examples

// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package com.daml.navigator.test

import java.util.concurrent.atomic.AtomicReference

import com.daml.navigator.test.config.Arguments
import com.daml.navigator.test.runner.{HeadNavigator, PackagedDamlc, PackagedSandbox}
import com.typesafe.scalalogging.LazyLogging

import scala.io.Source
import scala.util.{Failure, Success, Try}

class ComponentsFixture(
    val args: Arguments,
    val navigatorPort: Int,
    val sandboxPort: Int,
    val scenario: String
) extends LazyLogging {

  // A list of commands on how to destroy started processes
  private val killProcs: AtomicReference[List[Unit => Unit]] = new AtomicReference(List.empty)

  private val onlineUrl = s"http://localhost:$navigatorPort/api/about"

  private def get(
      url: String,
      connectTimeout: Int = 1000,
      readTimeout: Int = 1000,
      requestMethod: String = "GET"
  ): String = {
    import java.net.{URL, HttpURLConnection}
    val connection = (new URL(url)).openConnection.asInstanceOf[HttpURLConnection]
    connection.setConnectTimeout(connectTimeout)
    connection.setReadTimeout(readTimeout)
    connection.setRequestMethod(requestMethod)
    val inputStream = connection.getInputStream
    val content = Source.fromInputStream(inputStream).mkString
    if (inputStream != null) inputStream.close()
    content
  }

  def startup(): Try[Unit] = {
    if (args.startComponents) {
      logger.info("Starting the sandbox and the Navigator")
      for {
        (darFile, tempFiles) <- Try(PackagedDamlc.run(args.damlPath))
        sandbox <- Try(PackagedSandbox.runAsync(sandboxPort, darFile, scenario))
        _ = killProcs.updateAndGet(s => sandbox :: s)
        navigator <- Try(
          HeadNavigator.runAsync(args.navConfPAth, args.navigatorDir, navigatorPort, sandboxPort))
        _ = killProcs.updateAndGet(s => navigator :: s)
      } yield { () }
    } else {
      Success(())
    }
  }

  private def retry[R](action: => R, maxRetries: Int, delayMillis: Int): Try[R] = {
    def retry0(count: Int): Try[R] = {
      Try(action) match {
        case Success(r) => Success(r)
        case Failure(e) =>
          if (count > maxRetries) {
            logger.error(
              s"Navigator is not available after $maxRetries retries with $delayMillis millis interval.")
            Failure(e)
          } else {
            logger.info(s"Navigator is not available yet, waiting $delayMillis millis ")
            Thread.sleep(delayMillis.toLong)
            retry0(count + 1)
          }
      }
    }

    retry0(0)
  }

  def waitForNavigator(): Try[Unit] = {
    logger.info(s"Waiting for the Navigator to start up (waiting for $onlineUrl)")
    retry({ get(onlineUrl); () }, 120, 1000)
  }

  def shutdown(): Unit = {
    killProcs.getAndUpdate(procs => {
      procs.foreach(killAction => Try { killAction(()) })
      List.empty
    })
    ()
  }
} 

package featherbed.circe

import cats.implicits._
import com.twitter.util.Future
import io.circe._
import io.circe.generic.auto._
import io.circe.parser.parse
import io.circe.syntax._
import org.scalatest.FlatSpec
import shapeless.{Coproduct, Witness}
import shapeless.union.Union

case class Foo(someText: String, someInt: Int)

class CirceSpec extends FlatSpec {

  "post request of a case class" should "derive JSON encoder" in {

    import com.twitter.util.{Future, Await}
    import com.twitter.finagle.{Service, Http}
    import com.twitter.finagle.http.{Request, Response}
    import java.net.InetSocketAddress

    val server = Http.serve(new InetSocketAddress(8766), new Service[Request, Response] {
      def apply(request: Request): Future[Response] = Future {
        val rep = Response()
        rep.contentString = s"${request.contentString}"
        rep.setContentTypeJson()
        rep
      }
    })

    import java.net.URL
    val client = new featherbed.Client(new URL("http://localhost:8766/api/"))

    import io.circe.generic.auto._

    val req = client.post("foo/bar")
      .withContent(Foo("Hello world!", 42), "application/json")
        .accept[Coproduct.`"application/json"`.T]

    val result = Await.result {
       req.send[Foo]()
    }

    Foo("test", 42).asJson.toString

    parse("""{"someText": "test", "someInt": 42}""").toValidated.map(_.as[Foo])

    Await.result(server.close())

  }

  "API example" should "compile" in {
    import shapeless.Coproduct
    import java.net.URL
    import com.twitter.util.Await
    case class Post(userId: Int, id: Int, title: String, body: String)

    case class Comment(postId: Int, id: Int, name: String, email: String, body: String)
    class JSONPlaceholderAPI(baseUrl: URL) {

      private val client = new featherbed.Client(baseUrl)
      type JSON = Coproduct.`"application/json"`.T

      object posts {

        private val listRequest = client.get("posts").accept[JSON]
        private val getRequest = (id: Int) => client.get(s"posts/$id").accept[JSON]

        def list(): Future[Seq[Post]] = listRequest.send[Seq[Post]]()
        def get(id: Int): Future[Post] = getRequest(id).send[Post]()
      }

      object comments {
        private val listRequest = client.get("comments").accept[JSON]
        private val getRequest = (id: Int) => client.get(s"comments/$id").accept[JSON]

        def list(): Future[Seq[Comment]] = listRequest.send[Seq[Comment]]()
        def get(id: Int): Future[Comment] = getRequest(id).send[Comment]()
      }
    }

    val apiClient = new JSONPlaceholderAPI(new URL("http://jsonplaceholder.typicode.com/"))

    Await.result(apiClient.posts.list())
  }

} 

package org.apache.gearpump.experiments.yarn.client

import java.io.IOException
import java.net.{HttpURLConnection, URL}
import java.nio.charset.StandardCharsets
import akka.actor.{ActorRef, ActorSystem}
import org.apache.commons.io.IOUtils
import org.apache.gearpump.experiments.yarn.glue.Records.{ApplicationId, ApplicationReport}
import org.apache.gearpump.experiments.yarn.glue.YarnClient
import org.apache.gearpump.util.{AkkaHelper, LogUtil}
import org.apache.hadoop.hdfs.web.URLConnectionFactory
import org.apache.hadoop.yarn.conf.YarnConfiguration
import scala.util.Try


class AppMasterResolver(yarnClient: YarnClient, system: ActorSystem) {
  val LOG = LogUtil.getLogger(getClass)
  val RETRY_INTERVAL_MS = 3000 // ms

  def resolve(appId: ApplicationId, timeoutSeconds: Int = 30): ActorRef = {
    val appMaster = retry(connect(appId), 1 + timeoutSeconds * 1000 / RETRY_INTERVAL_MS)
    appMaster
  }

  private def connect(appId: ApplicationId): ActorRef = {
    val report = yarnClient.getApplicationReport(appId)

    AppMasterResolver.resolveAppMasterAddress(report, system)
  }

  private def retry(fun: => ActorRef, times: Int): ActorRef = {
    var index = 0
    var result: ActorRef = null
    while (index < times && result == null) {
      Thread.sleep(RETRY_INTERVAL_MS)
      index += 1
      val tryConnect = Try(fun)
      if (tryConnect.isFailure) {
        LOG.error(s"Failed to connect YarnAppMaster(tried $index)... " +
          tryConnect.failed.get.getMessage)
      } else {
        result = tryConnect.get
      }
    }
    result
  }
}

object AppMasterResolver {
  val LOG = LogUtil.getLogger(getClass)

  def resolveAppMasterAddress(report: ApplicationReport, system: ActorSystem): ActorRef = {
    val appMasterPath = s"${report.getTrackingURL}/supervisor-actor-path"
    LOG.info(s"appMasterPath=$appMasterPath")

    val connectionFactory: URLConnectionFactory = URLConnectionFactory
      .newDefaultURLConnectionFactory(new YarnConfiguration())
    val url: URL = new URL(appMasterPath)
    val connection: HttpURLConnection = connectionFactory.openConnection(url)
      .asInstanceOf[HttpURLConnection]
    connection.setInstanceFollowRedirects(true)

    try {
      connection.connect()
    } catch {
      case e: IOException =>
        LOG.error(s"Failed to connect to AppMaster" + e.getMessage)
    }

    val status = connection.getResponseCode
    if (status == 200) {
      val stream: java.io.InputStream = connection.getInputStream
      val response = IOUtils.toString(stream, StandardCharsets.UTF_8)
      LOG.info("Successfully resolved AppMaster address: " + response)
      connection.disconnect()
      AkkaHelper.actorFor(system, response)
    } else {
      connection.disconnect()
      throw new IOException("Fail to resolve AppMaster address, please make sure " +
        s"${report.getTrackingURL} is accessible...")
    }
  }
} 

package org.apache.spark.deploy

import java.net.URL

import scala.collection.mutable
import scala.io.Source

import org.apache.spark.{LocalSparkContext, SparkContext, SparkFunSuite}
import org.apache.spark.scheduler.{SparkListener, SparkListenerExecutorAdded}
import org.apache.spark.scheduler.cluster.ExecutorInfo
import org.apache.spark.util.SparkConfWithEnv

class LogUrlsStandaloneSuite extends SparkFunSuite with LocalSparkContext {

  
  private val WAIT_TIMEOUT_MILLIS = 10000

  test("verify that correct log urls get propagated from workers") {
    sc = new SparkContext("local-cluster[2,1,1024]", "test")

    val listener = new SaveExecutorInfo
    sc.addSparkListener(listener)

    // Trigger a job so that executors get added
    sc.parallelize(1 to 100, 4).map(_.toString).count()

    sc.listenerBus.waitUntilEmpty(WAIT_TIMEOUT_MILLIS)
    listener.addedExecutorInfos.values.foreach { info =>
      assert(info.logUrlMap.nonEmpty)
      // Browse to each URL to check that it's valid
      info.logUrlMap.foreach { case (logType, logUrl) =>
        val html = Source.fromURL(logUrl).mkString
        assert(html.contains(s"$logType log page"))
      }
    }
  }

  test("verify that log urls reflect SPARK_PUBLIC_DNS (SPARK-6175)") {
    val SPARK_PUBLIC_DNS = "public_dns"
    val conf = new SparkConfWithEnv(Map("SPARK_PUBLIC_DNS" -> SPARK_PUBLIC_DNS)).set(
      "spark.extraListeners", classOf[SaveExecutorInfo].getName)
    sc = new SparkContext("local-cluster[2,1,1024]", "test", conf)

    // Trigger a job so that executors get added
    sc.parallelize(1 to 100, 4).map(_.toString).count()

    sc.listenerBus.waitUntilEmpty(WAIT_TIMEOUT_MILLIS)
    val listeners = sc.listenerBus.findListenersByClass[SaveExecutorInfo]
    assert(listeners.size === 1)
    val listener = listeners(0)
    listener.addedExecutorInfos.values.foreach { info =>
      assert(info.logUrlMap.nonEmpty)
      info.logUrlMap.values.foreach { logUrl =>
        assert(new URL(logUrl).getHost === SPARK_PUBLIC_DNS)
      }
    }
  }
}

private[spark] class SaveExecutorInfo extends SparkListener {
  val addedExecutorInfos = mutable.Map[String, ExecutorInfo]()

  override def onExecutorAdded(executor: SparkListenerExecutorAdded) {
    addedExecutorInfos(executor.executorId) = executor.executorInfo
  }
} 

package org.apache.spark.util

import java.net.{URL, URLClassLoader}
import java.util.Enumeration

import scala.collection.JavaConverters._


private[spark] class ChildFirstURLClassLoader(urls: Array[URL], parent: ClassLoader)
  extends MutableURLClassLoader(urls, null) {

  private val parentClassLoader = new ParentClassLoader(parent)

  override def loadClass(name: String, resolve: Boolean): Class[_] = {
    try {
      super.loadClass(name, resolve)
    } catch {
      case e: ClassNotFoundException =>
        parentClassLoader.loadClass(name, resolve)
    }
  }

  override def getResource(name: String): URL = {
    val url = super.findResource(name)
    val res = if (url != null) url else parentClassLoader.getResource(name)
    res
  }

  override def getResources(name: String): Enumeration[URL] = {
    val childUrls = super.findResources(name).asScala
    val parentUrls = parentClassLoader.getResources(name).asScala
    (childUrls ++ parentUrls).asJavaEnumeration
  }

  override def addURL(url: URL) {
    super.addURL(url)
  }

} 

package org.apache.spark.sql.hive

import java.net.URL

import org.apache.spark.SparkFunSuite


class ClasspathDependenciesSuite extends SparkFunSuite {
  private val classloader = this.getClass.getClassLoader

  private def assertLoads(classname: String): Unit = {
    val resourceURL: URL = Option(findResource(classname)).getOrElse {
      fail(s"Class $classname not found as ${resourceName(classname)}")
    }

    logInfo(s"Class $classname at $resourceURL")
    classloader.loadClass(classname)
  }

  private def findResource(classname: String): URL = {
    val resource = resourceName(classname)
    classloader.getResource(resource)
  }

  private def resourceName(classname: String): String = {
    classname.replace(".", "/") + ".class"
  }

  private def assertClassNotFound(classname: String): Unit = {
    Option(findResource(classname)).foreach { resourceURL =>
      fail(s"Class $classname found at $resourceURL")
    }

    intercept[ClassNotFoundException] {
      classloader.loadClass(classname)
    }
  }

  test("shaded Protobuf") {
    assertLoads("org.apache.hive.com.google.protobuf.ServiceException")
  }

  test("shaded Kryo") {
    assertLoads("org.apache.hive.com.esotericsoftware.kryo.Kryo")
  }

  test("hive-common") {
    assertLoads("org.apache.hadoop.hive.conf.HiveConf")
  }

  test("hive-exec") {
    assertLoads("org.apache.hadoop.hive.ql.CommandNeedRetryException")
  }

  private val STD_INSTANTIATOR = "org.objenesis.strategy.StdInstantiatorStrategy"

  test("Forbidden Dependencies") {
    assertClassNotFound("com.esotericsoftware.shaded." + STD_INSTANTIATOR)
    assertClassNotFound("org.apache.hive.com.esotericsoftware.shaded." + STD_INSTANTIATOR)
  }

  test("parquet-hadoop-bundle") {
    assertLoads("parquet.hadoop.ParquetOutputFormat")
    assertLoads("parquet.hadoop.ParquetInputFormat")
  }
} 

package com.lightbend.lagom.internal.log4j2

import java.io.File
import java.net.URL

import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.core.LoggerContext
import org.apache.logging.log4j.core.config.Configurator
import org.slf4j.ILoggerFactory
import org.slf4j.impl.StaticLoggerBinder
import play.api._

object Log4j2LoggerConfigurator {
  private final val DevLog4j2Config: String     = "log4j2-lagom-dev.xml"
  private final val DefaultLog4j2Config: String = "log4j2-lagom-default.xml"
}

class Log4j2LoggerConfigurator extends LoggerConfigurator {
  import Log4j2LoggerConfigurator._

  override def loggerFactory: ILoggerFactory = {
    StaticLoggerBinder.getSingleton.getLoggerFactory
  }

  override def init(rootPath: File, mode: Mode): Unit = {
    val properties   = Map("application.home" -> rootPath.getAbsolutePath)
    val resourceName = if (mode == Mode.Dev) DevLog4j2Config else DefaultLog4j2Config
    val resourceUrl  = Option(this.getClass.getClassLoader.getResource(resourceName))
    configure(properties, resourceUrl)
  }

  override def configure(env: Environment): Unit =
    configure(env, Configuration.empty, Map.empty)

  override def configure(
      env: Environment,
      configuration: Configuration,
      optionalProperties: Map[String, String]
  ): Unit = {
    val properties = LoggerConfigurator.generateProperties(env, configuration, optionalProperties)
    configure(properties, configUrl(env))
  }

  override def configure(properties: Map[String, String], config: Option[URL]): Unit = {
    if (config.isEmpty) {
      System.err.println("Could not detect a log4j2 configuration file, not configuring log4j2")
    }

    config.foreach { url =>
      val context = LogManager.getContext(false).asInstanceOf[LoggerContext]
      context.setConfigLocation(url.toURI)
    }
  }

  private def configUrl(env: Environment) = {
    // Get an explicitly configured resource URL
    // Fallback to a file in the conf directory if the resource wasn't found on the classpath
    def explicitResourceUrl = sys.props.get("logger.resource").flatMap { r =>
      env.resource(r).map(_.toURI.toURL)
    }

    // Get an explicitly configured file URL
    def explicitFileUrl = sys.props.get("logger.file").map(new File(_).toURI.toURL)

    // log4j2.xml is the documented method, log4j2-lagom-default.xml is the fallback that Lagom uses
    // if no other file is found
    def resourceUrl =
      env
        .resource("log4j2.xml")
        .orElse(
          env.resource(
            if (env.mode == Mode.Dev) DevLog4j2Config else DefaultLog4j2Config
          )
        )

    explicitResourceUrl.orElse(explicitFileUrl).orElse(resourceUrl)
  }

  override def shutdown(): Unit = LogManager.shutdown()
} 

package org.apache.livy

import java.io.{Closeable, File, InputStreamReader}
import java.net.URL
import java.nio.charset.StandardCharsets.UTF_8
import java.security.SecureRandom
import java.util.Properties

import scala.annotation.tailrec
import scala.collection.JavaConverters._
import scala.concurrent.TimeoutException
import scala.concurrent.duration.Duration

import org.apache.commons.codec.binary.Base64

object Utils {
  def getPropertiesFromFile(file: File): Map[String, String] = {
    loadProperties(file.toURI().toURL())
  }

  def loadProperties(url: URL): Map[String, String] = {
    val inReader = new InputStreamReader(url.openStream(), UTF_8)
    try {
      val properties = new Properties()
      properties.load(inReader)
      properties.stringPropertyNames().asScala.map { k =>
        (k, properties.getProperty(k).trim())
      }.toMap
    } finally {
      inReader.close()
    }
  }

  
  def isProcessAlive(process: Process): Boolean = {
    try {
      process.exitValue()
      false
    } catch {
      case _: IllegalThreadStateException =>
        true
    }
  }

  def startDaemonThread(name: String)(f: => Unit): Thread = {
    val thread = new Thread(name) {
      override def run(): Unit = f
    }
    thread.setDaemon(true)
    thread.start()
    thread
  }

  def usingResource[A <: Closeable, B](resource: A)(f: A => B): B = {
    try {
      f(resource)
    } finally {
      resource.close()
    }
  }

  def createSecret(secretBitLength: Int): String = {
    val rnd = new SecureRandom()
    val secretBytes = new Array[Byte](secretBitLength / java.lang.Byte.SIZE)
    rnd.nextBytes(secretBytes)

    Base64.encodeBase64String(secretBytes)
  }
} 

package org.apache.toree.dependencies

import java.net.URL
import java.nio.file.Files

import org.scalatest.{FunSpec, Matchers, OneInstancePerTest}

class CoursierDependencyDownloaderSpec extends FunSpec with Matchers
  with OneInstancePerTest
{
  private val coursierDependencyDownloader = new CoursierDependencyDownloader

  describe("CoursierDependencyDownloader") {
    describe("#addMavenRepository") {
      it("should add to the list of repositories") {
        val repo = new URL("http://some-repo.com")

        coursierDependencyDownloader.addMavenRepository(repo, None)

        val repos = coursierDependencyDownloader.getRepositories

        repos should contain (repo.toURI)
      }
    }

    describe("#removeMavenRepository") {
      it("should remove from the list of repositories") {
        val repo = new URL("http://some-repo.com")

        coursierDependencyDownloader.addMavenRepository(repo, None)
        coursierDependencyDownloader.removeMavenRepository(repo)

        val repos = coursierDependencyDownloader.getRepositories

        repos should not contain (repo.toURI)
      }
    }

    describe("#setDownloadDirectory") {
      it("should set the new download directory if valid") {
        val validDir = Files.createTempDirectory("tmpdir").toFile
        validDir.deleteOnExit()

        val result = coursierDependencyDownloader.setDownloadDirectory(validDir)
        result should be (true)

        val dir = coursierDependencyDownloader.getDownloadDirectory
        dir should be (validDir.getAbsolutePath)
      }

      it("should not change the directory if given a file") {
        val invalidDir = Files.createTempFile("tmp", "file").toFile
        invalidDir.deleteOnExit()

        val result = coursierDependencyDownloader.setDownloadDirectory(invalidDir)
        result should be (false)

        val dir = coursierDependencyDownloader.getDownloadDirectory
        dir should not be (invalidDir.getAbsolutePath)
      }

      it("should support creating missing directories") {
        val baseDir = Files.createTempDirectory("tmpdir").toFile
        val validDir = baseDir.toPath.resolve("otherdir").toFile
        validDir.deleteOnExit()
        baseDir.deleteOnExit()

        val result = coursierDependencyDownloader.setDownloadDirectory(validDir)
        result should be (true)

        val dir = coursierDependencyDownloader.getDownloadDirectory
        dir should be (validDir.getAbsolutePath)
      }
    }

    describe("#getRepositories") {
      it("should have the default repositories") {
        val expected = Seq(DependencyDownloader.DefaultMavenRepository.toURI)

        val actual = coursierDependencyDownloader.getRepositories

        actual should be (expected)
      }
    }

    describe("#getDownloadDirectory") {
      it("should have the default download directory") {
        val expected = DependencyDownloader.DefaultDownloadDirectory.getAbsolutePath

        val actual = coursierDependencyDownloader.getDownloadDirectory

        actual should be (expected)
      }
    }
  }
} 

package org.apache.toree.utils

import java.io.FileNotFoundException
import java.net.URL

import org.scalatest.{BeforeAndAfter, Matchers, FunSpec}
import scala.io.Source
import scala.tools.nsc.io.File

class DownloadSupportSpec extends FunSpec with Matchers with BeforeAndAfter {
  val downloadDestinationUrl = new URL("file:///tmp/testfile2.ext")

  val testFileContent = "This is a test"
  val testFileName = "/tmp/testfile.txt"

  //  Create a test file for downloading
  before {
    File(testFileName).writeAll(testFileContent)
  }

  //  Cleanup what we made
  after {
    if (File(testFileName).exists) File(testFileName).delete()
    if (File(downloadDestinationUrl.getPath).exists) File(downloadDestinationUrl.getPath).delete()
  }

  describe("DownloadSupport"){
    describe("#downloadFile( String, String )"){
      it("should download a file to the download directory"){
        val testFileUrl = "file:///tmp/testfile.txt"

        //  Create our utility and download the file
        val downloader = new Object with DownloadSupport
        downloader.downloadFile(
          testFileUrl,
          downloadDestinationUrl.getProtocol + "://" +
            downloadDestinationUrl.getPath)

        //  Verify the file contents are what was in the original file
        val downloadedFileContent: String =
          Source.fromFile(downloadDestinationUrl.getPath).mkString

        downloadedFileContent should be (testFileContent)
      }

    }

    describe("#downloadFile( URL, URL )"){
      it("should download a file to the download directory"){
        val testFileUrl = new URL("file:///tmp/testfile.txt")

        val downloader = new Object with DownloadSupport
        downloader.downloadFile(testFileUrl, downloadDestinationUrl)

        //  Verify the file contents are what was in the original file
        val downloadedFileContent: String =
          Source.fromFile(downloadDestinationUrl.getPath).mkString

        downloadedFileContent should be (testFileContent)
      }

      it("should throw FileNotFoundException if the download URL is bad"){
        val badFilename = "file:///tmp/testbadfile.txt"
        if (File(badFilename).exists) File(badFilename).delete()

        val badFileUrl = new URL(badFilename)

        val downloader = new Object with DownloadSupport
        intercept[FileNotFoundException] {
          downloader.downloadFile(badFileUrl, downloadDestinationUrl)
        }
      }

      it("should throw FileNotFoundException if the download ") {
        val testFileUrl = new URL("file:///tmp/testfile.txt")
        val badDestinationUrl =
          new URL("file:///tmp/badloc/that/doesnt/exist.txt")

        val downloader = new Object with DownloadSupport
        intercept[FileNotFoundException] {
          downloader.downloadFile(testFileUrl, badDestinationUrl)
        }
      }
    }
  }

} 

package org.apache.toree.magic.builtin

import java.io.OutputStream
import java.net.URL

import org.apache.toree.interpreter.Interpreter
import org.apache.toree.magic.dependencies.{IncludeOutputStream, IncludeInterpreter}
import org.apache.toree.magic.{CellMagic, LineMagic}
import org.apache.spark.SparkContext
import org.scalatest.{Matchers, FunSpec}
import org.scalatest.mock.MockitoSugar

import org.mockito.Mockito._
import org.mockito.Matchers._

class TestLSMagic(sc: SparkContext, intp: Interpreter, os: OutputStream)
  extends LSMagic
  with IncludeInterpreter
  with IncludeOutputStream
  {
    override val interpreter: Interpreter = intp
    override val outputStream: OutputStream = os
  }

class LSMagicSpec extends FunSpec with Matchers with MockitoSugar {
  describe("LSMagic") {

    describe("#execute") {
      it("should call println with a magics message") {
        val lsm = spy(new TestLSMagic(
          mock[SparkContext], mock[Interpreter], mock[OutputStream])
        )
        val classList = new BuiltinLoader().loadClasses()
        lsm.execute("")
        verify(lsm).magicNames("%", classOf[LineMagic], classList)
        verify(lsm).magicNames("%%", classOf[CellMagic], classList)
      }
    }

    describe("#magicNames") {
      it("should filter classnames by interface") {
        val prefix = "%"
        val interface = classOf[LineMagic]
        val classes : List[Class[_]] = List(classOf[LSMagic], classOf[Integer])
        val lsm = new TestLSMagic(
          mock[SparkContext], mock[Interpreter], mock[OutputStream])
        lsm.magicNames(prefix, interface, classes).length should be(1)
      }
      it("should prepend prefix to each name"){
        val prefix = "%"
        val className = classOf[LSMagic].getSimpleName
        val interface = classOf[LineMagic]
        val expected = s"${prefix}${className}"
        val classes : List[Class[_]] = List(classOf[LSMagic], classOf[Integer])
        val lsm = new TestLSMagic(
          mock[SparkContext], mock[Interpreter], mock[OutputStream])
        lsm.magicNames(prefix, interface, classes) should be(List(expected))
      }
    }

  }

} 

package org.apache.toree.magic.builtin

import java.io.{File, PrintStream}
import java.net.{URL, URI}
import java.nio.file.{Files, Paths}
import java.util.zip.ZipFile
import org.apache.toree.magic._
import org.apache.toree.magic.builtin.AddJar._
import org.apache.toree.magic.dependencies._
import org.apache.toree.utils.{ArgumentParsingSupport, DownloadSupport, LogLike, FileUtils}
import com.typesafe.config.Config
import org.apache.hadoop.fs.Path
import org.apache.toree.plugins.annotations.Event

object AddJar {
  val HADOOP_FS_SCHEMES = Set("hdfs", "s3", "s3n", "file")

  private var jarDir:Option[String] = None

  def getJarDir(config: Config): String = {
    jarDir.getOrElse({
      jarDir = Some(
        if(config.hasPath("jar_dir") && Files.exists(Paths.get(config.getString("jar_dir")))) {
          config.getString("jar_dir")
        } else {
          FileUtils.createManagedTempDirectory("toree_add_jars").getAbsolutePath
        }
      )
      jarDir.get
    })
  }
}

class AddJar
  extends LineMagic with IncludeInterpreter
  with IncludeOutputStream with DownloadSupport with ArgumentParsingSupport
  with IncludeKernel with IncludePluginManager with IncludeConfig with LogLike
{
  // Option to mark re-downloading of jars
  private val _force =
    parser.accepts("f", "forces re-download of specified jar")

  // Option to mark re-downloading of jars
  private val _magic =
    parser.accepts("magic", "loads jar as a magic extension")

  // Lazy because the outputStream is not provided at construction
  private def printStream = new PrintStream(outputStream)

  )
      } else {
        downloadFile(
          new URL(jarRemoteLocation),
          new File(downloadLocation).toURI.toURL
        )
      }

      // Report download finished
      printStream.println(s"Finished download of $jarName")
    } else {
      printStream.println(s"Using cached version of $jarName")
    }

    // validate jar file
    if(! isValidJar(fileDownloadLocation)) {
      throw new IllegalArgumentException(s"Jar '$jarName' is not valid.")
    }

    if (_magic) {
      val plugins = pluginManager.loadPlugins(fileDownloadLocation)
      pluginManager.initializePlugins(plugins)
    } else {
      kernel.addJars(fileDownloadLocation.toURI)
    }
  }
} 

package org.apache.toree.magic.builtin

import java.io.{File, PrintStream}
import java.net.URL

import org.apache.toree.dependencies.Credentials
import org.apache.toree.magic._
import org.apache.toree.magic.dependencies._
import org.apache.toree.utils.ArgumentParsingSupport

import scala.util.Try
import org.apache.toree.plugins.annotations.Event


class AddDeps extends LineMagic with IncludeInterpreter
  with IncludeOutputStream with ArgumentParsingSupport
  with IncludeDependencyDownloader with IncludeKernel
{

  private def printStream = new PrintStream(outputStream)

  private val _transitive = parser.accepts(
    "transitive", "Retrieve dependencies recursively"
  )

  private val _verbose = parser.accepts(
    "verbose", "Prints out additional information"
  )

  private val _trace = parser.accepts(
    "trace", "Prints out trace of download progress"
  )

  private val _abortOnResolutionErrors = parser.accepts(
    "abort-on-resolution-errors", "Abort (no downloads) when resolution fails"
  )

  private val _exclude = parser.accepts("exclude", "exclude dependency").withRequiredArg().ofType(classOf[String])

  private val _repository = parser.accepts(
    "repository", "Adds an additional repository to available list"
  ).withRequiredArg().ofType(classOf[String])

  private val _credentials = parser.accepts(
    "credential", "Adds a credential file to be used to the list"
  ).withRequiredArg().ofType(classOf[String])

  private val _configuration = parser.accepts(
    "ivy-configuration", "Sets the Ivy configuration for the dependency; defaults to \"default\""
  ).withRequiredArg().ofType(classOf[String])

  private val _classifier = parser.accepts(
    "classifier", "Sets the dependency's classifier"
  ).withRequiredArg().ofType(classOf[String])

  
  @Event(name = "adddeps")
  override def execute(code: String): Unit = {
    val nonOptionArgs = parseArgs(code)
    dependencyDownloader.setPrintStream(printStream)

    val repository = getAll(_repository).getOrElse(Nil)
    val credentials = getAll(_credentials).getOrElse(Nil)
    val excludes = getAll(_exclude).getOrElse(Nil)

    val excludesSet = excludes.map((x: String) => {
      if (x.contains(":")) {
        (x.split(":")(0), x.split(":")(1))
      } else {
        (x, "*")
      }
    }: (String, String)).toSet

    val repositoriesWithCreds = dependencyDownloader.resolveRepositoriesAndCredentials(repository, credentials)

    if (nonOptionArgs.size == 3) {
      // get the jars and hold onto the paths at which they reside
      val uris = dependencyDownloader.retrieve(
        groupId                 = nonOptionArgs.head,
        artifactId              = nonOptionArgs(1),
        version                 = nonOptionArgs(2),
        transitive              = _transitive,
        ignoreResolutionErrors  = !_abortOnResolutionErrors,
        extraRepositories       = repositoriesWithCreds,
        verbose                 = _verbose,
        trace                   = _trace,
        excludes                = excludesSet,
        configuration           = get(_configuration),
        artifactClassifier      = get(_classifier)
      )

      // pass the new Jars to the kernel
      kernel.addJars(uris.filter(_.getPath.endsWith(".jar")): _*)
    } else {
      printHelp(printStream, """%AddDeps my.company artifact-id version""")
    }
  }


} 

package org.apache.spark.toree.test.utils

import sys.process._
import java.net.URL
import java.io.File
import java.util.UUID

import scala.language.postfixOps

import org.apache.spark.TestUtils.{JavaSourceFromString, _}

object JarUtils {

  def createTemporaryDir() = {
    val tempDir = System.getProperty("java.io.tmpdir")
    val dir = new File(tempDir, UUID.randomUUID.toString)
    dir.mkdirs()
    dir.deleteOnExit()
    dir.getCanonicalFile
  }

  def createDummyJar(destDir: String, packageName: String, className: String) : URL = {
    val srcDir = new File(destDir, packageName)
    srcDir.mkdirs()
    val source =
        s"""package $packageName;
            |
            |public class $className implements java.io.Serializable {
            |  public static String sayHello(String arg) { return "Hello, " + arg; }
            |  public static int addStuff(int arg1, int arg2) { return arg1 + arg2; }
            |}
         """.stripMargin

    val sourceFile =
      new JavaSourceFromString(new File(srcDir, className).toURI.getPath, source)
    val compiledFile = createCompiledClass(className, srcDir, sourceFile, Seq.empty)
    val jarFile = new File(destDir,
        s"$packageName-$className-%s.jar".format(System.currentTimeMillis()))
    val jarURL = createJar(Seq(compiledFile), jarFile, directoryPrefix = Some(packageName))
    jarFile.deleteOnExit()
    jarURL
  }

  def downloadJar(destDir: String, artifactURL: String) : URL = {
    val fileName = getFileName(artifactURL).
      replace(".jar", s"%s.jar".format(System.currentTimeMillis()))
    val jarFile = new File(destDir, fileName)
    jarFile.deleteOnExit()
    (new URL(artifactURL) #> jarFile !!)
    jarFile.toURI.toURL
  }

  private def getFileName(artifactURL: String) = {
    artifactURL.split("/").last
  }
} 

package com.bfm.topnotch.tnengine

import java.net.URL

import com.bfm.topnotch.SparkApplicationTester
import org.scalatest.{Tag, Matchers}

/**
 * The tests for [[com.bfm.topnotch.tnengine.TnWriter TnWriter]].
 */
class TnWriterTest extends SparkApplicationTester with Matchers {

  /**
   * The tags
   */
  object getWriterTag extends Tag("getWriter")

  lazy val fileReader = new TnFileReader
  lazy val engine = new TnEngine(spark)

  "getWriter" should "return an HDFS writer when given a config missing the io namespace" taggedAs (getWriterTag) in {
    engine.getWriter(fileReader.readConfiguration("src/test/resources/com/bfm/topnotch/tnengine/emptyPlan.json")) shouldBe a [TnHDFSWriter]
  }

  it should "return an HDFS writer when given HDFS as the config string with no path" taggedAs (getWriterTag) in {
    engine.getWriter(fileReader.readConfiguration("src/test/resources/com/bfm/topnotch/tnengine/writer/hdfsNoFile.json")) shouldBe a [TnHDFSWriter]
  }

  it should "return an HDFS writer with a non-default destination when given hdfs as the writer with a destination string" taggedAs (getWriterTag) in {
    val writer = engine.getWriter(fileReader.readConfiguration("src/test/resources/com/bfm/topnotch/tnengine/writer/hdfsWithFile.json"))
    writer shouldBe a [TnHDFSWriter]
    writer.asInstanceOf[TnHDFSWriter].dest.get shouldBe "/user/testUser/"
  }

  it should "return an Hbase writer when given HBase as the config string" taggedAs (getWriterTag) in {
    engine.getWriter(fileReader.readConfiguration("src/test/resources/com/bfm/topnotch/tnengine/writer/hbase.json")) shouldBe a [TnHBaseWriter]
  }

  it should "return a REST writer with a non-default URL when given rest as the writer with a destination string" taggedAs (getWriterTag) in {
    val writer = engine.getWriter(fileReader.readConfiguration("src/test/resources/com/bfm/topnotch/tnengine/writer/rest.json"))
    writer shouldBe a [TnRESTWriter]
    writer.asInstanceOf[TnRESTWriter].dest shouldBe "http://www.testurl.com"
  }
} 

package sjsonnew

import java.util.{ UUID, Optional }
import java.net.{ URI, URL }
import java.io.File
import java.math.{ BigInteger, BigDecimal => JBigDecimal }

trait JavaExtraFormats {
  this: PrimitiveFormats with AdditionalFormats with IsoFormats =>

  private[this] type JF[A] = JsonFormat[A] // simple alias for reduced verbosity

  implicit val javaBigIntegerFormat: JF[BigInteger] =
    projectFormat[BigInteger, BigInt](BigInt.apply, _.bigInteger)

  implicit val javaBigDecimalFormat: JF[JBigDecimal] =
    projectFormat[JBigDecimal, BigDecimal](BigDecimal.apply, _.bigDecimal)

  implicit val uuidStringIso: IsoString[UUID] = IsoString.iso[UUID](
    _.toString, UUID.fromString)
  implicit val uriStringIso: IsoString[URI] = IsoString.iso[URI](
    _.toASCIIString, new URI(_))
  implicit val urlStringIso: IsoString[URL] = IsoString.iso[URL](
    _.toURI.toASCIIString, (s: String) => (new URI(s)).toURL)

  private[this] final val FileScheme = "file"

  implicit val fileStringIso: IsoString[File] = IsoString.iso[File](
    (f: File) => {
      if (f.isAbsolute) {
        f.toPath.toUri.toASCIIString
      } else {
        new URI(null, normalizeName(f.getPath), null).toASCIIString
      }
    },
    (s: String) => uriToFile(new URI(s)))

  private[this] def normalizeName(name: String) = {
    val sep = File.separatorChar
    if (sep == '/') name else name.replace(sep, '/')
  }

  private[this] def uriToFile(uri: URI): File = {
    val part = uri.getSchemeSpecificPart
    // scheme might be omitted for relative URI reference.
    assert(
      Option(uri.getScheme) match {
        case None | Some(FileScheme) => true
        case _                       => false
      },
      s"Expected protocol to be '$FileScheme' or empty in URI $uri"
    )
    Option(uri.getAuthority) match {
      case None if part startsWith "/" => new File(uri)
      case _                           =>
        if (!(part startsWith "/") && (part contains ":")) new File("//" + part)
        else new File(part)
    }
  }

  implicit def optionalFormat[A :JF]: JF[Optional[A]] = new OptionalFormat[A]
  final class OptionalFormat[A :JF] extends JF[Optional[A]] {
    lazy val elemFormat = implicitly[JF[A]]
    def write[J](o: Optional[A], builder: Builder[J]): Unit =
      if (o.isPresent) elemFormat.write(o.get, builder)
      else builder.writeNull
    override def addField[J](name: String, o: Optional[A], builder: Builder[J]): Unit =
      if (o.isPresent) {
        builder.addFieldName(name)
        write(o, builder)
      } else ()
    def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Optional[A] =
      jsOpt match {
        case Some(js) =>
          if (unbuilder.isJnull(js)) Optional.empty[A]
          else Optional.ofNullable(elemFormat.read(jsOpt, unbuilder))
        case None => Optional.empty[A]
      }
  }
} 

package sjsonnew
package support.spray

import spray.json.{ JsValue, JsNumber, JsString, JsNull, JsTrue, JsFalse, JsObject }
import org.specs2.mutable._
import java.util.{ UUID, Optional }
import java.net.{ URI, URL }
import java.io.File

class JavaExtraFormatsSpec extends Specification with BasicJsonProtocol {
  case class Person(name: Optional[String], value: Optional[Int])
  implicit object PersonFormat extends JsonFormat[Person] {
    def write[J](x: Person, builder: Builder[J]): Unit = {
      builder.beginObject()
      builder.addField("name", x.name)
      builder.addField("value", x.value)
      builder.endObject()
    }
    def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Person =
      jsOpt match {
        case Some(js) =>
          unbuilder.beginObject(js)
          val name = unbuilder.readField[Optional[String]]("name")
          val value = unbuilder.readField[Optional[Int]]("value")
          unbuilder.endObject()
          Person(name, value)
        case None =>
          deserializationError("Expected JsObject but found None")
      }
  }

  "The uuidStringIso" should {
    val uuid = UUID.fromString("abc220ea-2a01-11e6-b67b-9e71128cae77")
    "convert a UUID to JsString" in {
      Converter.toJsonUnsafe(uuid) mustEqual JsString("abc220ea-2a01-11e6-b67b-9e71128cae77")
    }
    "convert the JsString back to the UUID" in {
      Converter.fromJsonUnsafe[UUID](JsString("abc220ea-2a01-11e6-b67b-9e71128cae77")) mustEqual uuid
    }
  }

  "The uriStringIso" should {
    val uri = new URI("http://localhost")
    "convert a URI to JsString" in {
      Converter.toJsonUnsafe(uri) mustEqual JsString("http://localhost")
    }
    "convert the JsString back to the URI" in {
      Converter.fromJsonUnsafe[URI](JsString("http://localhost")) mustEqual uri
    }
  }

  "The urlStringIso" should {
    val url = new URL("http://localhost")
    "convert a URL to JsString" in {
      Converter.toJsonUnsafe(url) mustEqual JsString("http://localhost")
    }
    "convert the JsString back to the URI" in {
      Converter.fromJsonUnsafe[URL](JsString("http://localhost")) mustEqual url
    }
  }

  "The fileStringIso" should {
    val f = new File("/tmp")
    val f2 = new File(new File("src"), "main")
    "convert a File to JsString" in {
      Converter.toJsonUnsafe(f) mustEqual JsString("file:///tmp/")
    }
    "convert a relative path to JsString" in {
      // https://tools.ietf.org/html/rfc3986#section-4.2
      Converter.toJsonUnsafe(f2) mustEqual JsString("src/main")
    }
    "convert the JsString back to the File" in {
      Converter.fromJsonUnsafe[File](JsString("file:///tmp/")) mustEqual f
    }
    "convert the JsString back to the relative path" in {
      Converter.fromJsonUnsafe[File](JsString("src/main")) mustEqual f2
    }
  }

  "The optionalFormat" should {
    "convert Optional.empty to JsNull" in {
      Converter.toJsonUnsafe(Optional.empty[Int]) mustEqual JsNull
    }
    "convert JsNull to None" in {
      Converter.fromJsonUnsafe[Optional[Int]](JsNull) mustEqual Optional.empty[Int]
    }
    "convert Some(Hello) to JsString(Hello)" in {
      Converter.toJsonUnsafe(Optional.of("Hello")) mustEqual JsString("Hello")
    }
    "convert JsString(Hello) to Some(Hello)" in {
      Converter.fromJsonUnsafe[Optional[String]](JsString("Hello")) mustEqual Optional.of("Hello")
    }
    "omit None fields" in {
      Converter.toJsonUnsafe(Person(Optional.empty[String], Optional.empty[Int])) mustEqual JsObject()
    }
  }
} 

package com.eclipsesource.schema

import java.net.URL

import com.eclipsesource.schema.drafts.Version4
import org.specs2.mutable.Specification
import play.api.libs.json.{JsValue, Json}

class SimplePerformanceSpec extends Specification {

  import Version4._

  def timed(name: String)(body: => Unit) {
    val start = System.currentTimeMillis()
    body
    println(name + ": " + (System.currentTimeMillis() - start) + " ms")
  }

  val validator = SchemaValidator(Some(Version4))
  val schemaUrl: URL = getClass.getResource("/issue-99-1.json")
  val schema: SchemaType = JsonSource.schemaFromUrl(schemaUrl).get

  val instance: JsValue = Json.parse("""{ "mything": "the thing" }""".stripMargin)

  timed("preloaded") {
    for (_ <- 1 to 1000) validator.validate(schema, instance)
  }
  timed("url based") {
    for (_ <- 1 to 1000) validator.validate(schemaUrl)(instance)
  }

} 

package com.eclipsesource.schema

import java.io.InputStream
import java.net.URL

import play.api.libs.json._

import scala.io.Source
import scala.util.{Failure, Success, Try}


  def schemaFromUrl(url: URL)(implicit reads: Reads[SchemaType]): JsResult[SchemaType] = {
    for {
      schemaJson <- JsonSource.fromUrl(url) match {
        case Success(json) => JsSuccess(json)
        case Failure(throwable) => JsError(throwable.getMessage)
      }
      schema <- Json.fromJson[SchemaType](schemaJson)
    } yield schema
  }
} 

package com.stratio.sparta.serving.core.utils

import java.io.File
import java.net.URL
import java.util.{Calendar, UUID}

import akka.event.slf4j.SLF4JLogging
import com.stratio.sparta.serving.core.helpers.JarsHelper
import org.apache.commons.io.FileUtils

trait PluginsFilesUtils extends SLF4JLogging {

  def addPluginsToClassPath(pluginsFiles: Array[String]): Unit = {
    log.info(pluginsFiles.mkString(","))
    pluginsFiles.foreach(filePath => {
      log.info(s"Adding to classpath plugin file: $filePath")
      if (filePath.startsWith("/") || filePath.startsWith("file://")) addFromLocal(filePath)
      if (filePath.startsWith("hdfs")) addFromHdfs(filePath)
      if (filePath.startsWith("http")) addFromHttp(filePath)
    })
  }

  private def addFromLocal(filePath: String): Unit = {
    log.info(s"Getting file from local: $filePath")
    val file = new File(filePath.replace("file://", ""))
    JarsHelper.addToClasspath(file)
  }

  private def addFromHdfs(fileHdfsPath: String): Unit = {
    log.info(s"Getting file from HDFS: $fileHdfsPath")
    val inputStream = HdfsUtils().getFile(fileHdfsPath)
    val fileName = fileHdfsPath.split("/").last
    log.info(s"HDFS file name is $fileName")
    val file = new File(s"/tmp/sparta/userjars/${UUID.randomUUID().toString}/$fileName")
    log.info(s"Downloading HDFS file to local file system: ${file.getAbsoluteFile}")
    FileUtils.copyInputStreamToFile(inputStream, file)
    JarsHelper.addToClasspath(file)
  }

  private def addFromHttp(fileURI: String): Unit = {
    log.info(s"Getting file from HTTP: $fileURI")
    val tempFile = File.createTempFile(s"sparta-plugin-${Calendar.getInstance().getTimeInMillis}", ".jar")
    val url = new URL(fileURI)
    FileUtils.copyURLToFile(url, tempFile)
    JarsHelper.addToClasspath(tempFile)
  }
} 

package com.krux.hyperion.contrib.activity.notification

import java.net.{ HttpURLConnection, URL }

import org.json4s.JsonAST.{ JString, JObject }
import org.json4s.jackson.JsonMethods._
import scopt.OptionParser

object SendSlackMessage {
  case class Options(
    failOnError: Boolean = false,
    webhookUrl: String = "",
    user: Option[String] = None,
    message: Seq[String] = Seq.empty,
    iconEmoji: Option[String] = None,
    channel: Option[String] = None
  )

  def apply(options: Options): Boolean = try {
    // Setup the connection
    val connection = new URL(options.webhookUrl).openConnection().asInstanceOf[HttpURLConnection]
    connection.setDoOutput(true)
    connection.setRequestProperty("Content-Type", "application/json")
    connection.setRequestProperty("Accept", "application/json")

    // Write the message
    val output = connection.getOutputStream
    try {
      val message = Seq(
        "icon_emoji" -> options.iconEmoji,
        "channel" -> options.channel,
        "username" -> options.user,
        "text" -> Option(options.message.mkString("\n"))
      ).flatMap {
        case (k, None) => None
        case (k, Some(v)) => Option(k -> JString(v))
      }

      output.write(compact(render(JObject(message: _*))).getBytes)
    } finally {
      output.close()
    }

    // Check the response code
    connection.getResponseCode == 200 || !options.failOnError
  } catch {
    case e: Throwable =>
      System.err.println(e.toString)
      !options.failOnError
  }

  def main(args: Array[String]): Unit = {
    val parser = new OptionParser[Options](s"hyperion-notification-slack-activity") {
      override def showUsageOnError = true

      note("Sends a notification message to a Slack incoming webhook.")
      help("help").text("prints this usage text")
      opt[Unit]("fail-on-error").optional().action((_, c) => c.copy(failOnError = true))
        .text("Causes the activity to fail if any error received from the webhook")
      opt[String]("webhook-url").valueName("WEBHOOK").required().action((x, c) => c.copy(webhookUrl = x))
        .text("Sends the message to the given WEBHOOK url")
      opt[String]("user").valueName("NAME").optional().action((x, c) => c.copy(user = Option(x)))
        .text("Sends the message as the user with NAME")
      opt[String]("emoji").valueName("EMOJI").optional().action((x, c) => c.copy(iconEmoji = Option(x)))
        .text("Use EMOJI for the icon")
      opt[String]("to").valueName("CHANNEL or USERNAME").optional().action((x, c) => c.copy(channel = Option(x)))
        .text("Sends the message to #CHANNEL or @USERNAME")
      arg[String]("MESSAGE").required().unbounded().action((x, c) => c.copy(message = c.message :+ x))
        .text("Sends the given MESSAGE")
    }

    if (!parser.parse(args, Options()).exists(apply)) {
      System.exit(3)
    }
  }
} 

package com.krux.hyperion.contrib.activity.notification

import java.net.{ HttpURLConnection, URL }

import org.json4s.JsonDSL._
import org.json4s.jackson.JsonMethods._
import scopt.OptionParser

object SendFlowdockMessage {
  case class Options(
    failOnError: Boolean = false,
    apiKey: String = "",
    message: String = "",
    user: String = "hyperion",
    tags: Seq[String] = Seq.empty
  )

  def apply(options: Options): Boolean = try {
    // Setup the connection
    val connection = new URL(s"https://api.flowdock.com/v1/messages/chat/${options.apiKey}")
      .openConnection().asInstanceOf[HttpURLConnection]
    connection.setDoOutput(true)
    connection.setRequestProperty("Content-Type", "application/json")
    connection.setRequestProperty("Accept", "application/json")

    // Write the message
    val output = connection.getOutputStream
    try {
      output.write(compact(render(("event" -> "message") ~
        ("external_user_name" -> options.user) ~
        ("content" -> options.message) ~
        ("tags" -> options.tags))).getBytes)
    } finally {
      output.close()
    }

    // Check the response code
    connection.getResponseCode == 200 || !options.failOnError
  } catch {
    case e: Throwable =>
      System.err.println(e.toString)
      !options.failOnError
  }

  def main(args: Array[String]): Unit = {
    val parser = new OptionParser[Options](s"hyperion-notification-flowdock-activity") {
      override def showUsageOnError = true

      note("Sends a notification message to a Flowdock flow.")
      help("help").text("prints this usage text")
      opt[Unit]("fail-on-error").optional().action((_, c) => c.copy(failOnError = true))
      opt[String]("api-key").valueName("KEY").required().action((x, c) => c.copy(apiKey = x))
        .text("Sends the given TEXT as the subject")
      opt[String]("user").valueName("NAME").optional().action((x, c) => c.copy(user = x))
        .text("Sends the message as the user with NAME")
      opt[Seq[String]]("tags").valueName("TAG1,TAG2").optional().action((x, c) => c.copy(tags = x))
        .text("Adds the tags to the message")
      arg[String]("MESSAGE").required().unbounded().action((x, c) => c.copy(message = s"${c.message} $x"))
        .text("Sends the given MESSAGE")
    }

    if (!parser.parse(args, Options()).exists(apply)) {
      System.exit(3)
    }
  }
} 

package com.wix.e2e.http.drivers

import java.io.DataOutputStream
import java.net.{HttpURLConnection, URL}

import akka.http.scaladsl.model.HttpMethods.GET
import akka.http.scaladsl.model._
import akka.http.scaladsl.model.headers.`Transfer-Encoding`
import akka.stream.scaladsl.Source
import com.wix.e2e.http.client.extractors._
import com.wix.e2e.http.info.HttpTestkitVersion
import com.wix.e2e.http.matchers.{RequestMatcher, ResponseMatcher}
import com.wix.e2e.http.{BaseUri, HttpRequest, RequestHandler}
import com.wix.test.random._

import scala.collection.immutable
import scala.collection.mutable.ListBuffer

trait HttpClientTestSupport {
  val parameter = randomStrPair
  val header = randomStrPair
  val formData = randomStrPair
  val userAgent = randomStr
  val cookie = randomStrPair
  val path = s"$randomStr/$randomStr"
  val anotherPath = s"$randomStr/$randomStr"
  val someObject = SomeCaseClass(randomStr, randomInt)

  val somePort = randomPort
  val content = randomStr
  val anotherContent = randomStr

  val requestData = ListBuffer.empty[String]


  val bigResponse = 1024 * 1024

  def issueChunkedPostRequestWith(content: String, toPath: String)(implicit baseUri: BaseUri) = {
    val serverUrl = new URL(s"http://localhost:${baseUri.port}/$toPath")
    val conn = serverUrl.openConnection.asInstanceOf[HttpURLConnection]
    conn.setRequestMethod("POST")
    conn.setRequestProperty("Content-Type", "text/plain")
    conn.setChunkedStreamingMode(0)
    conn.setDoOutput(true)
    conn.setDoInput(true)
    conn.setUseCaches(false)
    conn.connect()

    val out = new DataOutputStream(conn.getOutputStream)
    out.writeBytes(content)
    out.flush()
    out.close()
    conn.disconnect()
  }
}

object HttpClientTestResponseHandlers {
  def handlerFor(path: String, returnsBody: String): RequestHandler = {
    case r: HttpRequest if r.uri.path.toString.endsWith(path) => HttpResponse(entity = returnsBody)
  }

  def unmarshallingAndStoringHandlerFor(path: String, storeTo: ListBuffer[String]): RequestHandler = {
    case r: HttpRequest if r.uri.path.toString.endsWith(path) =>
      storeTo.append( r.extractAsString )
      HttpResponse()
  }

  def bigResponseWith(size: Int): RequestHandler = {
    case HttpRequest(GET, uri, _, _, _) if uri.path.toString().contains("big-response") =>
      HttpResponse(entity = HttpEntity(randomStrWith(size)))
  }

  def chunkedResponseFor(path: String): RequestHandler = {
    case r: HttpRequest if r.uri.path.toString.endsWith(path) =>
      HttpResponse(entity = HttpEntity.Chunked(ContentTypes.`text/plain(UTF-8)`, Source.single(randomStr)))
  }

  def alwaysRespondWith(transferEncoding: TransferEncoding, toPath: String): RequestHandler = {
    case r: HttpRequest if r.uri.path.toString.endsWith(toPath) =>
      HttpResponse().withHeaders(immutable.Seq(`Transfer-Encoding`(transferEncoding)))
  }

  val slowRespondingServer: RequestHandler = { case _ => Thread.sleep(500); HttpResponse() }
}

case class SomeCaseClass(s: String, i: Int)

object HttpClientMatchers {
  import com.wix.e2e.http.matchers.RequestMatchers._

  def haveClientHttpTestkitUserAgentWithLibraryVersion: RequestMatcher =
    haveAnyHeadersOf("User-Agent" -> s"client-http-testkit/$HttpTestkitVersion")
}

object HttpServerMatchers {
  import com.wix.e2e.http.matchers.ResponseMatchers._

  def haveServerHttpTestkitHeaderWithLibraryVersion: ResponseMatcher =
    haveAnyHeadersOf("Server" -> s"server-http-testkit/$HttpTestkitVersion")
} 

package flaky

import java.io._
import java.net.{HttpURLConnection, URL}
import java.util.Scanner

import sbt.{File, Logger}

import scala.language.postfixOps
import scala.util.{Failure, Success, Try}

object Io {

  def writeToFile(file: File, content: String): Unit = {
    new PrintWriter(file) {
      write(content)
      close()
    }
  }

  def writeToFile(file: File, content: Array[Byte]): Unit = {
    new FileOutputStream(file) {
      write(content)
      close()
    }
  }

  def writeToFile(file: File, is: InputStream): Unit = {
    val array: Array[Byte] = Stream.continually(is.read).takeWhile(_ != -1).map(_.toByte).toArray
    writeToFile(file, array)
  }


  def sendToSlack(webHook: String, jsonMsg: String, log: Logger, backupFile: File): Unit = {
    log.info("Sending report to slack")
    log.debug("Dumping slack msg to file")
    new PrintWriter(backupFile) {
      write(jsonMsg)
      close()
    }

    val send: Try[Unit] = Try {
      val url = new URL(webHook)
      val urlConnection = url.openConnection().asInstanceOf[HttpURLConnection]
      // Indicate that we want to write to the HTTP request body
      urlConnection.setDoOutput(true)
      urlConnection.setRequestMethod("POST")

      // Writing the post data to the HTTP request body
      log.debug(jsonMsg)
      val httpRequestBodyWriter = new BufferedWriter(new OutputStreamWriter(urlConnection.getOutputStream))
      httpRequestBodyWriter.write(jsonMsg)
      httpRequestBodyWriter.close()

      val scanner = new Scanner(urlConnection.getInputStream)
      log.debug("Response from SLACK:")
      while (scanner.hasNextLine) {
        log.debug(s"Response from SLACK: ${scanner.nextLine()}")
      }
      scanner.close()
    }
    send match {
      case Success(_) => log.info("Notification successfully send to Slack")
      case Failure(e) => log.error(s"Can't send message to slack: ${e.getMessage}")
    }

  }
} 

package io.flow.dependency.api.lib

import io.flow.dependency.v0.models.{Credentials, CredentialsUndefinedType, UsernamePassword}
import org.htmlcleaner.HtmlCleaner
import org.apache.commons.codec.binary.Base64
import org.apache.commons.lang3.StringUtils
import org.apache.commons.text.StringEscapeUtils
import java.net.URL


import scala.util.{Failure, Success, Try}



object RemoteDirectory {

  case class Result(
    directories: Seq[String] = Nil,
    files: Seq[String] = Nil
  )

  def fetch(
    url: String,
    credentials: Option[Credentials] = None
  ) (
    filter: String => Boolean = { !_.startsWith(".") }
  ): Result = {
    val base = Result()
    val cleaner = new HtmlCleaner()

    val uc = (new URL(url)).openConnection()
    credentials.map { cred =>
      cred match {
        case UsernamePassword(username, password) =>{
          val userpass = username + ":" + password.getOrElse("")
          val basicAuth = "Basic " + new String(new Base64().encode(userpass.getBytes()))
          uc.setRequestProperty ("Authorization", basicAuth)
        }
        case CredentialsUndefinedType(_) => {
          // No-op
        }
      }
    }

    Try(cleaner.clean(uc.getInputStream())) match {
      case Failure(_) => {
        base
      }
      case Success(rootNode) => {
        rootNode.getElementsByName("a", true).foldLeft(base) { case (result, elem) =>
          Option(elem.getAttributeByName("href")) match {
            case None => {
              result
            }
            case Some(_) => {
              val text = StringEscapeUtils.unescapeHtml4(elem.getText.toString)
              filter(StringUtils.stripEnd(text, "/")) match {
                case false => {
                  result
                }
                case true => {
                  text.endsWith("/") match {
                    case true => result.copy(directories = result.directories ++ Seq(text))
                    case false => result.copy(files = result.files ++ Seq(text))
                  }
                }
              }
            }
          }
        }
      }
    }
  }
} 

package com.amazon.milan

import java.io.{File, FileOutputStream}
import java.net.URL
import java.nio.file.{Files, Path}

import com.typesafe.scalalogging.Logger
import org.slf4j.LoggerFactory


package object tools {
  private lazy val logger = Logger(LoggerFactory.getLogger("milan"))

  def addToSbtClasspath(paths: Seq[Path]): Unit = {
    val urls = paths.map(_.toUri.toURL).toList

    urls.foreach(url => logger.info(s"Adding {$url} to classpath."))

    val classLoader = this.getClass.getClassLoader
    val addMethod = classLoader.getClass.getDeclaredMethod("add", classOf[Seq[URL]])
    addMethod.invoke(classLoader, urls)
  }

  
  def compileApplicationInstance(providerClassName: String,
                                 providerParameters: List[(String, String)],
                                 compilerClassName: String,
                                 compilerParameters: List[(String, String)],
                                 outputFile: Path): File = {
    val providerClass = ClassHelper.loadClass(providerClassName)

    val provider =
      providerClass.getConstructors.find(_.getParameterCount == 0) match {
        case None =>
          throw new Exception(s"Provider class $providerClassName does not have a default constructor.")

        case Some(constructor) =>
          constructor.newInstance().asInstanceOf[ApplicationInstanceProvider]
      }

    val instance = provider.getApplicationInstance(providerParameters)

    val actualCompilerClassName = KnownCompilers.convertFromKnownCompiler(compilerClassName)
    val compilerClass = ClassHelper.loadClass(actualCompilerClassName)

    val compiler = compilerClass.getConstructors.find(_.getParameterCount == 0) match {
      case None =>
        throw new Exception(s"Compiler class $actualCompilerClassName does not have a default constructor.")

      case Some(constructor) =>
        constructor.newInstance().asInstanceOf[ApplicationInstanceCompiler]
    }

    println(s"Writing generated code to output file '$outputFile'.")

    Files.createDirectories(outputFile.getParent)

    val outputStream = new FileOutputStream(outputFile.toFile)

    try {
      compiler.compile(instance, compilerParameters, outputStream)
      outputFile.toFile
    }
    finally {
      outputStream.close()
    }
  }
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object additional_properties_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿KeyedArrays") → 
		TypeDef(Reference("⌿definitions⌿KeyedArrays"), 
			Seq(
					Field(Reference("⌿definitions⌿KeyedArrays⌿additionalProperties"), CatchAll(Arr(BInt(TypeMeta(None, List())), TypeMeta(None, List()), "csv"), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 1"), List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String = "/api" 
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq()

def packageName: Option[String] = None

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object options_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿Basic") → 
		TypeDef(Reference("⌿definitions⌿Basic"), 
			Seq(
					Field(Reference("⌿definitions⌿Basic⌿id"), Lng(TypeMeta(Some("int64"), List()))),
					Field(Reference("⌿definitions⌿Basic⌿required"), TypeRef(Reference("⌿definitions⌿Basic⌿required"))),
					Field(Reference("⌿definitions⌿Basic⌿optional"), TypeRef(Reference("⌿definitions⌿Basic⌿optional")))
			), TypeMeta(Some("Named types: 3"), List())),
	Reference("⌿definitions⌿Basic⌿required") → 
		Arr(Str(None, TypeMeta(None, List())), TypeMeta(None, List()), "csv"),
	Reference("⌿definitions⌿Basic⌿optional") → 
		Opt(TypeRef(Reference("⌿definitions⌿Basic⌿required")), TypeMeta(None, List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String = "/api" 
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq()

def packageName: Option[String] = None

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object basic_auth_api_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿paths⌿/⌿get⌿responses⌿200") → 
		Null(TypeMeta(None, List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String =null
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	"basicAuth" -> Basic(Some("HTTP Basic Authentication. Works over `HTTP` and `HTTPS`"))
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq(
	ApiCall(GET, Path(Reference("⌿")), 
		HandlerCall(
			"basic.auth.api.yaml",
			"BasicAuthApiYaml",
			instantiate = false,
			"get",parameters = 
			Seq(

				)
			), 
		Set.empty[MimeType], 
		Set.empty[MimeType], 
		Map.empty[String, Seq[Class[Exception]]], 
		TypesResponseInfo(
			Map[Int, ParameterRef](
			200 -> ParameterRef(Reference("⌿paths⌿/⌿get⌿responses⌿200"))
		), None), 
		StateResponseInfo(
				Map[Int, State](
					200 -> Self
			), None), 
		Set(
			BasicConstraint("basicAuth", Basic(Some("HTTP Basic Authentication. Works over `HTTP` and `HTTPS`")))
		)))

def packageName: Option[String] = Some("basic.auth.api.yaml")

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object nested_options_validation_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿Basic") → 
		TypeDef(Reference("⌿definitions⌿Basic"), 
			Seq(
					Field(Reference("⌿definitions⌿Basic⌿optional"), TypeRef(Reference("⌿definitions⌿Basic⌿optional")))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿definitions⌿Basic⌿optional") → 
		Opt(TypeRef(Reference("⌿definitions⌿Basic⌿optional⌿Opt")), TypeMeta(None, List())),
	Reference("⌿definitions⌿Basic⌿optional⌿Opt") → 
		TypeDef(Reference("⌿definitions⌿Basic⌿optional"), 
			Seq(
					Field(Reference("⌿definitions⌿Basic⌿optional⌿nested_optional"), TypeRef(Reference("⌿definitions⌿Basic⌿optional⌿nested_optional")))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿definitions⌿Basic⌿optional⌿nested_optional") → 
		Opt(Str(None, TypeMeta(None, List("maxLength(6)", "minLength(5)"))), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿responses⌿200") → 
		Null(TypeMeta(None, List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿basic")) → Parameter("basic", TypeRef(Reference("⌿definitions⌿Basic")), None, None, ".+", encode = false, ParameterPlace.withName("body"))
) 
 def basePath: String = "/api" 
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq(
	ApiCall(GET, Path(Reference("⌿")), 
		HandlerCall(
			"nested_options_validation.yaml",
			"Nested_options_validationYaml",
			instantiate = false,
			"get",parameters = 
			Seq(
				ParameterRef(Reference("⌿paths⌿/⌿get⌿basic"))
				)
			), 
		Set(MimeType("application/json")), 
		Set(MimeType("application/json")), 
		Map.empty[String, Seq[Class[Exception]]], 
		TypesResponseInfo(
			Map[Int, ParameterRef](
			200 -> ParameterRef(Reference("⌿paths⌿/⌿get⌿responses⌿200"))
		), None), 
		StateResponseInfo(
				Map[Int, State](
					200 -> Self
			), None), 
		Set.empty[Security.Constraint]))

def packageName: Option[String] = Some("nested_options_validation.yaml")

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object minimal_api_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿paths⌿/⌿get⌿responses⌿200") → 
		Null(TypeMeta(None, List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String =null
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq(
	ApiCall(GET, Path(Reference("⌿")), 
		HandlerCall(
			"admin",
			"Dashboard",
			instantiate = false,
			"index",parameters = 
			Seq(

				)
			), 
		Set.empty[MimeType], 
		Set.empty[MimeType], 
		Map.empty[String, Seq[Class[Exception]]], 
		TypesResponseInfo(
			Map[Int, ParameterRef](
			200 -> ParameterRef(Reference("⌿paths⌿/⌿get⌿responses⌿200"))
		), None), 
		StateResponseInfo(
				Map[Int, State](
					200 -> Self
			), None), 
		Set.empty[Security.Constraint]))

def packageName: Option[String] = Some("admin")

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object security_api_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿ErrorModel") → 
		TypeDef(Reference("⌿definitions⌿ErrorModel"), 
			Seq(
					Field(Reference("⌿definitions⌿ErrorModel⌿code"), Intgr(TypeMeta(Some("int32"), List()))),
					Field(Reference("⌿definitions⌿ErrorModel⌿message"), Str(None, TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 2"), List())),
	Reference("⌿definitions⌿Pet") → 
		TypeDef(Reference("⌿definitions⌿Pet"), 
			Seq(
					Field(Reference("⌿definitions⌿Pet⌿name"), Str(None, TypeMeta(None, List()))),
					Field(Reference("⌿definitions⌿Pet⌿tag"), TypeRef(Reference("⌿definitions⌿Pet⌿tag")))
			), TypeMeta(Some("Named types: 2"), List())),
	Reference("⌿definitions⌿Pet⌿tag") → 
		Opt(Str(None, TypeMeta(None, List())), TypeMeta(None, List())),
	Reference("⌿paths⌿/pets/{id}⌿get⌿id") → 
		Arr(Str(None, TypeMeta(None, List())), TypeMeta(None, List()), "csv"),
	Reference("⌿paths⌿/pets/{id}⌿get⌿responses⌿200") → 
		ArrResult(TypeRef(Reference("⌿definitions⌿Pet")), TypeMeta(None, List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](
	ParameterRef(	Reference("⌿paths⌿/pets/{id}⌿get⌿id")) → Parameter("id", TypeRef(Reference("⌿paths⌿/pets/{id}⌿get⌿id")), None, None, "[^/]+", encode = true, ParameterPlace.withName("path"))
) 
 def basePath: String = "/v1" 
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	"petstoreImplicit" -> OAuth2Definition(None, Some(new URL("http://petstore.swagger.wordnik.com/oauth/dialog")), Map[String, String]( "admin:org" -> "Fully manage organization, teams, and memberships." ,  "user:email" -> "Grants read access to a user’s email addresses." ,  "read:org" -> "Read-only access to organization, teams, and membership." ,  "public_repo" -> "Grants read/write access to code, commit statuses, and deployment statuses for public repositories and organizations." ,  "write:public_key" -> "Create, list, and view details for public keys." ,  "repo_deployment" -> "Grants access to deployment statuses for public and private repositories. This scope is only necessary to grant other users or services access to deployment statuses, without granting access to the code." ,  "write:repo_hook" -> "Grants read, write, and ping access to hooks in public or private repositories." ,  "admin:public_key" -> "Fully manage public keys." ,  "repo:status" -> "Grants read/write access to public and private repository commit statuses. This scope is only necessary to grant other users or services access to private repository commit statuses without granting access to the code." ,  "gist" -> "Grants write access to gists." ,  "user:follow" -> "Grants access to follow or unfollow other users." ,  "repo" -> "Grants read/write access to code, commit statuses, and deployment statuses for public and private repositories and organizations." ,  "read:repo_hook" -> "Grants read and ping access to hooks in public or private repositories." ,  "notifications" -> "Grants read access to a user’s notifications. repo also provides this access." ,  "read:public_key" -> "List and view details for public keys." ,  "admin:repo_hook" -> "Grants read, write, ping, and delete access to hooks in public or private repositories." ,  "user" -> "Grants read/write access to profile info only. Note that this scope includes user:email and user:follow." ,  "write:org" -> "Publicize and unpublicize organization membership." ,  "delete_repo" -> "Grants access to delete adminable repositories." )),
	"githubAccessCode" -> OAuth2Definition(None, Some(new URL("https://github.com/login/oauth/access_token")), Map[String, String]( "admin:org" -> "Fully manage organization, teams, and memberships." ,  "user:email" -> "Grants read access to a user’s email addresses." ,  "read:org" -> "Read-only access to organization, teams, and membership." ,  "public_repo" -> "Grants read/write access to code, commit statuses, and deployment statuses for public repositories and organizations." ,  "write:public_key" -> "Create, list, and view details for public keys." ,  "repo_deployment" -> "Grants access to deployment statuses for public and private repositories. This scope is only necessary to grant other users or services access to deployment statuses, without granting access to the code." ,  "write:repo_hook" -> "Grants read, write, and ping access to hooks in public or private repositories." ,  "admin:public_key" -> "Fully manage public keys." ,  "repo:status" -> "Grants read/write access to public and private repository commit statuses. This scope is only necessary to grant other users or services access to private repository commit statuses without granting access to the code." ,  "gist" -> "Grants write access to gists." ,  "user:follow" -> "Grants access to follow or unfollow other users." ,  "repo" -> "Grants read/write access to code, commit statuses, and deployment statuses for public and private repositories and organizations." ,  "read:repo_hook" -> "Grants read and ping access to hooks in public or private repositories." ,  "notifications" -> "Grants read access to a user’s notifications. repo also provides this access." ,  "read:public_key" -> "List and view details for public keys." ,  "admin:repo_hook" -> "Grants read, write, ping, and delete access to hooks in public or private repositories." ,  "user" -> "Grants read/write access to profile info only. Note that this scope includes user:email and user:follow." ,  "write:org" -> "Publicize and unpublicize organization membership." ,  "delete_repo" -> "Grants access to delete adminable repositories." )),
	"petstorePassword" -> OAuth2Definition(None, Some(new URL("http://petstore.swagger.wordnik.com/oauth/dialog")), Map[String, String]( "user" -> "Grants read/write access to profile" ,  "admin" -> "Fully manage" )),
	"justBasicStuff" -> Basic(None),
	"petstoreApplication" -> OAuth2Definition(None, Some(new URL("http://petstore.swagger.wordnik.com/oauth/token")), Map[String, String]( "user" -> "Grants read/write access to profile" ,  "admin" -> "Fully manage" )),
	"internalApiKey" -> ApiKey(None, "api_key", ParameterPlace.withName("header"))
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq(
	ApiCall(GET, Path(Reference("⌿pets⌿{id}")), 
		HandlerCall(
			"security.api.yaml",
			"SecurityApiYaml",
			instantiate = false,
			"getPetsById",parameters = 
			Seq(
				ParameterRef(Reference("⌿paths⌿/pets/{id}⌿get⌿id"))
				)
			), 
		Set(MimeType("application/json")), 
		Set(MimeType("application/json"), MimeType("text/html")), 
		Map.empty[String, Seq[Class[Exception]]], 
		TypesResponseInfo(
			Map[Int, ParameterRef](
			200 -> ParameterRef(Reference("⌿paths⌿/pets/{id}⌿get⌿responses⌿200"))
		), Some(	ParameterRef(Reference("⌿definitions⌿ErrorModel")))), 
		StateResponseInfo(
				Map[Int, State](
					200 -> Self
			), Some(Self)), 
		Set(
			OAuth2Constraint("githubAccessCode", OAuth2Definition(None, Some(new URL("https://github.com/login/oauth/access_token")), Map[String, String]( "admin:org" -> "Fully manage organization, teams, and memberships." ,  "user:email" -> "Grants read access to a user’s email addresses." ,  "read:org" -> "Read-only access to organization, teams, and membership." ,  "public_repo" -> "Grants read/write access to code, commit statuses, and deployment statuses for public repositories and organizations." ,  "write:public_key" -> "Create, list, and view details for public keys." ,  "repo_deployment" -> "Grants access to deployment statuses for public and private repositories. This scope is only necessary to grant other users or services access to deployment statuses, without granting access to the code." ,  "write:repo_hook" -> "Grants read, write, and ping access to hooks in public or private repositories." ,  "admin:public_key" -> "Fully manage public keys." ,  "repo:status" -> "Grants read/write access to public and private repository commit statuses. This scope is only necessary to grant other users or services access to private repository commit statuses without granting access to the code." ,  "gist" -> "Grants write access to gists." ,  "user:follow" -> "Grants access to follow or unfollow other users." ,  "repo" -> "Grants read/write access to code, commit statuses, and deployment statuses for public and private repositories and organizations." ,  "read:repo_hook" -> "Grants read and ping access to hooks in public or private repositories." ,  "notifications" -> "Grants read access to a user’s notifications. repo also provides this access." ,  "read:public_key" -> "List and view details for public keys." ,  "admin:repo_hook" -> "Grants read, write, ping, and delete access to hooks in public or private repositories." ,  "user" -> "Grants read/write access to profile info only. Note that this scope includes user:email and user:follow." ,  "write:org" -> "Publicize and unpublicize organization membership." ,  "delete_repo" -> "Grants access to delete adminable repositories." )), Set("user")),
			ApiKeyConstraint("internalApiKey", ApiKey(None, "api_key", ParameterPlace.withName("header")))
		)))

def packageName: Option[String] = Some("security.api.yaml")

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object basic_polymorphism_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿Cat") → 
					AllOf(Reference("⌿definitions⌿Cat⌿Cat"), TypeMeta(Some("Schemas: 2"), List()),  Seq(
			TypeRef(Reference("⌿definitions⌿Pet")),
			TypeRef(Reference("⌿definitions⌿Cat⌿AllOf1"))) , Some(Reference("⌿definitions⌿Pet⌿petType"))),
	Reference("⌿definitions⌿Dog") → 
					AllOf(Reference("⌿definitions⌿Dog⌿Dog"), TypeMeta(Some("Schemas: 2"), List()),  Seq(
			TypeRef(Reference("⌿definitions⌿Pet")),
			TypeRef(Reference("⌿definitions⌿Dog⌿AllOf1"))) , Some(Reference("⌿definitions⌿Pet⌿petType"))),
	Reference("⌿definitions⌿CatNDog") → 
					AllOf(Reference("⌿definitions⌿CatNDog⌿CatNDog"), TypeMeta(Some("Schemas: 2"), List()),  Seq(
			TypeRef(Reference("⌿definitions⌿Dog")),
			TypeRef(Reference("⌿definitions⌿Cat"))) , Some(Reference("⌿definitions⌿Pet⌿petType"))),
	Reference("⌿definitions⌿Pet") → 
		TypeDef(Reference("⌿definitions⌿Pet"), 
			Seq(
					Field(Reference("⌿definitions⌿Pet⌿name"), Str(None, TypeMeta(None, List()))),
					Field(Reference("⌿definitions⌿Pet⌿petType"), Str(None, TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 2"), List())),
	Reference("⌿definitions⌿Labrador") → 
					AllOf(Reference("⌿definitions⌿Labrador⌿Labrador"), TypeMeta(Some("Schemas: 2"), List()),  Seq(
			TypeRef(Reference("⌿definitions⌿Dog")),
			TypeRef(Reference("⌿definitions⌿Labrador⌿AllOf1"))) , Some(Reference("⌿definitions⌿Pet⌿petType"))),
	Reference("⌿definitions⌿Cat⌿AllOf1") → 
		TypeDef(Reference("⌿definitions⌿Cat"), 
			Seq(
					Field(Reference("⌿definitions⌿Cat⌿huntingSkill"), TypeRef(Reference("⌿definitions⌿Cat⌿huntingSkill")))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿definitions⌿Dog⌿AllOf1") → 
		TypeDef(Reference("⌿definitions⌿Dog"), 
			Seq(
					Field(Reference("⌿definitions⌿Dog⌿packSize"), Intgr(TypeMeta(Some("the size of the pack the dog is from"), List("min(0.toInt, false)"))))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿definitions⌿Cat⌿huntingSkill") → 
					EnumTrait(Str(None, TypeMeta(Some("The measured skill for hunting"), List("""enum("clueless,lazy,adventurous,aggressive")"""))), TypeMeta(Some("Enum type : 4"), List()), 
				Set(
					EnumObject(Str(None, TypeMeta(Some("The measured skill for hunting"), List("""enum("clueless,lazy,adventurous,aggressive")"""))), "clueless", TypeMeta(Some("clueless"), List())),
					EnumObject(Str(None, TypeMeta(Some("The measured skill for hunting"), List("""enum("clueless,lazy,adventurous,aggressive")"""))), "lazy", TypeMeta(Some("lazy"), List())),
					EnumObject(Str(None, TypeMeta(Some("The measured skill for hunting"), List("""enum("clueless,lazy,adventurous,aggressive")"""))), "adventurous", TypeMeta(Some("adventurous"), List())),
					EnumObject(Str(None, TypeMeta(Some("The measured skill for hunting"), List("""enum("clueless,lazy,adventurous,aggressive")"""))), "aggressive", TypeMeta(Some("aggressive"), List()))

				)),
	Reference("⌿definitions⌿Labrador⌿AllOf1") → 
		TypeDef(Reference("⌿definitions⌿Labrador"), 
			Seq(
					Field(Reference("⌿definitions⌿Labrador⌿cuteness"), Intgr(TypeMeta(Some("the cuteness of the animal in percent"), List("min(0.toInt, false)"))))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿definitions⌿Cat⌿huntingSkill⌿lazy") → 
					EnumObject(Str(None, TypeMeta(Some("The measured skill for hunting"), List("""enum("clueless,lazy,adventurous,aggressive")"""))), "lazy", TypeMeta(Some("lazy"), List())),
	Reference("⌿definitions⌿Cat⌿huntingSkill⌿clueless") → 
					EnumObject(Str(None, TypeMeta(Some("The measured skill for hunting"), List("""enum("clueless,lazy,adventurous,aggressive")"""))), "clueless", TypeMeta(Some("clueless"), List())),
	Reference("⌿definitions⌿CatNDog⌿huntingSkill⌿aggressive") → 
					EnumObject(Str(None, TypeMeta(Some("The measured skill for hunting"), List("""enum("clueless,lazy,adventurous,aggressive")"""))), "aggressive", TypeMeta(Some("aggressive"), List())),
	Reference("⌿definitions⌿Cat⌿huntingSkill⌿adventurous") → 
					EnumObject(Str(None, TypeMeta(Some("The measured skill for hunting"), List("""enum("clueless,lazy,adventurous,aggressive")"""))), "adventurous", TypeMeta(Some("adventurous"), List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String =null
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
		Reference("⌿definitions⌿CatNDog") -> Reference("⌿definitions⌿Pet⌿petType"),
	Reference("⌿definitions⌿Dog") -> Reference("⌿definitions⌿Pet⌿petType"),
	Reference("⌿definitions⌿Cat") -> Reference("⌿definitions⌿Pet⌿petType"),
	Reference("⌿definitions⌿Labrador") -> Reference("⌿definitions⌿Pet⌿petType"),
	Reference("⌿definitions⌿Pet") -> Reference("⌿definitions⌿Pet⌿petType"))
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq()

def packageName: Option[String] = None

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object nested_arrays_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿Activity") → 
		TypeDef(Reference("⌿definitions⌿Activity"), 
			Seq(
					Field(Reference("⌿definitions⌿Activity⌿actions"), TypeRef(Reference("⌿definitions⌿Activity⌿actions")))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿definitions⌿Example") → 
		TypeDef(Reference("⌿definitions⌿Example"), 
			Seq(
					Field(Reference("⌿definitions⌿Example⌿messages"), TypeRef(Reference("⌿definitions⌿Example⌿messages"))),
					Field(Reference("⌿definitions⌿Example⌿nestedArrays"), TypeRef(Reference("⌿definitions⌿Example⌿nestedArrays")))
			), TypeMeta(Some("Named types: 2"), List())),
	Reference("⌿definitions⌿Example⌿messages") → 
		Opt(TypeRef(Reference("⌿definitions⌿Example⌿messages⌿Opt")), TypeMeta(None, List())),
	Reference("⌿definitions⌿Example⌿nestedArrays") → 
		Opt(TypeRef(Reference("⌿definitions⌿Example⌿nestedArrays⌿Opt")), TypeMeta(None, List())),
	Reference("⌿definitions⌿Activity⌿actions") → 
		Opt(Str(None, TypeMeta(Some("The text of the error message"), List())), TypeMeta(None, List())),
	Reference("⌿definitions⌿Example⌿nestedArrays⌿Opt") → 
		Arr(TypeRef(Reference("⌿definitions⌿Example⌿nestedArrays⌿Opt⌿Arr")), TypeMeta(None, List()), "csv"),
	Reference("⌿definitions⌿Example⌿messages⌿Opt") → 
		Arr(TypeRef(Reference("⌿definitions⌿Example⌿messages⌿Opt⌿Arr")), TypeMeta(Some("The text of the error message"), List()), "csv"),
	Reference("⌿definitions⌿Example⌿nestedArrays⌿Opt⌿Arr") → 
		Arr(TypeRef(Reference("⌿definitions⌿Example⌿nestedArrays⌿Opt⌿Arr⌿Arr")), TypeMeta(None, List()), "csv"),
	Reference("⌿definitions⌿Example⌿messages⌿Opt⌿Arr") → 
		Arr(TypeRef(Reference("⌿definitions⌿Activity")), TypeMeta(None, List()), "csv"),
	Reference("⌿definitions⌿Example⌿nestedArrays⌿Opt⌿Arr⌿Arr") → 
		Arr(TypeRef(Reference("⌿definitions⌿Example⌿nestedArrays⌿Opt⌿Arr⌿Arr⌿Arr")), TypeMeta(None, List()), "csv"),
	Reference("⌿definitions⌿Example⌿nestedArrays⌿Opt⌿Arr⌿Arr⌿Arr") → 
		Arr(Str(Some("nested arrays"), TypeMeta(Some("nested arrays"), List())), TypeMeta(None, List()), "csv")
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String = "/api" 
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq()

def packageName: Option[String] = None

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.swagger

import java.net.URL

import de.zalando.apifirst.Application.SecurityDefinitionsTable
import de.zalando.apifirst.{ParameterPlace, Security}
import de.zalando.swagger.strictModel._


object SecurityConverter {

  def convertDefinitions(swaggerDefinitions: SecurityDefinitions): SecurityDefinitionsTable =
    if (swaggerDefinitions == null) Map.empty
    else swaggerDefinitions.collect {
      case (name, basic: BasicAuthenticationSecurity) =>
        name -> Security.Basic(Option(basic.description))
      case (name, apiKey: ApiKeySecurity) =>
        require(apiKey.name != null && apiKey.name.nonEmpty)
        require(apiKey.in != null && apiKey.in.nonEmpty)
        val place = ParameterPlace.withName(apiKey.in.toLowerCase)
        require(place == ParameterPlace.HEADER || place == ParameterPlace.QUERY)
        name -> Security.ApiKey(Option(apiKey.description), apiKey.name, place)
      case (name, oauth: Oauth2SecurityDefinition) =>
        val validationURL = oauth.validationUrl.map(new URL(_))
        name -> Security.OAuth2Definition(Option(oauth.description), validationURL, oauth.scopes)
    }
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object string_formats_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿paths⌿/⌿get⌿base64") → 
		Opt(Base64String(TypeMeta(Some("byte"), List())), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿petId") → 
		BinaryString(TypeMeta(Some("binary"), List())),
	Reference("⌿paths⌿/⌿get⌿date_time") → 
		Opt(DateTime(TypeMeta(Some("date-time"), List())), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿uuid") → 
		Opt(UUID(TypeMeta(Some("UUID"), List())), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿date") → 
		Opt(Date(TypeMeta(Some("date"), List())), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿responses⌿200") → 
		Null(TypeMeta(None, List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿date_time")) → Parameter("date_time", Opt(DateTime(TypeMeta(Some("date-time"), List())), TypeMeta(None, List())), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿date")) → Parameter("date", Opt(Date(TypeMeta(Some("date"), List())), TypeMeta(None, List())), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿base64")) → Parameter("base64", Opt(Base64String(TypeMeta(Some("byte"), List())), TypeMeta(None, List())), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿uuid")) → Parameter("uuid", Opt(UUID(TypeMeta(Some("UUID"), List())), TypeMeta(None, List())), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿petId")) → Parameter("petId", BinaryString(TypeMeta(Some("binary"), List())), None, None, ".+", encode = false, ParameterPlace.withName("body"))
) 
 def basePath: String =null
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq(
	ApiCall(GET, Path(Reference("⌿")),
		HandlerCall(
			"string_formats.yaml",
			"String_formatsYaml",
			instantiate = false,
			"get",parameters = 
			Seq(
				ParameterRef(Reference("⌿paths⌿/⌿get⌿date_time")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿date")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿base64")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿uuid")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿petId"))
				)
			),
		Set.empty[MimeType],
		Set(MimeType("application/json"), MimeType("application/yaml")),
		Map.empty[String, Seq[Class[Exception]]],
		TypesResponseInfo(
			Map[Int, ParameterRef](
			200 -> ParameterRef(Reference("⌿paths⌿/⌿get⌿responses⌿200"))
		), None),
		StateResponseInfo(
				Map[Int, State](
					200 -> Self
			), None),
		Set.empty[Security.Constraint]))

def packageName: Option[String] = Some("string_formats.yaml")

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object nested_options_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿Basic") → 
		TypeDef(Reference("⌿definitions⌿Basic"), 
			Seq(
					Field(Reference("⌿definitions⌿Basic⌿optional"), Opt(TypeDef(Reference("⌿definitions⌿Basic⌿optional"), 
			Seq(
						Field(Reference("⌿definitions⌿Basic⌿optional⌿nested_optional"), Opt(Str(None, TypeMeta(None, List())), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 1"), List())), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 1"), List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String = "/api" 
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq()

def packageName: Option[String] = None

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object nested_objects_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿NestedObjects") → 
		TypeDef(Reference("⌿definitions⌿NestedObjects"), 
			Seq(
					Field(Reference("⌿definitions⌿NestedObjects⌿plain"), Opt(TypeDef(Reference("⌿definitions⌿NestedObjects⌿plain"), 
			Seq(
						Field(Reference("⌿definitions⌿NestedObjects⌿plain⌿simple"), Str(None, TypeMeta(None, List("""pattern("""+"""""""""+"""the pattern"""+"""""""""+""".r)"""))))
			), TypeMeta(Some("Named types: 1"), List())), TypeMeta(None, List()))),
					Field(Reference("⌿definitions⌿NestedObjects⌿nested"), Opt(TypeDef(Reference("⌿definitions⌿NestedObjects⌿nested"), 
			Seq(
						Field(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2"), TypeDef(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2"), 
			Seq(
							Field(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3"), Opt(TypeDef(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3"), 
			Seq(
								Field(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3⌿bottom"), Opt(Str(None, TypeMeta(None, List("maxLength(30)", "minLength(3)"))), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 1"), List())), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 1"), List())))
			), TypeMeta(Some("Named types: 1"), List())), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 2"), List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String = "/api" 
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq()

def packageName: Option[String] = None

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object basic_extension_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿ErrorModel") → 
		TypeDef(Reference("⌿definitions⌿ErrorModel"), 
			Seq(
					Field(Reference("⌿definitions⌿ErrorModel⌿message"), Str(None, TypeMeta(Some("The text of the error message"), List()))),
					Field(Reference("⌿definitions⌿ErrorModel⌿code"), BInt(TypeMeta(Some("The error code"), List("""max(BigInt("600"), false)""", """min(BigInt("100"), false)"""))))
			), TypeMeta(Some("Named types: 2"), List())),
	Reference("⌿definitions⌿ExtendedErrorModel") → 
					AllOf(Reference("⌿definitions⌿ExtendedErrorModel⌿ExtendedErrorModel"), TypeMeta(Some("Schemas: 2"), List()),  Seq(
			TypeDef(Reference("⌿definitions⌿ErrorModel"), 
			Seq(
						Field(Reference("⌿definitions⌿ErrorModel⌿message"), Str(None, TypeMeta(Some("The text of the error message"), List()))),
						Field(Reference("⌿definitions⌿ErrorModel⌿code"), BInt(TypeMeta(Some("The error code"), List("""max(BigInt("600"), false)""", """min(BigInt("100"), false)"""))))
			), TypeMeta(Some("Named types: 2"), List())),
			TypeDef(Reference("⌿definitions⌿ExtendedErrorModel"), 
			Seq(
						Field(Reference("⌿definitions⌿ExtendedErrorModel⌿rootCause"), Str(None, TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 1"), List()))) , None)
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String =null
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq()

def packageName: Option[String] = None

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package org.apache.spark.deploy.master.ui

import java.io.DataOutputStream
import java.net.{HttpURLConnection, URL}
import java.nio.charset.StandardCharsets
import java.util.Date

import scala.collection.mutable.HashMap

import org.mockito.Mockito.{mock, times, verify, when}
import org.scalatest.BeforeAndAfterAll

import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
import org.apache.spark.deploy.DeployMessages.{KillDriverResponse, RequestKillDriver}
import org.apache.spark.deploy.DeployTestUtils._
import org.apache.spark.deploy.master._
import org.apache.spark.rpc.{RpcEndpointRef, RpcEnv}


class MasterWebUISuite extends SparkFunSuite with BeforeAndAfterAll {

  val conf = new SparkConf
  val securityMgr = new SecurityManager(conf)
  val rpcEnv = mock(classOf[RpcEnv])
  val master = mock(classOf[Master])
  val masterEndpointRef = mock(classOf[RpcEndpointRef])
  when(master.securityMgr).thenReturn(securityMgr)
  when(master.conf).thenReturn(conf)
  when(master.rpcEnv).thenReturn(rpcEnv)
  when(master.self).thenReturn(masterEndpointRef)
  val masterWebUI = new MasterWebUI(master, 0)

  override def beforeAll() {
    super.beforeAll()
    masterWebUI.bind()
  }

  override def afterAll() {
    masterWebUI.stop()
    super.afterAll()
  }

  test("kill application") {
    val appDesc = createAppDesc()
    // use new start date so it isn't filtered by UI
    val activeApp = new ApplicationInfo(
      new Date().getTime, "app-0", appDesc, new Date(), null, Int.MaxValue)

    when(master.idToApp).thenReturn(HashMap[String, ApplicationInfo]((activeApp.id, activeApp)))

    val url = s"http://localhost:${masterWebUI.boundPort}/app/kill/"
    val body = convPostDataToString(Map(("id", activeApp.id), ("terminate", "true")))
    val conn = sendHttpRequest(url, "POST", body)
    conn.getResponseCode

    // Verify the master was called to remove the active app
    verify(master, times(1)).removeApplication(activeApp, ApplicationState.KILLED)
  }

  test("kill driver") {
    val activeDriverId = "driver-0"
    val url = s"http://localhost:${masterWebUI.boundPort}/driver/kill/"
    val body = convPostDataToString(Map(("id", activeDriverId), ("terminate", "true")))
    val conn = sendHttpRequest(url, "POST", body)
    conn.getResponseCode

    // Verify that master was asked to kill driver with the correct id
    verify(masterEndpointRef, times(1)).ask[KillDriverResponse](RequestKillDriver(activeDriverId))
  }

  private def convPostDataToString(data: Map[String, String]): String = {
    (for ((name, value) <- data) yield s"$name=$value").mkString("&")
  }

  
  private def sendHttpRequest(
      url: String,
      method: String,
      body: String = ""): HttpURLConnection = {
    val conn = new URL(url).openConnection().asInstanceOf[HttpURLConnection]
    conn.setRequestMethod(method)
    if (body.nonEmpty) {
      conn.setDoOutput(true)
      conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded")
      conn.setRequestProperty("Content-Length", Integer.toString(body.length))
      val out = new DataOutputStream(conn.getOutputStream)
      out.write(body.getBytes(StandardCharsets.UTF_8))
      out.close()
    }
    conn
  }
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object numbers_validation_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿paths⌿/⌿get⌿double_optional") → 
		Opt(Dbl(TypeMeta(Some("double"), List("max(10.toDouble, true)", "min(0.toDouble, true)", "multipleOf(5.toDouble)"))), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿integer_required") → 
		Intgr(TypeMeta(Some("int32"), List("max(10.toInt, false)", "min(0.toInt, false)", "multipleOf(5.toInt)"))),
	Reference("⌿paths⌿/⌿get⌿integer_optional") → 
		Opt(Intgr(TypeMeta(Some("int32"), List("max(10.toInt, true)", "min(-10.toInt, true)", "multipleOf(5.toInt)"))), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿double_required") → 
		Dbl(TypeMeta(Some("double"), List("max(10.toDouble, false)", "min(2.toDouble, false)", "multipleOf(5.toDouble)"))),
	Reference("⌿paths⌿/⌿get⌿long_optional") → 
		Opt(Lng(TypeMeta(Some("int64"), List("max(10.toLong, true)", "min(10.toLong, true)", "multipleOf(10.toLong)"))), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿float_required") → 
		Flt(TypeMeta(Some("float"), List("max(10.toFloat, true)", "min(10.toFloat, true)", "multipleOf(5.toFloat)"))),
	Reference("⌿paths⌿/⌿get⌿float_optional") → 
		Opt(Flt(TypeMeta(Some("float"), List("max(10.toFloat, false)", "min(1.toFloat, false)", "multipleOf(5.toFloat)"))), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿long_required") → 
		Lng(TypeMeta(Some("int64"), List("max(10.toLong, true)", "min(0.toLong, true)", "multipleOf(5.toLong)"))),
	Reference("⌿paths⌿/⌿get⌿responses⌿200") → 
		Null(TypeMeta(None, List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿float_required")) → Parameter("float_required", Flt(TypeMeta(Some("float"), List("max(10.toFloat, true)", "min(10.toFloat, true)", "multipleOf(5.toFloat)"))), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿double_required")) → Parameter("double_required", Dbl(TypeMeta(Some("double"), List("max(10.toDouble, false)", "min(2.toDouble, false)", "multipleOf(5.toDouble)"))), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿integer_optional")) → Parameter("integer_optional", Opt(Intgr(TypeMeta(Some("int32"), List("max(10.toInt, true)", "min(-10.toInt, true)", "multipleOf(5.toInt)"))), TypeMeta(None, List())), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿long_required")) → Parameter("long_required", Lng(TypeMeta(Some("int64"), List("max(10.toLong, true)", "min(0.toLong, true)", "multipleOf(5.toLong)"))), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿integer_required")) → Parameter("integer_required", Intgr(TypeMeta(Some("int32"), List("max(10.toInt, false)", "min(0.toInt, false)", "multipleOf(5.toInt)"))), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿float_optional")) → Parameter("float_optional", Opt(Flt(TypeMeta(Some("float"), List("max(10.toFloat, false)", "min(1.toFloat, false)", "multipleOf(5.toFloat)"))), TypeMeta(None, List())), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿double_optional")) → Parameter("double_optional", Opt(Dbl(TypeMeta(Some("double"), List("max(10.toDouble, true)", "min(0.toDouble, true)", "multipleOf(5.toDouble)"))), TypeMeta(None, List())), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿long_optional")) → Parameter("long_optional", Opt(Lng(TypeMeta(Some("int64"), List("max(10.toLong, true)", "min(10.toLong, true)", "multipleOf(10.toLong)"))), TypeMeta(None, List())), None, None, ".+", encode = true, ParameterPlace.withName("query"))
) 
 def basePath: String =null
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq(
	ApiCall(GET, Path(Reference("⌿")), 
		HandlerCall(
			"numbers_validation.yaml",
			"Numbers_validationYaml",
			instantiate = false,
			"get",parameters = 
			Seq(
				ParameterRef(Reference("⌿paths⌿/⌿get⌿float_required")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿double_required")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿integer_optional")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿long_required")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿integer_required")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿float_optional")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿double_optional")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿long_optional"))
				)
			), 
		Set.empty[MimeType], 
		Set(MimeType("application/json"), MimeType("application/yaml")), 
		Map.empty[String, Seq[Class[Exception]]], 
		TypesResponseInfo(
			Map[Int, ParameterRef](
			200 -> ParameterRef(Reference("⌿paths⌿/⌿get⌿responses⌿200"))
		), None), 
		StateResponseInfo(
				Map[Int, State](
					200 -> Self
			), None), 
		Set.empty[Security.Constraint]))

def packageName: Option[String] = Some("numbers_validation.yaml")

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object options_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿Basic") → 
		TypeDef(Reference("⌿definitions⌿Basic"), 
			Seq(
					Field(Reference("⌿definitions⌿Basic⌿id"), Lng(TypeMeta(Some("int64"), List()))),
					Field(Reference("⌿definitions⌿Basic⌿required"), Arr(Str(None, TypeMeta(None, List())), TypeMeta(None, List()), "csv")),
					Field(Reference("⌿definitions⌿Basic⌿optional"), Opt(Arr(Str(None, TypeMeta(None, List())), TypeMeta(None, List()), "csv"), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 3"), List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String = "/api" 
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq()

def packageName: Option[String] = None

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object basic_auth_api_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿paths⌿/⌿get⌿responses⌿200") → 
		Null(TypeMeta(None, List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String =null
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	"basicAuth" -> Basic(Some("HTTP Basic Authentication. Works over `HTTP` and `HTTPS`"))
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq(
	ApiCall(GET, Path(Reference("⌿")), 
		HandlerCall(
			"basic.auth.api.yaml",
			"BasicAuthApiYaml",
			instantiate = false,
			"get",parameters = 
			Seq(

				)
			), 
		Set.empty[MimeType], 
		Set.empty[MimeType], 
		Map.empty[String, Seq[Class[Exception]]], 
		TypesResponseInfo(
			Map[Int, ParameterRef](
			200 -> ParameterRef(Reference("⌿paths⌿/⌿get⌿responses⌿200"))
		), None), 
		StateResponseInfo(
				Map[Int, State](
					200 -> Self
			), None), 
		Set(
			BasicConstraint("basicAuth", Basic(Some("HTTP Basic Authentication. Works over `HTTP` and `HTTPS`")))
		)))

def packageName: Option[String] = Some("basic.auth.api.yaml")

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object nested_options_validation_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿Basic") → 
		TypeDef(Reference("⌿definitions⌿Basic"), 
			Seq(
					Field(Reference("⌿definitions⌿Basic⌿optional"), Opt(TypeDef(Reference("⌿definitions⌿Basic⌿optional"), 
			Seq(
						Field(Reference("⌿definitions⌿Basic⌿optional⌿nested_optional"), Opt(Str(None, TypeMeta(None, List("maxLength(6)", "minLength(5)"))), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 1"), List())), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿paths⌿/⌿get⌿basic") → 
		TypeDef(Reference("⌿definitions⌿Basic"), 
			Seq(
					Field(Reference("⌿definitions⌿Basic⌿optional"), Opt(TypeDef(Reference("⌿paths⌿/⌿get⌿basic⌿optional"), 
			Seq(
						Field(Reference("⌿paths⌿/⌿get⌿basic⌿optional⌿nested_optional"), Opt(Str(None, TypeMeta(None, List("maxLength(6)", "minLength(5)"))), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 1"), List())), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿paths⌿/⌿get⌿responses⌿200") → 
		Null(TypeMeta(None, List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿basic")) → Parameter("basic", TypeDef(Reference("⌿definitions⌿Basic"), 
			Seq(
		Field(Reference("⌿definitions⌿Basic⌿optional"), Opt(TypeDef(Reference("⌿paths⌿/⌿get⌿basic⌿optional"), 
			Seq(
			Field(Reference("⌿paths⌿/⌿get⌿basic⌿optional⌿nested_optional"), Opt(Str(None, TypeMeta(None, List("maxLength(6)", "minLength(5)"))), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 1"), List())), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 1"), List())), None, None, ".+", encode = false, ParameterPlace.withName("body"))
) 
 def basePath: String = "/api" 
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq(
	ApiCall(GET, Path(Reference("⌿")), 
		HandlerCall(
			"nested_options_validation.yaml",
			"Nested_options_validationYaml",
			instantiate = false,
			"get",parameters = 
			Seq(
				ParameterRef(Reference("⌿paths⌿/⌿get⌿basic"))
				)
			), 
		Set(MimeType("application/json")), 
		Set(MimeType("application/json")), 
		Map.empty[String, Seq[Class[Exception]]], 
		TypesResponseInfo(
			Map[Int, ParameterRef](
			200 -> ParameterRef(Reference("⌿paths⌿/⌿get⌿responses⌿200"))
		), None), 
		StateResponseInfo(
				Map[Int, State](
					200 -> Self
			), None), 
		Set.empty[Security.Constraint]))

def packageName: Option[String] = Some("nested_options_validation.yaml")

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object minimal_api_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿paths⌿/⌿get⌿responses⌿200") → 
		Null(TypeMeta(None, List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String =null
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq(
	ApiCall(GET, Path(Reference("⌿")), 
		HandlerCall(
			"admin",
			"Dashboard",
			instantiate = false,
			"index",parameters = 
			Seq(

				)
			), 
		Set.empty[MimeType], 
		Set.empty[MimeType], 
		Map.empty[String, Seq[Class[Exception]]], 
		TypesResponseInfo(
			Map[Int, ParameterRef](
			200 -> ParameterRef(Reference("⌿paths⌿/⌿get⌿responses⌿200"))
		), None), 
		StateResponseInfo(
				Map[Int, State](
					200 -> Self
			), None), 
		Set.empty[Security.Constraint]))

def packageName: Option[String] = Some("admin")

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object security_api_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿ErrorModel") → 
		TypeDef(Reference("⌿definitions⌿ErrorModel"), 
			Seq(
					Field(Reference("⌿definitions⌿ErrorModel⌿code"), Intgr(TypeMeta(Some("int32"), List()))),
					Field(Reference("⌿definitions⌿ErrorModel⌿message"), Str(None, TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 2"), List())),
	Reference("⌿definitions⌿Pet") → 
		TypeDef(Reference("⌿definitions⌿Pet"), 
			Seq(
					Field(Reference("⌿definitions⌿Pet⌿name"), Str(None, TypeMeta(None, List()))),
					Field(Reference("⌿definitions⌿Pet⌿tag"), Opt(Str(None, TypeMeta(None, List())), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 2"), List())),
	Reference("⌿paths⌿/pets/{id}⌿get⌿id") → 
		Arr(Str(None, TypeMeta(None, List())), TypeMeta(None, List()), "csv"),
	Reference("⌿paths⌿/pets/{id}⌿get⌿responses⌿default") → 
		TypeDef(Reference("⌿definitions⌿ErrorModel"), 
			Seq(
					Field(Reference("⌿definitions⌿ErrorModel⌿code"), Intgr(TypeMeta(Some("int32"), List()))),
					Field(Reference("⌿definitions⌿ErrorModel⌿message"), Str(None, TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 2"), List())),
	Reference("⌿paths⌿/pets/{id}⌿get⌿responses⌿200") → 
		ArrResult(TypeDef(Reference("⌿definitions⌿Pet"), 
			Seq(
					Field(Reference("⌿definitions⌿Pet⌿name"), Str(None, TypeMeta(None, List()))),
					Field(Reference("⌿definitions⌿Pet⌿tag"), Opt(Str(None, TypeMeta(None, List())), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 2"), List())), TypeMeta(None, List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](
	ParameterRef(	Reference("⌿paths⌿/pets/{id}⌿get⌿id")) → Parameter("id", Arr(Str(None, TypeMeta(None, List())), TypeMeta(None, List()), "csv"), None, None, "[^/]+", encode = true, ParameterPlace.withName("path"))
) 
 def basePath: String = "/v1" 
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	"petstoreImplicit" -> OAuth2Definition(None, Some(new URL("http://petstore.swagger.wordnik.com/oauth/dialog")), Map[String, String]( "admin:org" -> "Fully manage organization, teams, and memberships." ,  "user:email" -> "Grants read access to a user’s email addresses." ,  "read:org" -> "Read-only access to organization, teams, and membership." ,  "public_repo" -> "Grants read/write access to code, commit statuses, and deployment statuses for public repositories and organizations." ,  "write:public_key" -> "Create, list, and view details for public keys." ,  "repo_deployment" -> "Grants access to deployment statuses for public and private repositories. This scope is only necessary to grant other users or services access to deployment statuses, without granting access to the code." ,  "write:repo_hook" -> "Grants read, write, and ping access to hooks in public or private repositories." ,  "admin:public_key" -> "Fully manage public keys." ,  "repo:status" -> "Grants read/write access to public and private repository commit statuses. This scope is only necessary to grant other users or services access to private repository commit statuses without granting access to the code." ,  "gist" -> "Grants write access to gists." ,  "user:follow" -> "Grants access to follow or unfollow other users." ,  "repo" -> "Grants read/write access to code, commit statuses, and deployment statuses for public and private repositories and organizations." ,  "read:repo_hook" -> "Grants read and ping access to hooks in public or private repositories." ,  "notifications" -> "Grants read access to a user’s notifications. repo also provides this access." ,  "read:public_key" -> "List and view details for public keys." ,  "admin:repo_hook" -> "Grants read, write, ping, and delete access to hooks in public or private repositories." ,  "user" -> "Grants read/write access to profile info only. Note that this scope includes user:email and user:follow." ,  "write:org" -> "Publicize and unpublicize organization membership." ,  "delete_repo" -> "Grants access to delete adminable repositories." )),
	"githubAccessCode" -> OAuth2Definition(None, Some(new URL("https://github.com/login/oauth/access_token")), Map[String, String]( "admin:org" -> "Fully manage organization, teams, and memberships." ,  "user:email" -> "Grants read access to a user’s email addresses." ,  "read:org" -> "Read-only access to organization, teams, and membership." ,  "public_repo" -> "Grants read/write access to code, commit statuses, and deployment statuses for public repositories and organizations." ,  "write:public_key" -> "Create, list, and view details for public keys." ,  "repo_deployment" -> "Grants access to deployment statuses for public and private repositories. This scope is only necessary to grant other users or services access to deployment statuses, without granting access to the code." ,  "write:repo_hook" -> "Grants read, write, and ping access to hooks in public or private repositories." ,  "admin:public_key" -> "Fully manage public keys." ,  "repo:status" -> "Grants read/write access to public and private repository commit statuses. This scope is only necessary to grant other users or services access to private repository commit statuses without granting access to the code." ,  "gist" -> "Grants write access to gists." ,  "user:follow" -> "Grants access to follow or unfollow other users." ,  "repo" -> "Grants read/write access to code, commit statuses, and deployment statuses for public and private repositories and organizations." ,  "read:repo_hook" -> "Grants read and ping access to hooks in public or private repositories." ,  "notifications" -> "Grants read access to a user’s notifications. repo also provides this access." ,  "read:public_key" -> "List and view details for public keys." ,  "admin:repo_hook" -> "Grants read, write, ping, and delete access to hooks in public or private repositories." ,  "user" -> "Grants read/write access to profile info only. Note that this scope includes user:email and user:follow." ,  "write:org" -> "Publicize and unpublicize organization membership." ,  "delete_repo" -> "Grants access to delete adminable repositories." )),
	"petstorePassword" -> OAuth2Definition(None, Some(new URL("http://petstore.swagger.wordnik.com/oauth/dialog")), Map[String, String]( "user" -> "Grants read/write access to profile" ,  "admin" -> "Fully manage" )),
	"justBasicStuff" -> Basic(None),
	"petstoreApplication" -> OAuth2Definition(None, Some(new URL("http://petstore.swagger.wordnik.com/oauth/token")), Map[String, String]( "user" -> "Grants read/write access to profile" ,  "admin" -> "Fully manage" )),
	"internalApiKey" -> ApiKey(None, "api_key", ParameterPlace.withName("header"))
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq(
	ApiCall(GET, Path(Reference("⌿pets⌿{id}")), 
		HandlerCall(
			"security.api.yaml",
			"SecurityApiYaml",
			instantiate = false,
			"getPetsById",parameters = 
			Seq(
				ParameterRef(Reference("⌿paths⌿/pets/{id}⌿get⌿id"))
				)
			), 
		Set(MimeType("application/json")), 
		Set(MimeType("application/json"), MimeType("text/html")), 
		Map.empty[String, Seq[Class[Exception]]], 
		TypesResponseInfo(
			Map[Int, ParameterRef](
			200 -> ParameterRef(Reference("⌿paths⌿/pets/{id}⌿get⌿responses⌿200"))
		), Some(	ParameterRef(Reference("⌿paths⌿/pets/{id}⌿get⌿responses⌿default")))), 
		StateResponseInfo(
				Map[Int, State](
					200 -> Self
			), Some(Self)), 
		Set(
			OAuth2Constraint("githubAccessCode", OAuth2Definition(None, Some(new URL("https://github.com/login/oauth/access_token")), Map[String, String]( "admin:org" -> "Fully manage organization, teams, and memberships." ,  "user:email" -> "Grants read access to a user’s email addresses." ,  "read:org" -> "Read-only access to organization, teams, and membership." ,  "public_repo" -> "Grants read/write access to code, commit statuses, and deployment statuses for public repositories and organizations." ,  "write:public_key" -> "Create, list, and view details for public keys." ,  "repo_deployment" -> "Grants access to deployment statuses for public and private repositories. This scope is only necessary to grant other users or services access to deployment statuses, without granting access to the code." ,  "write:repo_hook" -> "Grants read, write, and ping access to hooks in public or private repositories." ,  "admin:public_key" -> "Fully manage public keys." ,  "repo:status" -> "Grants read/write access to public and private repository commit statuses. This scope is only necessary to grant other users or services access to private repository commit statuses without granting access to the code." ,  "gist" -> "Grants write access to gists." ,  "user:follow" -> "Grants access to follow or unfollow other users." ,  "repo" -> "Grants read/write access to code, commit statuses, and deployment statuses for public and private repositories and organizations." ,  "read:repo_hook" -> "Grants read and ping access to hooks in public or private repositories." ,  "notifications" -> "Grants read access to a user’s notifications. repo also provides this access." ,  "read:public_key" -> "List and view details for public keys." ,  "admin:repo_hook" -> "Grants read, write, ping, and delete access to hooks in public or private repositories." ,  "user" -> "Grants read/write access to profile info only. Note that this scope includes user:email and user:follow." ,  "write:org" -> "Publicize and unpublicize organization membership." ,  "delete_repo" -> "Grants access to delete adminable repositories." )), Set("user")),
			ApiKeyConstraint("internalApiKey", ApiKey(None, "api_key", ParameterPlace.withName("header")))
		)))

def packageName: Option[String] = Some("security.api.yaml")

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object nested_arrays_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿Activity") → 
		TypeDef(Reference("⌿definitions⌿Activity"), 
			Seq(
					Field(Reference("⌿definitions⌿Activity⌿actions"), Opt(Str(None, TypeMeta(Some("The text of the error message"), List())), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿definitions⌿Example") → 
		TypeDef(Reference("⌿definitions⌿Example"), 
			Seq(
					Field(Reference("⌿definitions⌿Example⌿messages"), Opt(Arr(Arr(TypeDef(Reference("⌿definitions⌿Activity"), 
			Seq(
						Field(Reference("⌿definitions⌿Activity⌿actions"), Opt(Str(None, TypeMeta(Some("The text of the error message"), List())), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 1"), List())), TypeMeta(None, List()), "csv"), TypeMeta(Some("The text of the error message"), List()), "csv"), TypeMeta(None, List()))),
					Field(Reference("⌿definitions⌿Example⌿nestedArrays"), Opt(Arr(Arr(Arr(Arr(Str(Some("nested arrays"), TypeMeta(Some("nested arrays"), List())), TypeMeta(None, List()), "csv"), TypeMeta(None, List()), "csv"), TypeMeta(None, List()), "csv"), TypeMeta(None, List()), "csv"), TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 2"), List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String = "/api" 
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq()

def packageName: Option[String] = None

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package org.apache.spark.banzaicloud.metrics.sink

import java.net.URL
import java.util.Properties

import com.banzaicloud.spark.metrics.sink.PrometheusSink.SinkConfig
import com.codahale.metrics.MetricRegistry
import io.prometheus.client.exporter.PushGateway
import org.apache.spark.banzaicloud.metrics.sink.PrometheusSink.SinkConfigProxy
import org.apache.spark.internal.config
import org.apache.spark.metrics.sink.Sink
import org.apache.spark.{SecurityManager, SparkConf, SparkEnv}

object PrometheusSink {

  class SinkConfigProxy extends SinkConfig {
    // SparkEnv may become available only after metrics sink creation thus retrieving
    // SparkConf from spark env here and not during the creation/initialisation of PrometheusSink.
    @transient
    private lazy val sparkConfig = Option(SparkEnv.get).map(_.conf).getOrElse(new SparkConf(true))

    // Don't use sparkConf.getOption("spark.metrics.namespace") as the underlying string won't be substituted.
    def metricsNamespace: Option[String] = sparkConfig.get(config.METRICS_NAMESPACE)
    def sparkAppId: Option[String] = sparkConfig.getOption("spark.app.id")
    def sparkAppName: Option[String] = sparkConfig.getOption("spark.app.name")
    def executorId: Option[String] = sparkConfig.getOption("spark.executor.id")
  }
}

class PrometheusSink(property: Properties,
                     registry: MetricRegistry,
                     securityMgr: SecurityManager,
                     sinkConfig: SinkConfig,
                     pushGatewayBuilder: URL => PushGateway)
  extends com.banzaicloud.spark.metrics.sink.PrometheusSink(property, registry, sinkConfig, pushGatewayBuilder) with Sink {

  // Constructor required by MetricsSystem::registerSinks()
  def this(property: Properties, registry: MetricRegistry, securityMgr: SecurityManager) = {
    this(
      property,
      registry,
      securityMgr,
      new SinkConfigProxy,
      new PushGateway(_)
    )
  }
} 

package org.apache.spark.deploy

import java.net.URL

import scala.collection.mutable
import scala.io.Source

import org.scalatest.FunSuite

import org.apache.spark.scheduler.cluster.ExecutorInfo
import org.apache.spark.scheduler.{SparkListenerExecutorAdded, SparkListener}
import org.apache.spark.{SparkConf, SparkContext, LocalSparkContext}

class LogUrlsStandaloneSuite extends FunSuite with LocalSparkContext {

  
  private val WAIT_TIMEOUT_MILLIS = 10000

  test("verify that correct log urls get propagated from workers") {
    sc = new SparkContext("local-cluster[2,1,512]", "test")

    val listener = new SaveExecutorInfo
    sc.addSparkListener(listener)

    // Trigger a job so that executors get added
    sc.parallelize(1 to 100, 4).map(_.toString).count()

    assert(sc.listenerBus.waitUntilEmpty(WAIT_TIMEOUT_MILLIS))
    listener.addedExecutorInfos.values.foreach { info =>
      assert(info.logUrlMap.nonEmpty)
      // Browse to each URL to check that it's valid
      info.logUrlMap.foreach { case (logType, logUrl) =>
        val html = Source.fromURL(logUrl).mkString
        assert(html.contains(s"$logType log page"))
      }
    }
  }

  test("verify that log urls reflect SPARK_PUBLIC_DNS (SPARK-6175)") {
    val SPARK_PUBLIC_DNS = "public_dns"
    class MySparkConf extends SparkConf(false) {
      override def getenv(name: String) = {
        if (name == "SPARK_PUBLIC_DNS") SPARK_PUBLIC_DNS
        else super.getenv(name)
      }

      override def clone: SparkConf = {
        new MySparkConf().setAll(getAll)
      }
    }
    val conf = new MySparkConf()
    sc = new SparkContext("local-cluster[2,1,512]", "test", conf)

    val listener = new SaveExecutorInfo
    sc.addSparkListener(listener)

    // Trigger a job so that executors get added
    sc.parallelize(1 to 100, 4).map(_.toString).count()

    assert(sc.listenerBus.waitUntilEmpty(WAIT_TIMEOUT_MILLIS))
    listener.addedExecutorInfos.values.foreach { info =>
      assert(info.logUrlMap.nonEmpty)
      info.logUrlMap.values.foreach { logUrl =>
        assert(new URL(logUrl).getHost === SPARK_PUBLIC_DNS)
      }
    }
  }

  private class SaveExecutorInfo extends SparkListener {
    val addedExecutorInfos = mutable.Map[String, ExecutorInfo]()

    override def onExecutorAdded(executor: SparkListenerExecutorAdded) {
      addedExecutorInfos(executor.executorId) = executor.executorInfo
    }
  }
} 

package org.apache.spark.util

import java.net.{URLClassLoader, URL}
import java.util.Enumeration
import java.util.concurrent.ConcurrentHashMap

import scala.collection.JavaConversions._

import org.apache.spark.util.ParentClassLoader


  private val locks = new ConcurrentHashMap[String, Object]()

  override def loadClass(name: String, resolve: Boolean): Class[_] = {
    var lock = locks.get(name)
    if (lock == null) {
      val newLock = new Object()
      lock = locks.putIfAbsent(name, newLock)
      if (lock == null) {
        lock = newLock
      }
    }

    lock.synchronized {
      try {
        super.loadClass(name, resolve)
      } catch {
        case e: ClassNotFoundException =>
          parentClassLoader.loadClass(name, resolve)
      }
    }
  }

  override def getResource(name: String): URL = {
    val url = super.findResource(name)
    val res = if (url != null) url else parentClassLoader.getResource(name)
    res
  }

  override def getResources(name: String): Enumeration[URL] = {
    val urls = super.findResources(name)
    val res =
      if (urls != null && urls.hasMoreElements()) {
        urls
      } else {
        parentClassLoader.getResources(name)
      }
    res
  }

  override def addURL(url: URL) {
    super.addURL(url)
  }

} 

package wvlet.airframe.http.finagle

import java.io.IOException
import java.net.URL

import wvlet.airframe.control.Control
import wvlet.airframe.http.{Endpoint, Router}
import wvlet.airspec.AirSpec
import wvlet.log.io.IOUtil


class FinagleDesignTest extends AirSpec {
  trait MyTestServer {
    @Endpoint(path = "/hello")
    def hello: String = {
      "hello"
    }
  }

  def newConfig = FinagleServerConfig(router = Router.of[MyTestServer])

  def `start server`: Unit = {
    finagleDefaultDesign
      .bind[FinagleServerConfig].toInstance(newConfig)
      .bind[FinagleSyncClient].toProvider { server: FinagleServer => Finagle.newSyncClient(server.localAddress) }
      .noLifeCycleLogging
      .build[FinagleSyncClient] { client =>
        // The server will start here
        val msg = client.get[String]("/hello")
        msg shouldBe "hello"
      }
  }

  def `no-server design` = {
    val config = newConfig
    finagleBaseDesign
      .bind[FinagleServerConfig].toInstance(config)
      .noLifeCycleLogging
      .build[FinagleServerFactory] { factory =>
        // No server should start here
        intercept[IOException] {
          Control.withResource(new URL(s"http://localhost:${config.port}").openStream()) { in =>
            IOUtil.readAsString(in)
          }
        }
      }
  }

  def `build a server from factory` = {
    finagleBaseDesign.noLifeCycleLogging.build[FinagleServerFactory] { factory =>
      val s1 = factory.newFinagleServer(newConfig)
      Control.withResource(FinagleClient.newSyncClient(s1.localAddress)) { client =>
        client.get[String]("/hello") shouldBe "hello"
      }
    }
  }
} 

package slide

import java.io.{File, FileOutputStream}
import java.net.{URL, URLConnection}
import java.nio.channels.{Channels, ReadableByteChannel}

class FileManager {

    var currentFile: String = ""
    var numberOfDownloads: Int = 0

    def downloadFile(dlsite: String, path: String): Unit = {
        val url: URL = new URL(dlsite)
        val file: File = new File(path)

        if (isConnected(url)) {
            currentFile = path
            onDownloadStart()

            new Thread(new Runnable {
                override def run(): Unit = {
                    try {
                        val rbc: ReadableByteChannel = Channels.newChannel(url.openStream())
                        val fos: FileOutputStream = new FileOutputStream(file)

                        fos.getChannel.transferFrom(rbc, 0, java.lang.Long.MAX_VALUE)
                        fos.close()

                        numberOfDownloads += 1
                        onDownloadFinished()
                    } catch {
                        case e: Exception =>
                            println("Error: Could not download ADB, please run as Administrator")
                    }
                }
            }).start()
        }
    }

    def isConnected(site: URL): Boolean = {
        try {
            // test connection
            val conn: URLConnection = site.openConnection()
            conn.setConnectTimeout(5000)
            conn.getContent

            true
        } catch {
            case e: Exception => false
        }
    }

    def onDownloadStart(): Unit = {}

    def onDownloadFinished(): Unit = {}

    // var onDownloadStart: () => Unit = null
    // var onDownloadFinished: () => Unit = null
} 

package sbtorgpolicies.arbitraries

import org.scalacheck.{Arbitrary, Gen}
import sbtorgpolicies.model.{ApacheLicense, CustomLicense, License, MITLicense}
import java.net.URL

trait modelArbitratries {

  implicit val URLArbitratry: Arbitrary[URL] =
    Arbitrary {
      for {
        protocol <- Gen.oneOf("http", "https", "ftp", "file")
        domain   <- Gen.alphaNumStr
        tld      <- Gen.oneOf("com", "io", "net")
        path     <- Arbitrary.arbitrary[String]
      } yield new URL(s"$protocol://$domain.$tld/$path")
    }

  val customLicenseArbitrary: Arbitrary[License] =
    Arbitrary {
      for {
        name <- Arbitrary.arbitrary[String]
        url  <- Arbitrary.arbitrary[URL]
      } yield CustomLicense(name, url)
    }

  implicit val licenseArbitrary: Arbitrary[License] =
    Arbitrary {
      Gen.oneOf[License](
        Gen.oneOf[License](MITLicense, ApacheLicense),
        customLicenseArbitrary.arbitrary
      )
    }
} 

package sbtorgpolicies

import java.io._
import java.net.URL
import java.nio.charset.Charset
import java.nio.file.Path
import java.nio.file.Paths.get

import cats.syntax.either._
import sbtorgpolicies.exceptions.IOException

import scala.io.Source
import scala.language.implicitConversions

package object io {

  type IOResult[T] = Either[IOException, T]

  object syntax {

    implicit def eitherFilterSyntax[T](either: Either[Throwable, T]): FilteredEitherOps[T] =
      new FilteredEitherOps(either)

    implicit def fileNameSyntax(fileName: String): FileNameOps = new FileNameOps(fileName)

    final class FilteredEitherOps[T](either: Either[Throwable, T]) {

      def withFilter(f: T => Boolean): Either[Throwable, T] = either match {
        case Right(r) if !f(r) =>
          new IllegalStateException("Filter condition has not been satisfied").asLeft[T]
        case _ =>
          either
      }
    }

    final class FileNameOps(filename: String) {

      def toPath: Path = get(filename)

      def toFile: File = new File(filename.fixPath)

      def fixPath: String = filename.replaceAll("/", File.separator)

      def ensureFinalSlash: String =
        filename +
          (if (filename.endsWith(File.separator)) ""
           else File.separator)
    }
  }

  object IO {

    def file(path: String): File = new File(path)

    def url(address: String): URL = new URL(address)

    def readLines(file: File): Iterator[String] =
      Source.fromFile(file).getLines()

    def readBytes(file: File): Array[Byte] = {
      val is: InputStream    = new FileInputStream(file)
      val array: Array[Byte] = Stream.continually(is.read).takeWhile(_ != -1).map(_.toByte).toArray
      is.close()
      array
    }

    def write(file: File, content: String, charset: Charset = Charset.forName("UTF-8")): Unit = {
      val writer = new BufferedWriter(
        new OutputStreamWriter(new FileOutputStream(file, false), charset)
      )
      writer.write(content)
      writer.close()
    }

    def relativize(base: File, file: File): Option[String] = {

      def ensureEndingSlash: Option[String] = {
        val path = base.getAbsolutePath
        path.lastOption.map {
          case c if c == File.separatorChar => path
          case _                            => path + File.separatorChar
        }
      }

      val baseFileString = if (base.isDirectory) ensureEndingSlash else None
      val pathString     = file.getAbsolutePath
      baseFileString flatMap {
        case baseString if pathString.startsWith(baseString) =>
          Some(pathString.substring(baseString.length))
        case _ => None
      }
    }

  }
} 

package sbtorgpolicies

import java.net.URL

import net.jcazevedo.moultingyaml._
import sbtorgpolicies.io.IO

object model {

  sealed abstract class License(val name: String, val url: URL) {
    def tupled: (String, URL) = (name, url)
    def custom: License       = CustomLicense(this)
  }

  
  case class Dev(id: String, name: Option[String] = None, url: Option[String] = None) {
    def pomExtra: xml.NodeSeq =
      <developer>
        <id>{id}</id>
        {name.fold(xml.NodeSeq.Empty)(x => <name>{x}</name>)}
        <url>http://github.com/{id}</url>
      </developer>
  }

  object sbtV {
    val `0.13`: String = "0.13.18"
    val `1.0`: String  = "1.2.8"

    val crossSbtVersions: List[String] = List(`0.13`, `1.0`)
  }

  object scalac {

    val `2.10`: String = "2.10.7"
    val `2.11`: String = "2.11.12"
    val `2.12`: String = "2.12.9"
    val `2.13`: String = "2.13.0"

    val latestScalaVersion: String = `2.13`

    val crossScalaVersions: List[String] = List(`2.11`, `2.12`, `2.13`)

  }

  object YamlFormats extends DefaultYamlProtocol {

    implicit object AnyYamlFormat extends YamlFormat[Any] {
      def write(x: Any): YamlValue = x match {
        case n: Int            => YamlNumber(n)
        case n: Long           => YamlNumber(n)
        case n: Double         => YamlNumber(n)
        case s: String         => YamlString(s)
        case b: Boolean        => YamlBoolean(b)
        case x: Seq[_]         => seqFormat[Any].write(x)
        case m: Map[String, _] => mapFormat[String, Any].write(m)
        case t =>
          serializationError("Serialization Error - Non expected type " + t.getClass.getName)
      }

      def read(value: YamlValue): Any = value match {
        case YamlNumber(n)  => n.intValue()
        case YamlString(s)  => s
        case YamlBoolean(b) => b
        case _: YamlArray   => listFormat[Any].read(value)
        case _: YamlObject  => mapFormat[String, Any].read(value)
        case x =>
          deserializationError("Deserialization Error - it failed the deserialization of " + x)
      }
    }
  }

} 

package com.lightbend.rp.sbtreactiveapp.magic

import sbt.{ Attributed, File, IO }
import scala.collection.JavaConverters._
import java.net.URL
import com.typesafe.config.{ Config, ConfigFactory }

object Build {
  def annotate(prependInclude: Boolean, unmanagedConfigName: String, config: String): String =
    s"""|# Generated by sbt-reactive-app. To disable this, set the `prependRpConf` sbt key to `""`.
        |
        |""".stripMargin +
      (if (prependInclude) s"""include "$unmanagedConfigName"""" else "") +
      s"""|
          |$config""".stripMargin

  def withHeader(comment: String, config: String): String =
    s"""|# $comment
        |
        |$config""".stripMargin

  def extractRpToolingConf(
    managedConfigNames: Seq[String],
    dependencyClasspath: Seq[Attributed[File]],
    prependInclude: Boolean,
    unmanagedConfigName: String): String = {
    val dependencyClassLoader = new java.net.URLClassLoader(dependencyClasspath.files.map(_.toURI.toURL).toArray)

    val managedConfigs: List[URL] =
      managedConfigNames
        .flatMap(dependencyClassLoader.findResources(_).asScala)
        .toList

    annotate(
      prependInclude,
      unmanagedConfigName,
      (managedConfigs
        .foldLeft(Seq.empty[String]) {
          case (accum, conf) =>
            accum :+ withHeader(conf.toString, IO.readLinesURL(conf).mkString(IO.Newline))
        }).mkString(IO.Newline))
  }

  def makeConfig(dependencyClasspath: Seq[File]): Config = {
    val dependencyClassLoader = new java.net.URLClassLoader(dependencyClasspath.map(_.toURI.toURL).toArray)
    ConfigFactory.load(dependencyClassLoader)
  }
} 

package com.owlandrews.nescala.helpers

import com.owlandrews.nescala.Console

object File {
  import java.io.File
  import java.net.URL
  import java.io.{FileFilter, FileInputStream, FileOutputStream, ObjectInputStream, ObjectOutputStream}
  import javax.imageio.ImageIO

  import scala.util.Try
  import scala.xml.XML
  import scala.language.postfixOps

  import sys.process._

  import com.typesafe.config.ConfigFactory

  def Download(url: String, filename: String) = (for{
    url <- Try(new URL(url))
    conn <- Try(url.openConnection().connect())
    file <- Try(new File(filename))
  } yield Try(url  #> file !!)) map {x => new File(filename)}

  def Writer(filename: String)(op: java.io.PrintWriter => Unit) = {
    val p = new java.io.PrintWriter(new File(filename))
    try op(p)
    finally p.close()
  }

  def Write(filename: String, content: String) = {
    val res = new java.io.PrintWriter(new File(filename))
    res.write(content)
    res.close()
  }

  def Filter = new FileFilter {
    override def accept(pathname: File): Boolean = pathname.getName.toLowerCase.endsWith(".nes")
  }

  def Image(file:Try[File]) = file.map(ImageIO.read)

  def Image(filename:String) = Try(ImageIO.read(resource(filename)))

  def Xml(filename:String) = XML.load(resource("/database.xml"))

  def Config(filename:String) = {
    val file = new File(filename)
    file.exists() match {
      case true => ConfigFactory.parseFile(file)
      case false => ConfigFactory.empty()
    }
  }

  def SaveState(console:Console) = {
    val fos = new FileOutputStream(s"$ApplicationFolder/${console.cartridge.CRC}.save")
    val oos = new ObjectOutputStream(fos)

    oos.writeObject(console)
    oos.close()
  }

  def LoadState(crc:String):Try[Console] = Try {
    val fis = new FileInputStream(s"$ApplicationFolder/$crc.save")
    val ois = new ObjectInputStreamWithCustomClassLoader(fis)

    val console = ois.readObject.asInstanceOf[Console]
    ois.close()
    console
  }

  // Taken from: https://gist.github.com/ramn/5566596
  private class ObjectInputStreamWithCustomClassLoader(fileInputStream: FileInputStream) extends ObjectInputStream(fileInputStream) {
    override def resolveClass(desc: java.io.ObjectStreamClass): Class[_] = {
      try { Class.forName(desc.getName, false, getClass.getClassLoader) }
      catch { case ex: ClassNotFoundException => super.resolveClass(desc) }
    }
  }

  lazy val ApplicationFolder: File = {
    val settingDirectory = System.getProperty("user.home") + "/.nescala"
    val settings = new java.io.File(settingDirectory)
    if (!settings.exists()) settings.mkdir()
    settings
  }

  private def resource(filename:String) = getClass.getResourceAsStream(filename)
} 

package _090_failure_handling

import java.io.FileReader
import java.net.{MalformedURLException, URL}

object _06_Finally {

  { // Example 1
    val file = new FileReader("input.txt")
    try {
      // Use the file
    } finally {
      file.close() // Be sure to close the file
    }
  }

  
  //noinspection RemoveRedundantReturn
  def f(): Int = try return 1 finally return 2

  def g(): Int = try 1 finally 2
} 

package it.almawave.linkeddata.kb.http

import play.api.inject.guice.GuiceApplicationBuilder
import org.junit.Test
import org.junit.After
import play.api.Application
import org.junit.Before
import it.almawave.linkeddata.kb.utils.JSONHelper
import play.api.libs.ws.WSClient
import org.asynchttpclient.DefaultAsyncHttpClient
import play.api.libs.ws.ssl.SystemConfiguration
import akka.stream.ActorMaterializer
import play.api.libs.ws.ahc.AhcWSClient
import scala.concurrent.Await
import scala.concurrent.duration.Duration
import java.net.URL
import com.typesafe.config.ConfigFactory

class TestingHttpApi {

  var app: Application = null
  var conf = ConfigFactory.empty()
  var ws: WSClient = null
  var app_url = new URL("http://localhost:8080")

  @Test
  def testing_contexts() {

    //    curl -X GET http://localhost:8999/kb/v1/prefixes/lookup?prefix=no_pref 
    //    -H  "accept: application/json" 
    //    -H  "content-type: application/json"

    val fut = ws.url(s"http://localhost:8999/kb/v1/prefixes/lookup")
      .withHeaders(("accept", "application/json"))
      .withHeaders(("content-type", "application/json"))
      .withFollowRedirects(true)
      .withQueryString(("prefix", "muapit"))
      .get()

    val results = Await.result(fut, Duration.Inf)
    println(results.body)

  }

  @Before
  def before() {

    app = GuiceApplicationBuilder()
      .build()

    conf = app.configuration.underlying

    // play.app.local.url
    // play.server.http.address
    // play.server.http.port

    println(JSONHelper.writeToString(conf.root().unwrapped()))

    app_url = new URL(conf.getString("app.local.url"))

    println(s"\n\nrunning at ${app_url}")

    val materializer = ActorMaterializer()(app.actorSystem)
    ws = AhcWSClient()(materializer)

  }

  @After
  def after() {
    ws.close()
    app.stop()
  }

} 

package it.almawave.linkeddata.kb.explore

import java.net.URL
import it.almawave.linkeddata.kb.file.RDFFileRepository
import org.eclipse.rdf4j.repository.Repository
import it.almawave.kb.sparqlnew.SPARQL
import org.eclipse.rdf4j.sail.memory.model.MemLiteral
import org.eclipse.rdf4j.model.Literal
import org.eclipse.rdf4j.model.IRI
import org.eclipse.rdf4j.model.BNode
import java.net.URI
import org.eclipse.rdf4j.model.impl.BooleanLiteral
import org.eclipse.rdf4j.model.impl.DecimalLiteral
import org.eclipse.rdf4j.model.impl.NumericLiteral
import org.eclipse.rdf4j.model.impl.IntegerLiteral
import org.eclipse.rdf4j.model.impl.SimpleLiteral
import it.almawave.kb.sparqlnew.Framing


object MainExplore extends App {

  val rdf_source = new URL("file:///C:/Users/Al.Serafini/repos/DAF/semantic_standardization/ontologie-vocabolari-controllati/Ontologie/POI/latest/POI-AP_IT.ttl")

  val repo = new RDFFileRepository(rdf_source)

  val exp = new ExploreBox(repo)

  val _concepts = exp.concepts.toList

  println("\n\n##################################à")
  _concepts.foreach { uri =>

    val details = exp.details_of_concept(uri)._2.toList
    println(s"\nURI: <${uri}>")
    println(details.toList.map(x => s"\t${x._1} -> ${x._2}").mkString("\n"))

  }

}

class ExploreBox(repo: Repository) {

  import it.almawave.linkeddata.kb.utils.ModelAdapter
  //  import it.almawave.linkeddata.kb.catalog.SPARQL

  //  ModelAdapter.fromMap(map)

  def concepts: Seq[String] = {

    val fields = List("concept_uri", "klass_uri")

    SPARQL(repo).queryTuple("""
      PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
      PREFIX owl: <http://www.w3.org/2002/07/owl#>
      SELECT DISTINCT * 
      WHERE {
        
      	?concept_uri a / rdfs:subclassof* ?klass_uri .
      	FILTER(?klass_uri IN (rdfs:Class, owl:Class))
      	
      	FILTER(!isBlank(?concept_uri))
      
      }
    """)
      .map(item => item.filterKeys(x => fields.contains(x)))
      .flatten
      .map(_._2.toString())
      .distinct

  }

  def details_of_concept(uri: String) = {
    val _query = s"""
      SELECT DISTINCT * 
      WHERE {
      
        ?uri ?property_uri ?object_uri .
        FILTER(?uri = <${uri}>)
        
        OPTIONAL { ?property_uri a ?property_type . }
        
      }
    """
    //    println(s"\n\nSPARQL> ${_query}")
    val tuples: Seq[Map[String, Any]] = SPARQL(repo).queryTuple(_query)
    //    println("TUPLES.............")
    //    tuples.foreach(println)
    //    println(".............TUPLES\n")

    val flatten = tuples.map { tuple =>

      val prp = tuple.getOrElse("property_uri", "").toString()
      val obj = tuple.getOrElse("object_uri", "")
      Map(prp -> obj)

    }.toList

    (uri, Framing.mergeTuples(flatten))
  }

} 

package cn.piflow.bundle.http

import java.io.{BufferedReader, InputStreamReader, PrintWriter}
import java.net.{HttpURLConnection, InetAddress, URL, URLConnection}

import cn.piflow.Runner
import cn.piflow.conf.bean.FlowBean
import cn.piflow.conf.util.{FileUtil, OptionUtil}
import cn.piflow.util.{PropertyUtil, ServerIpUtil}
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
import org.apache.spark.sql.SparkSession
import org.h2.tools.Server
import org.junit.Test

import scala.util.parsing.json.JSON

class GetUrlTest {

  @Test
  def testFlow(): Unit ={

    //parse flow json
    val file = "src/main/resources/flow/http/getUrl.json"
    val flowJsonStr = FileUtil.fileReader(file)
    val map = OptionUtil.getAny(JSON.parseFull(flowJsonStr)).asInstanceOf[Map[String, Any]]
    println(map)

    //create flow
    val flowBean = FlowBean(map)
    val flow = flowBean.constructFlow()


    val ip = InetAddress.getLocalHost.getHostAddress
    cn.piflow.util.FileUtil.writeFile("server.ip=" + ip, ServerIpUtil.getServerIpFile())
    val h2Server = Server.createTcpServer("-tcp", "-tcpAllowOthers", "-tcpPort","50001").start()
    //execute flow
    val spark = SparkSession.builder()
      .master("local[12]")
      .appName("hive")
      .config("spark.driver.memory", "4g")
      .config("spark.executor.memory", "8g")
      .config("spark.cores.max", "8")
      .config("hive.metastore.uris",PropertyUtil.getPropertyValue("hive.metastore.uris"))
      .enableHiveSupport()
      .getOrCreate()

    val process = Runner.create()
      .bind(classOf[SparkSession].getName, spark)
      .bind("checkpoint.path", "")
      .bind("debug.path","")
      .start(flow);

    process.awaitTermination();
    val pid = process.pid();
    println(pid + "!!!!!!!!!!!!!!!!!!!!!")
    spark.close();
  }

} 

package cn.piflow.bundle.script


import java.net.{MalformedURLException, URL}

import cn.piflow.conf.bean.PropertyDescriptor
import cn.piflow.conf.util.{ImageUtil, MapUtil, PluginClassLoader}
import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}

import scala.language.experimental.macros
import scala.reflect.runtime.{universe => ru}


class ExecuteScala extends ConfigurableStop{
  override val authorEmail: String = "[email protected]"
  override val description: String = "Execute scala script"
  override val inportList: List[String] = List(Port.DefaultPort)
  override val outportList: List[String] = List(Port.DefaultPort)

  var packageName : String = "cn.piflow.bundle.script"
  var script : String = _
  var plugin : String = _

  override def setProperties(map: Map[String, Any]): Unit = {

    script = MapUtil.get(map,"script").asInstanceOf[String]
    plugin = MapUtil.get(map,"plugin").asInstanceOf[String]
  }

  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
    var descriptor : List[PropertyDescriptor] = List()

    val pluginName = new PropertyDescriptor()
      .name("plugin")
      .displayName("Plugin")
      .description("The class name of scala code. This field is generated automaticly.")
      .defaultValue("")
      .required(true)
    descriptor = pluginName :: descriptor

    val script = new PropertyDescriptor()
      .name("script")
      .displayName("script")
      .description("The code of scala. \nUse in.read() to get dataframe from upstream component. \nUse out.write() to write datafram to downstream component.")
      .defaultValue("")
      .required(true)
      .example("val df = in.read() \nval df1 = df.select(\"author\").filter($\"author\".like(\"%xjzhu%\")) \ndf1.show() \ndf.createOrReplaceTempView(\"person\") \nval df2 = spark.sql(\"select * from person where author like '%xjzhu%'\") \ndf2.show() \nout.write(df2)")

    descriptor = script :: descriptor
    descriptor
  }

  override def getIcon(): Array[Byte] = {
    ImageUtil.getImage("icon/script/scala.jpg")
  }

  override def getGroup(): List[String] = {
    List(StopGroup.ScriptGroup)
  }

  override def initialize(ctx: ProcessContext): Unit = {
    val flowName = ctx.getFlow().getFlowName()
  }

  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {

    val execMethod = "perform"
    val loader = new PluginClassLoader

    //when local run, use these codes
    //val scalaDir = PropertyUtil.getScalaPath()
    //val pluginurl = s"jar:file:$scalaDir/$plugin.jar!/"

    val userDir = System.getProperty("user.dir")
    //FileUtil.getJarFile(new File(userDir)).foreach(println(_))

    val pluginurl = s"jar:file:$userDir/$plugin.jar!/"
    println(s"Scala Plugin url : $pluginurl")
    var url : URL = null
    try
      url = new URL(pluginurl)
    catch {
      case e: MalformedURLException =>
        e.printStackTrace()
    }
    loader.addURLFile(url)

    val className = plugin.split("/").last.split(".jar")(0)
    val classMirror = ru.runtimeMirror(loader)
    println("staticModule: " + s"$packageName.$className")
    val classTest = classMirror.staticModule(s"$packageName.$className")
    val methods = classMirror.reflectModule(classTest)
    val objectMirror = classMirror.reflect(methods.instance)
    val method = methods.symbol.typeSignature.member(ru.TermName(s"$execMethod")).asMethod
    val result = objectMirror.reflectMethod(method)(in,out,pec)

  }

} 

package org.apache.s2graph.core.fetcher.tensorflow

import java.net.URL
import java.nio.file.Paths

import com.typesafe.config.Config
import org.apache.commons.io.IOUtils
import org.apache.s2graph.core._
import org.apache.s2graph.core.types.VertexId

import scala.concurrent.{ExecutionContext, Future}


object InceptionFetcher {
  val ModelPath = "modelPath"

  def getImageBytes(urlText: String): Array[Byte] = {
    val url = new URL(urlText)

    IOUtils.toByteArray(url)
  }

  def predict(graphDef: Array[Byte],
              labels: Seq[String])(imageBytes: Array[Byte], topK: Int = 10): Seq[(String, Float)] = {
    try {
      val image = LabelImage.constructAndExecuteGraphToNormalizeImage(imageBytes)
      try {
        val labelProbabilities = LabelImage.executeInceptionGraph(graphDef, image)
        val topKIndices = labelProbabilities.zipWithIndex.sortBy(_._1).reverse
          .take(Math.min(labelProbabilities.length, topK)).map(_._2)

        val ls = topKIndices.map { idx => (labels(idx), labelProbabilities(idx)) }

        ls
      } catch {
        case e: Throwable => Nil
      } finally if (image != null) image.close()
    }
  }
}

class InceptionFetcher(graph: S2GraphLike) extends EdgeFetcher {

  import InceptionFetcher._

  import scala.collection.JavaConverters._
  import org.apache.s2graph.core.TraversalHelper._
  val builder = graph.elementBuilder

  var graphDef: Array[Byte] = _
  var labels: Seq[String] = _

  override def init(config: Config)(implicit ec: ExecutionContext): Unit = {
    val modelPath = config.getString(ModelPath)
    graphDef = LabelImage.readAllBytesOrExit(Paths.get(modelPath, "tensorflow_inception_graph.pb"))
    labels = LabelImage.readAllLinesOrExit(Paths.get(modelPath, "imagenet_comp_graph_label_strings.txt")).asScala
  }

  override def close(): Unit = {}

  override def fetches(queryRequests: Seq[QueryRequest],
                       prevStepEdges: Map[VertexId, Seq[EdgeWithScore]])(implicit ec: ExecutionContext): Future[Seq[StepResult]] = {
    val stepResultLs = queryRequests.map { queryRequest =>
      val vertex = queryRequest.vertex
      val queryParam = queryRequest.queryParam
      val shouldBuildParents = queryRequest.query.queryOption.returnTree || queryParam.whereHasParent
      val parentEdges = if (shouldBuildParents) prevStepEdges.getOrElse(queryRequest.vertex.id, Nil) else Nil

      val urlText = vertex.innerId.toIdString()

      val edgeWithScores = predict(graphDef, labels)(getImageBytes(urlText), queryParam.limit).flatMap { case (label, score) =>
        val tgtVertexId = builder.newVertexId(queryParam.label.service,
          queryParam.label.tgtColumnWithDir(queryParam.labelWithDir.dir), label)

        val props: Map[String, Any] = if (queryParam.label.metaPropsInvMap.contains("score")) Map("score" -> score) else Map.empty
        val edge = graph.toEdge(vertex.innerId.value, tgtVertexId.innerId.value, queryParam.labelName, queryParam.direction, props = props)

        edgeToEdgeWithScore(queryRequest, edge, parentEdges)
      }

      StepResult(edgeWithScores, Nil, Nil)
    }

    Future.successful(stepResultLs)
  }

  override def fetchEdgesAll()(implicit ec: ExecutionContext): Future[Seq[S2EdgeLike]] =
    Future.successful(Nil)
} 

package gui

import java.net.URL
import javax.swing.ImageIcon

import enums.ConnectionMode

object ImageIcons {
    var usbIcon: ImageIcon = new ImageIcon(getImagePath(ConnectionMode.USB))
    var wifiIcon: ImageIcon = new ImageIcon(getImagePath(ConnectionMode.WIFI))

    private def getImagePath(mode: ConnectionMode): URL = {
        if (mode == ConnectionMode.USB) {
            this.getClass.getResource("res/img/usb.png")
        }
        else if (mode == ConnectionMode.WIFI) {
            this.getClass.getResource("res/img/wifi.png")
        } else {
            new URL("")
        }
    }
} 

package com.horizen

import java.io.File
import java.net.URL
import java.util.{Optional => JOptional}

import com.typesafe.config.{Config, ConfigFactory}
import net.ceedubs.ficus.Ficus._
import net.ceedubs.ficus.readers.ArbitraryTypeReader._
import scorex.core.settings.{ScorexSettings, SettingsReaders}
import scorex.util.ScorexLogging

import scala.compat.java8.OptionConverters.toScala


object SidechainSettingsReader
  extends ScorexLogging
    with SettingsReaders
{
  protected val sidechainSettingsName = "sidechain-sdk-settings.conf"

  def fromConfig(config: Config): SidechainSettings = {
    val webSocketConnectorConfiguration = config.as[WebSocketSettings]("scorex.websocket")
    val scorexSettings = config.as[ScorexSettings]("scorex")
    val genesisSetting = config.as[GenesisDataSettings]("scorex.genesis")
    val backwardTransfer = config.as[withdrawalEpochCertificateSettings]("scorex.withdrawalEpochCertificate")
    val walletSetting = config.as[WalletSettings]("scorex.wallet")
    SidechainSettings(scorexSettings, genesisSetting, webSocketConnectorConfiguration, backwardTransfer, walletSetting)
  }

  def readConfigFromPath(userConfigPath: String, applicationConfigPath: Option[String]): Config = {

    val userConfigFile: File = new File(userConfigPath)

    val userConfig: Option[Config] = if (userConfigFile.exists()) {
      Some(ConfigFactory.parseFile(userConfigFile))
    } else None

    val applicationConfigURL: Option[URL] = applicationConfigPath.map(filename => new File(filename))
      .filter(_.exists()).map(_.toURI.toURL)
      .orElse(applicationConfigPath.map(r => getClass.getClassLoader.getResource(r)))

    val applicationConfig: Option[Config] = if (applicationConfigURL.isDefined) {
      Some(ConfigFactory.parseURL(applicationConfigURL.get))
    } else None

    var config: Config = ConfigFactory.defaultOverrides()

    if (userConfig.isDefined)
      config = config.withFallback(userConfig.get)

    if (applicationConfig.isDefined)
      config = config.withFallback(applicationConfig.get)

    config = config
      .withFallback(ConfigFactory.parseResources(sidechainSettingsName))
      .withFallback(ConfigFactory.defaultReference())
      .resolve()

    config
  }

  def readConfigFromPath(userConfigPath: String, applicationConfigPath: JOptional[String]) : Config =
    readConfigFromPath(userConfigPath, toScala(applicationConfigPath))

  def read(userConfigPath: String, applicationConfigPath: Option[String]) : SidechainSettings =
    fromConfig(readConfigFromPath(userConfigPath, applicationConfigPath))
} 

package org.archive.archivespark.sparkling.util

import java.net.URL

import org.archive.url.WaybackURLKeyMaker

import scala.util.Try

object SurtUtil {
  private lazy val keyMaker = new WaybackURLKeyMaker()
  private lazy val validHostPattern = "[a-z]+\\,[\\p{L}\\p{M}0-9\\-\\,]+".r

  def fromUrl(url: String): String = Try(keyMaker.makeKey(url)).getOrElse(url)

  def fromUrl(url: String, baseUrl: String): String = {
    val resolved = new URL(new URL(toUrl(baseUrl)), url).toString
    fromUrl(resolved)
  }

  def toUrl(surt: String): String = {
    if (RegexUtil.matchesAbsoluteUrlStart(surt)) return surt
    surt.splitAt(surt.indexOf(')'))
    val (host, path) = surt.splitAt(surt.indexOf(')'))
    val hostSplit = host.split(',')
    "http://" + hostSplit.reverse.mkString(".") + path.drop(1)
  }

  def urlToSurtPrefixes(url: String, subdomains: Boolean = true, urlInSurtFormat: Boolean = false): Set[String] = {
    val surt = if (urlInSurtFormat) url else SurtUtil.fromUrl(if (RegexUtil.matchesAbsoluteUrlStart(url)) url else "http://" + url)
    val hostPath = surt.split("\\)", 2)
    val site = hostPath(0).trim
    if (site.isEmpty) Set.empty
    else hostPath.drop(1).headOption.map(_.trim.stripPrefix("/").stripSuffix("/")).filter(_.nonEmpty) match {
      case Some(path) => Set(site + ")/" + path + " ") ++ (if (path.contains("?")) Set.empty else Set(site + ")/" + path + "/", site + ")/" + path + "?"))
      case None => Set(site + ")") ++ (if (subdomains) Set(site + ",") else Seq.empty)
    }
  }

  def host(surt: String): String = {
    val slash = surt.indexOf('/')
    (if (slash < 0) surt else surt.take(slash)).stripSuffix(")")
  }

  def validateHost(surt: String): Option[String] = {
    Some(host(surt)).filter { host =>
      validHostPattern.pattern.matcher(host).matches && !{
        host.contains("--") || host.contains(",,") || host.split(',').exists(p => p.isEmpty || p.startsWith("-") || p.endsWith("-"))
      }
    }
  }

  def validate(url: String, urlInSurtFormat: Boolean = false): Option[String] = Some(if (urlInSurtFormat) url else fromUrl(url)).filter(validateHost(_).isDefined)
} 

package org.archive.archivespark.sparkling.http

import java.io.{BufferedInputStream, InputStream}
import java.net.{HttpURLConnection, URL, URLConnection}

import org.archive.archivespark.sparkling.logging.LogContext
import org.archive.archivespark.sparkling.util.Common

import scala.collection.JavaConverters._
import scala.util.Try

object HttpClient {
  val DefaultRetries: Int = 30
  val DefaultSleepMillis: Int = 1000
  val DefaultTimeoutMillis: Int = -1

  implicit val logContext: LogContext = LogContext(this)

  def request[R](url: String, headers: Map[String, String] = Map.empty, retries: Int = DefaultRetries, sleepMillis: Int = DefaultSleepMillis, timeoutMillis: Int = DefaultTimeoutMillis)(action: InputStream => R): R = rangeRequest(url, headers, retries = retries, sleepMillis = sleepMillis, timeoutMillis = timeoutMillis)(action)

  def rangeRequest[R](url: String, headers: Map[String, String] = Map.empty, offset: Long = 0, length: Long = -1, retries: Int = DefaultRetries, sleepMillis: Int = DefaultSleepMillis, timeoutMillis: Int = DefaultTimeoutMillis)(action: InputStream => R): R = {
    rangeRequestConnection(url, headers, offset, length, retries, sleepMillis, timeoutMillis) { case connection: HttpURLConnection =>
      val in = new BufferedInputStream(connection.getInputStream)
      val r = action(in)
      Try(in.close())
      r
    }
  }

  def requestMessage[R](url: String, headers: Map[String, String] = Map.empty, retries: Int = DefaultRetries, sleepMillis: Int = DefaultSleepMillis, timeoutMillis: Int = DefaultTimeoutMillis)(action: HttpMessage => R): R = rangeRequestMessage(url, headers, retries = retries, sleepMillis = sleepMillis, timeoutMillis = timeoutMillis)(action)

  def rangeRequestMessage[R](url: String, headers: Map[String, String] = Map.empty, offset: Long = 0, length: Long = -1, retries: Int = DefaultRetries, sleepMillis: Int = DefaultSleepMillis, timeoutMillis: Int = DefaultTimeoutMillis)(action: HttpMessage => R): R = {
    rangeRequestConnection(url, headers, offset, length, retries, sleepMillis, timeoutMillis) { case connection: HttpURLConnection =>
      val in = new BufferedInputStream(connection.getInputStream)
      val responseHeaders = connection.getHeaderFields.asScala.toMap.flatMap{case (k, v) => v.asScala.headOption.map((if (k == null) "" else k) -> _)}
      val message = new HttpMessage(connection.getResponseMessage, responseHeaders, in)
      val r = action(message)
      Try(in.close())
      r
    }
  }

  def requestConnection[R](url: String, headers: Map[String, String] = Map.empty, retries: Int = DefaultRetries, sleepMillis: Int = DefaultSleepMillis, timeoutMillis: Int = DefaultTimeoutMillis)(action: URLConnection => R): R = rangeRequestConnection(url, headers, retries = retries, sleepMillis = sleepMillis, timeoutMillis = timeoutMillis)(action)

  def rangeRequestConnection[R](url: String, headers: Map[String, String] = Map.empty, offset: Long = 0, length: Long = -1, retries: Int = DefaultRetries, sleepMillis: Int = DefaultSleepMillis, timeoutMillis: Int = DefaultTimeoutMillis)(action: URLConnection => R): R = {
    Common.timeoutWithReporter(timeoutMillis) { reporter =>
      val connection = Common.retry(retries, sleepMillis, (retry, e) => {
        "Request failed (" + retry + "/" + retries + "): " + url + " (" + offset + "-" + (if (length >= 0) length else "") + ") - " + e.getMessage
      }) { _ =>
        reporter.alive()
        val connection = new URL(url).openConnection()
        for ((key, value) <- headers) connection.addRequestProperty(key, value)
        if (offset > 0 || length >= 0) connection.addRequestProperty("Range", "bytes=" + offset + "-" + (if (length >= 0) offset + length - 1 else ""))
        connection.asInstanceOf[HttpURLConnection]
      }
      val r = action(connection)
      Try(connection.disconnect())
      r
    }
  }
} 

package com.intel.analytics.bigdl.dataset.text

import java.io.FileInputStream
import java.net.{URI, URL}

import com.intel.analytics.bigdl.dataset.Transformer
import opennlp.tools.sentdetect.{SentenceDetector, SentenceDetectorME, SentenceModel}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path}

import scala.collection.Iterator


class SentenceSplitter(sentFile: Option[String] = None)
  extends Transformer[String, Array[String]] {

  var modelIn: FileInputStream = _
  var model: SentenceModel = _
  var sentenceDetector: SentenceDetector = _

  def this(sentFileURL: URL) {
    this(Some(sentFileURL.getPath))
  }

  def this(sentFile: String) {
    this(Some(sentFile))
  }

  def close(): Unit = {
    if (modelIn != null) {
      modelIn.close()
    }
  }

  override def apply(prev: Iterator[String]): Iterator[Array[String]] =
    prev.map(x => {
      if (!sentFile.isDefined) {
        x.split('.')
      } else {
        if (sentenceDetector == null) {
          val src: Path = new Path(sentFile.get)
          val fs = src.getFileSystem(new Configuration())
          val in = fs.open(src)

          model = new SentenceModel(in)
          sentenceDetector = new SentenceDetectorME(model)
        }
        sentenceDetector.sentDetect(x)
      }
    })
}

object SentenceSplitter {
  def apply(sentFile: Option[String] = None):
    SentenceSplitter = new SentenceSplitter(sentFile)
  def apply(sentFileURL: URL):
    SentenceSplitter = new SentenceSplitter(sentFileURL)
  def apply(sentFile: String):
  SentenceSplitter = new SentenceSplitter(sentFile)
} 

package com.intel.analytics.bigdl.dataset.text

import java.io.FileInputStream
import java.net.{URI, URL}

import com.intel.analytics.bigdl.dataset.Transformer

import scala.collection.Iterator
import opennlp.tools.tokenize.{SimpleTokenizer, Tokenizer, TokenizerME, TokenizerModel}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path}



class SentenceTokenizer(tokenFile: Option[String] = None)
  extends Transformer[String, Array[String]] {

  var modelIn: FileInputStream = _
  var model: TokenizerModel = _

  var tokenizer: Tokenizer = _

  def this(tokenFile: URL) {
    this(Some(tokenFile.getPath))
  }

  def close(): Unit = {
    if (modelIn != null) {
      modelIn.close()
    }
  }

  override def apply(prev: Iterator[String]): Iterator[Array[String]] =
    prev.map(x => {
      if (tokenizer == null) {
        if (!tokenFile.isDefined) {
          tokenizer = SimpleTokenizer.INSTANCE
        } else {
          val src: Path = new Path(tokenFile.get)
          val fs = src.getFileSystem(new Configuration())
          val in = fs.open(src)
          model = new TokenizerModel(in)
          tokenizer = new TokenizerME(model)
        }
      }
      val words = tokenizer.tokenize(x)
      words
    })
}

object SentenceTokenizer {
  def apply(tokenFile: Option[String] = None):
    SentenceTokenizer = new SentenceTokenizer(tokenFile)
  def apply(tokenFile: URL):
    SentenceTokenizer = new SentenceTokenizer(tokenFile)
} 

package org.apache.spark.deploy.master.ui

import java.io.DataOutputStream
import java.net.{HttpURLConnection, URL}
import java.nio.charset.StandardCharsets
import java.util.Date

import scala.collection.mutable.HashMap

import org.mockito.Mockito.{mock, times, verify, when}
import org.scalatest.BeforeAndAfterAll

import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
import org.apache.spark.deploy.DeployMessages.{KillDriverResponse, RequestKillDriver}
import org.apache.spark.deploy.DeployTestUtils._
import org.apache.spark.deploy.master._
import org.apache.spark.rpc.{RpcEndpointRef, RpcEnv}


class MasterWebUISuite extends SparkFunSuite with BeforeAndAfterAll {

  val conf = new SparkConf
  val securityMgr = new SecurityManager(conf)
  val rpcEnv = mock(classOf[RpcEnv])
  val master = mock(classOf[Master])
  val masterEndpointRef = mock(classOf[RpcEndpointRef])
  when(master.securityMgr).thenReturn(securityMgr)
  when(master.conf).thenReturn(conf)
  when(master.rpcEnv).thenReturn(rpcEnv)
  when(master.self).thenReturn(masterEndpointRef)
  val masterWebUI = new MasterWebUI(master, 0)

  override def beforeAll() {
    super.beforeAll()
    masterWebUI.bind()
  }

  override def afterAll() {
    masterWebUI.stop()
    super.afterAll()
  }

  test("kill application") {
    val appDesc = createAppDesc()
    // use new start date so it isn't filtered by UI
    val activeApp = new ApplicationInfo(
      new Date().getTime, "app-0", appDesc, new Date(), null, Int.MaxValue)

    when(master.idToApp).thenReturn(HashMap[String, ApplicationInfo]((activeApp.id, activeApp)))

    val url = s"http://localhost:${masterWebUI.boundPort}/app/kill/"
    val body = convPostDataToString(Map(("id", activeApp.id), ("terminate", "true")))
    val conn = sendHttpRequest(url, "POST", body)
    conn.getResponseCode

    // Verify the master was called to remove the active app
    verify(master, times(1)).removeApplication(activeApp, ApplicationState.KILLED)
  }

  test("kill driver") {
    val activeDriverId = "driver-0"
    val url = s"http://localhost:${masterWebUI.boundPort}/driver/kill/"
    val body = convPostDataToString(Map(("id", activeDriverId), ("terminate", "true")))
    val conn = sendHttpRequest(url, "POST", body)
    conn.getResponseCode

    // Verify that master was asked to kill driver with the correct id
    verify(masterEndpointRef, times(1)).ask[KillDriverResponse](RequestKillDriver(activeDriverId))
  }

  private def convPostDataToString(data: Map[String, String]): String = {
    (for ((name, value) <- data) yield s"$name=$value").mkString("&")
  }

  
  private def sendHttpRequest(
      url: String,
      method: String,
      body: String = ""): HttpURLConnection = {
    val conn = new URL(url).openConnection().asInstanceOf[HttpURLConnection]
    conn.setRequestMethod(method)
    if (body.nonEmpty) {
      conn.setDoOutput(true)
      conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded")
      conn.setRequestProperty("Content-Length", Integer.toString(body.length))
      val out = new DataOutputStream(conn.getOutputStream)
      out.write(body.getBytes(StandardCharsets.UTF_8))
      out.close()
    }
    conn
  }
} 

package org.apache.spark.deploy

import java.net.URL

import scala.collection.mutable
import scala.io.Source

import org.apache.spark.{LocalSparkContext, SparkContext, SparkFunSuite}
import org.apache.spark.scheduler.{SparkListener, SparkListenerExecutorAdded}
import org.apache.spark.scheduler.cluster.ExecutorInfo
import org.apache.spark.util.SparkConfWithEnv

class LogUrlsStandaloneSuite extends SparkFunSuite with LocalSparkContext {

  
  private val WAIT_TIMEOUT_MILLIS = 10000

  test("verify that correct log urls get propagated from workers") {
    sc = new SparkContext("local-cluster[2,1,1024]", "test")

    val listener = new SaveExecutorInfo
    sc.addSparkListener(listener)

    // Trigger a job so that executors get added
    sc.parallelize(1 to 100, 4).map(_.toString).count()

    sc.listenerBus.waitUntilEmpty(WAIT_TIMEOUT_MILLIS)
    listener.addedExecutorInfos.values.foreach { info =>
      assert(info.logUrlMap.nonEmpty)
      // Browse to each URL to check that it's valid
      info.logUrlMap.foreach { case (logType, logUrl) =>
        val html = Source.fromURL(logUrl).mkString
        assert(html.contains(s"$logType log page"))
      }
    }
  }

  test("verify that log urls reflect SPARK_PUBLIC_DNS (SPARK-6175)") {
    val SPARK_PUBLIC_DNS = "public_dns"
    val conf = new SparkConfWithEnv(Map("SPARK_PUBLIC_DNS" -> SPARK_PUBLIC_DNS)).set(
      "spark.extraListeners", classOf[SaveExecutorInfo].getName)
    sc = new SparkContext("local-cluster[2,1,1024]", "test", conf)

    // Trigger a job so that executors get added
    sc.parallelize(1 to 100, 4).map(_.toString).count()

    sc.listenerBus.waitUntilEmpty(WAIT_TIMEOUT_MILLIS)
    val listeners = sc.listenerBus.findListenersByClass[SaveExecutorInfo]
    assert(listeners.size === 1)
    val listener = listeners(0)
    listener.addedExecutorInfos.values.foreach { info =>
      assert(info.logUrlMap.nonEmpty)
      info.logUrlMap.values.foreach { logUrl =>
        assert(new URL(logUrl).getHost === SPARK_PUBLIC_DNS)
      }
    }
  }
}

private[spark] class SaveExecutorInfo extends SparkListener {
  val addedExecutorInfos = mutable.Map[String, ExecutorInfo]()

  override def onExecutorAdded(executor: SparkListenerExecutorAdded) {
    addedExecutorInfos(executor.executorId) = executor.executorInfo
  }
} 

package org.apache.spark.util

import java.net.{URL, URLClassLoader}
import java.util.Enumeration

import scala.collection.JavaConverters._


private[spark] class ChildFirstURLClassLoader(urls: Array[URL], parent: ClassLoader)
  extends MutableURLClassLoader(urls, null) {

  private val parentClassLoader = new ParentClassLoader(parent)

  override def loadClass(name: String, resolve: Boolean): Class[_] = {
    try {
      super.loadClass(name, resolve)
    } catch {
      case e: ClassNotFoundException =>
        parentClassLoader.loadClass(name, resolve)
    }
  }

  override def getResource(name: String): URL = {
    val url = super.findResource(name)
    val res = if (url != null) url else parentClassLoader.getResource(name)
    res
  }

  override def getResources(name: String): Enumeration[URL] = {
    val childUrls = super.findResources(name).asScala
    val parentUrls = parentClassLoader.getResources(name).asScala
    (childUrls ++ parentUrls).asJavaEnumeration
  }

  override def addURL(url: URL) {
    super.addURL(url)
  }

} 

package org.apache.spark.scheduler.local

import java.io.File
import java.net.URL
import java.nio.ByteBuffer

import org.apache.spark.{SparkConf, SparkContext, SparkEnv, TaskState}
import org.apache.spark.TaskState.TaskState
import org.apache.spark.executor.{Executor, ExecutorBackend}
import org.apache.spark.internal.Logging
import org.apache.spark.launcher.{LauncherBackend, SparkAppHandle}
import org.apache.spark.rpc.{RpcCallContext, RpcEndpointRef, RpcEnv, ThreadSafeRpcEndpoint}
import org.apache.spark.scheduler._
import org.apache.spark.scheduler.cluster.ExecutorInfo

private case class ReviveOffers()

private case class StatusUpdate(taskId: Long, state: TaskState, serializedData: ByteBuffer)

private case class KillTask(taskId: Long, interruptThread: Boolean)

private case class StopExecutor()


  def getUserClasspath(conf: SparkConf): Seq[URL] = {
    val userClassPathStr = conf.getOption("spark.executor.extraClassPath")
    userClassPathStr.map(_.split(File.pathSeparator)).toSeq.flatten.map(new File(_).toURI.toURL)
  }

  launcherBackend.connect()

  override def start() {
    val rpcEnv = SparkEnv.get.rpcEnv
    val executorEndpoint = new LocalEndpoint(rpcEnv, userClassPath, scheduler, this, totalCores)
    localEndpoint = rpcEnv.setupEndpoint("LocalSchedulerBackendEndpoint", executorEndpoint)
    listenerBus.post(SparkListenerExecutorAdded(
      System.currentTimeMillis,
      executorEndpoint.localExecutorId,
      new ExecutorInfo(executorEndpoint.localExecutorHostname, totalCores, Map.empty)))
    launcherBackend.setAppId(appId)
    launcherBackend.setState(SparkAppHandle.State.RUNNING)
  }

  override def stop() {
    stop(SparkAppHandle.State.FINISHED)
  }

  override def reviveOffers() {
    localEndpoint.send(ReviveOffers)
  }

  override def defaultParallelism(): Int =
    scheduler.conf.getInt("spark.default.parallelism", totalCores)

  override def killTask(taskId: Long, executorId: String, interruptThread: Boolean) {
    localEndpoint.send(KillTask(taskId, interruptThread))
  }

  override def statusUpdate(taskId: Long, state: TaskState, serializedData: ByteBuffer) {
    localEndpoint.send(StatusUpdate(taskId, state, serializedData))
  }

  override def applicationId(): String = appId

  private def stop(finalState: SparkAppHandle.State): Unit = {
    localEndpoint.ask(StopExecutor)
    try {
      launcherBackend.setState(finalState)
    } finally {
      launcherBackend.close()
    }
  }

} 

package org.apache.spark.sql.hive

import java.net.URL

import org.apache.spark.SparkFunSuite


class ClasspathDependenciesSuite extends SparkFunSuite {
  private val classloader = this.getClass.getClassLoader

  private def assertLoads(classname: String): Unit = {
    val resourceURL: URL = Option(findResource(classname)).getOrElse {
      fail(s"Class $classname not found as ${resourceName(classname)}")
    }

    logInfo(s"Class $classname at $resourceURL")
    classloader.loadClass(classname)
  }

  private def findResource(classname: String): URL = {
    val resource = resourceName(classname)
    classloader.getResource(resource)
  }

  private def resourceName(classname: String): String = {
    classname.replace(".", "/") + ".class"
  }

  private def assertClassNotFound(classname: String): Unit = {
    Option(findResource(classname)).foreach { resourceURL =>
      fail(s"Class $classname found at $resourceURL")
    }

    intercept[ClassNotFoundException] {
      classloader.loadClass(classname)
    }
  }

  test("shaded Protobuf") {
    assertLoads("org.apache.hive.com.google.protobuf.ServiceException")
  }

  test("shaded Kryo") {
    assertLoads("org.apache.hive.com.esotericsoftware.kryo.Kryo")
  }

  test("hive-common") {
    assertLoads("org.apache.hadoop.hive.conf.HiveConf")
  }

  test("hive-exec") {
    assertLoads("org.apache.hadoop.hive.ql.CommandNeedRetryException")
  }

  private val STD_INSTANTIATOR = "org.objenesis.strategy.StdInstantiatorStrategy"

  test("Forbidden Dependencies") {
    assertClassNotFound("com.esotericsoftware.shaded." + STD_INSTANTIATOR)
    assertClassNotFound("org.apache.hive.com.esotericsoftware.shaded." + STD_INSTANTIATOR)
  }

  test("parquet-hadoop-bundle") {
    assertLoads("parquet.hadoop.ParquetOutputFormat")
    assertLoads("parquet.hadoop.ParquetInputFormat")
  }
} 

package walfie.gbf.raidfinder.server

import akka.agent.Agent
import com.pastebin.Pj9d8jt5.ImagePHash
import java.awt.image.BufferedImage
import java.net.URL
import javax.imageio.ImageIO
import monix.execution.Scheduler
import monix.reactive._
import monix.reactive.subjects.ConcurrentSubject
import scala.concurrent.{ExecutionContext, Future}
import walfie.gbf.raidfinder.domain._
import walfie.gbf.raidfinder.util.BlockingIO

trait BossNameTranslator {
  import BossNameTranslator.Translation

  def translate(bossName: BossName): Option[BossName]
  def update(latestBosses: Map[BossName, RaidBoss]): Future[Unit]
  def observable(): Observable[Translation]
}

object BossNameTranslator {
  case class Translation(from: BossName, to: BossName)
}


  private def croppedImageFromUrl(url: URL): BufferedImage = {
    // TODO: Use a real HTTP client to get the image
    val image = ImageIO.read(url.openStream())
    image.getSubimage(0, 0, image.getWidth(), image.getHeight() * 3 / 4);
  }

  // This assumes there are only two languages (which is true currently)
  private def findTranslation(newData: TranslationData): Option[BossName] = {
    translationDataAgent.get.values.find { existingData =>
      newData.hash == existingData.hash &&
        newData.language != existingData.language &&
        newData.level == existingData.level
    }.map(_.name)
  }
}

object ImageBasedBossNameTranslator {
  case class TranslationData(name: BossName, level: Int, language: Language, hash: ImageHash)

  case class ImageHash(value: Long) extends AnyVal
} 

// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package com.daml.jwt

import java.net.{URI, URL}
import java.security.interfaces.RSAPublicKey
import java.util.concurrent.TimeUnit

import com.auth0.jwk.UrlJwkProvider
import com.daml.jwt.JwtVerifier.Error
import com.google.common.cache.{Cache, CacheBuilder}
import scalaz.{-\/, Show, \/}
import scalaz.syntax.show._


  private[this] def getCachedVerifier(keyId: String): Error \/ JwtVerifier = {
    if (keyId == null)
      -\/(Error('getCachedVerifier, "No Key ID found"))
    else
      \/.fromTryCatchNonFatal(
        cache.get(keyId, () => getVerifier(keyId).fold(e => sys.error(e.shows), x => x))
      ).leftMap(e => Error('getCachedVerifier, e.getMessage))
  }

  def verify(jwt: domain.Jwt): Error \/ domain.DecodedJwt[String] = {
    for {
      keyId <- \/.fromTryCatchNonFatal(com.auth0.jwt.JWT.decode(jwt.value).getKeyId)
        .leftMap(e => Error('verify, e.getMessage))
      verifier <- getCachedVerifier(keyId)
      decoded <- verifier.verify(jwt)
    } yield decoded
  }
}

object JwksVerifier {
  def apply(url: String) = new JwksVerifier(new URI(url).toURL)

  final case class Error(what: Symbol, message: String)

  object Error {
    implicit val showInstance: Show[Error] =
      Show.shows(e => s"JwksVerifier.Error: ${e.what}, ${e.message}")
  }
} 

package helpers

import java.net.URL

import javax.inject.Inject
import play.api.{Configuration, Environment}

import scala.io.Source
import scala.util.Try

class ViewHelpers @Inject()
(configuration: Configuration, environment: Environment) {
  val organizationName = configuration.get[String]("app.organization.name")
  val maybeOrganizationLogoUrl = configuration.getOptional[String]("app.organization.logo-url")
  val maybeOrganizationUrl = configuration.getOptional[String]("app.organization.url")
  val maybeOrganizationClaUrl = configuration.getOptional[String]("app.organization.cla-url")

  val claText: String = {
    maybeOrganizationClaUrl
      .flatMap(claUrl => Try(new URL(claUrl)).toOption)
      .orElse(environment.resource("sample-cla.html"))
      .map { claUrl =>
        val text = Source.fromURL(claUrl)
        text.mkString
      } getOrElse {
        throw new Exception("You must set the ORG_CLA environment variable.")
      }
  }
} 

package config

import java.net.{MalformedURLException, URL}

import play.api.i18n.Langs
import play.api.{Configuration, Environment}
import uk.gov.hmrc.domain.SaUtrGenerator
import uk.gov.hmrc.play.bootstrap.config.{RunMode, ServicesConfig}
import util.BaseSpec

class ConfigDecoratorSpec extends BaseSpec {
  val saUtr = new SaUtrGenerator().nextSaUtr.utr

  "Converting urls to sso" should {

    trait LocalSetup {

      lazy val configDecorator = new ConfigDecorator(
        injected[Environment],
        injected[Configuration],
        injected[RunMode],
        injected[Langs],
        injected[ServicesConfig]) {
        override lazy val portalBaseUrl = "http://portal.service"
        override lazy val companyAuthFrontendHost = "http://company-auth-frontend.service"
      }
    }

    "return a properly encoded sso url when calling transformUrlForSso" in new LocalSetup {

      configDecorator.transformUrlForSso(new URL("http://example.com/some/path?key=val")) shouldBe
        "http://company-auth-frontend.service/ssoout/non-digital?continue=http%3A%2F%2Fexample.com%2Fsome%2Fpath%3Fkey%3Dval"
    }

    "return a properly formatted sa302 url when calling sa302Url" in new LocalSetup {

      configDecorator.sa302Url(saUtr, "1516") shouldBe
        s"/self-assessment-file/1516/ind/$saUtr/return/viewYourCalculation/reviewYourFullCalculation"
    }

    "return a properly formatted SA Account Summary Url url when calling ssoToSaAccountSummaryUrl" in new LocalSetup {

      configDecorator.ssoToSaAccountSummaryUrl(saUtr, "1516") shouldBe
        s"http://company-auth-frontend.service/ssoout/non-digital?continue=http%3A%2F%2Fportal.service%2Fself-assessment%2Find%2F$saUtr%2Ftaxreturn%2F1516%2Foptions"
    }
  }

  "Calling toPortalUrl" should {

    trait LocalSetup {

      def portalBaseUrlToTest: Option[String]

      lazy val configDecorator = new ConfigDecorator(
        injected[Environment],
        injected[Configuration],
        injected[RunMode],
        injected[Langs],
        injected[ServicesConfig]) {
        override lazy val portalBaseUrl = portalBaseUrlToTest.getOrElse("")
      }
    }

    "return a URL if portalBaseUrl is present and fully qualified" in new LocalSetup {

      val portalBaseUrlToTest = Some("http://portal.service")

      configDecorator.toPortalUrl("/some/path").toString shouldBe "http://portal.service/some/path"
    }

    "fail with a MalformedURLException if portalBaseUrl is not present" in new LocalSetup {

      val portalBaseUrlToTest = None

      a[MalformedURLException] should be thrownBy {
        configDecorator.toPortalUrl("/some/path")
      }
    }

    "fail with a MalformedURLException if portalBaseUrl is not fully qualified" in new LocalSetup {

      val portalBaseUrlToTest = Some("/")

      a[MalformedURLException] should be thrownBy {
        configDecorator.toPortalUrl("/some/path")
      }
    }

    "fail with a MalformedURLException if portalBaseUrl is protocol-relative" in new LocalSetup {

      val portalBaseUrlToTest = Some("//portal.service")

      a[MalformedURLException] should be thrownBy {
        configDecorator.toPortalUrl("/some/path")
      }
    }

  }
} 

package wvlet.log.io

import java.io._
import java.net.{ServerSocket, URL}
import java.nio.charset.StandardCharsets


object IOUtil {
  def withResource[Resource <: AutoCloseable, U](resource: Resource)(body: Resource => U): U = {
    try {
      body(resource)
    } finally {
      resource.close
    }
  }

  def withTempFile[U](name: String, suffix: String = ".tmp", dir: String = "target")(body: File => U) = {
    val d = new File(dir)
    d.mkdirs()
    val f = File.createTempFile(name, suffix, d)
    try {
      body(f)
    } finally {
      f.delete()
    }
  }

  def randomPort: Int = unusedPort
  def unusedPort: Int = {
    withResource(new ServerSocket(0)) { socket => socket.getLocalPort }
  }

  def findPath(path: String): Option[File] = findPath(new File(path))

  def findPath(path: File): Option[File] = {
    if (path.exists()) {
      Some(path)
    } else {
      val defaultPath = new File(new File(System.getProperty("prog.home", "")), path.getPath)
      if (defaultPath.exists()) {
        Some(defaultPath)
      } else {
        None
      }
    }
  }

  def readAsString(f: File): String = {
    readAsString(f.toURI.toURL)
  }

  def readAsString(url: URL): String = {
    withResource(url.openStream()) { in => readAsString(in) }
  }

  def readAsString(resourcePath: String): String = {
    require(resourcePath != null, s"resourcePath is null")
    Resource
      .find(resourcePath)
      .map(readAsString(_))
      .getOrElse {
        val file = findPath(new File(resourcePath))
        if (file.isEmpty) {
          throw new FileNotFoundException(s"Not found ${resourcePath}")
        }
        readAsString(new FileInputStream(file.get))
      }
  }

  def readAsString(in: InputStream): String = {
    readFully(in) { data => new String(data, StandardCharsets.UTF_8) }
  }

  def readFully[U](in: InputStream)(f: Array[Byte] => U): U = {
    val byteArray = withResource(new ByteArrayOutputStream) { b =>
      val buf = new Array[Byte](8192)
      withResource(in) { src =>
        var readBytes = 0
        while ({
          readBytes = src.read(buf);
          readBytes != -1
        }) {
          b.write(buf, 0, readBytes)
        }
      }
      b.toByteArray
    }
    f(byteArray)
  }
} 

package wvlet.airframe.control
import java.io.{ByteArrayOutputStream, File, InputStream}
import java.net.URL
import java.nio.charset.StandardCharsets

import wvlet.airframe.control.Control.withResource


object IO {

  def readAsString(f: File): String = {
    readAsString(f.toURI.toURL)
  }

  def readAsString(url: URL): String = {
    withResource(url.openStream()) { in => readAsString(in) }
  }

  def readAsString(in: InputStream): String = {
    new String(readFully(in), StandardCharsets.UTF_8)
  }

  def readFully(in: InputStream): Array[Byte] = {
    val byteArray =
      if (in == null) {
        Array.emptyByteArray
      } else {
        withResource(new ByteArrayOutputStream) { b =>
          val buf = new Array[Byte](8192)
          withResource(in) { src =>
            var readBytes = 0
            while ({
              readBytes = src.read(buf);
              readBytes != -1
            }) {
              b.write(buf, 0, readBytes)
            }
          }
          b.toByteArray
        }
      }
    byteArray
  }

} 

package wvlet.airframe.http.codegen
import java.io.File
import java.net.{URL, URLClassLoader}
import java.util.jar.JarFile

import wvlet.log.LogSupport


object ClassScanner extends LogSupport {

  def scanClasses(cl: ClassLoader, targetPackageNames: Seq[String]): Seq[String] = {
    def loop(c: ClassLoader): Seq[URL] = {
      c match {
        case null =>
          Seq.empty
        case u: URLClassLoader =>
          u.getURLs.toSeq ++ loop(u.getParent)
        case _ =>
          loop(c.getParent)
      }
    }

    val urls = loop(cl)
    val b    = Seq.newBuilder[String]

    def findClasses(url: URL): Seq[String] = {
      url match {
        case dir if dir.getProtocol == "file" && { val d = new File(dir.getPath); d.exists() && d.isDirectory } =>
          scanClassesInDirectory(dir.getPath, targetPackageNames)
        case jarFile if jarFile.getProtocol == "file" && jarFile.getPath.endsWith(".jar") =>
          scanClassesInJar(jarFile.getPath, targetPackageNames)
        case _ =>
          Seq.empty
      }
    }

    urls.foreach(x => b ++= findClasses(x))

    b.result().filterNot { x => x.contains("$anon$") }.distinct
  }

  private def toFilePath(packageName: String): String = {
    packageName.replaceAll("\\.", "/") + "/"
  }

  private def scanClassesInDirectory(dir: String, targetPackageNames: Seq[String]): Seq[String] = {
    val classes = Seq.newBuilder[String]

    def loop(baseDir: File, f: File): Unit = {
      f match {
        case d: File if f.isDirectory =>
          val files = d.listFiles()
          if (files != null) {
            files.foreach(loop(baseDir, _))
          }
        case f: File if f.getPath.endsWith(".class") =>
          val className = f.getPath
            .stripSuffix(".class").replaceAllLiterally(baseDir.getPath, "").replaceFirst("\\/", "")
            .replaceAll("\\/", ".")

          classes += className
        case _ =>
      }
    }

    val baseDir = new File(dir)
    if (baseDir.exists() && baseDir.isDirectory) {
      val dirs = targetPackageNames.map(toFilePath).map { path => new File(baseDir, path) }
      dirs.foreach(x => loop(baseDir, x))
    }

    classes.result()
  }

  private def scanClassesInJar(jarFile: String, targetPackageNames: Seq[String]): Seq[String] = {
    val jf: JarFile = new JarFile(jarFile)
    val entryEnum   = jf.entries

    val targetPaths = targetPackageNames.map(toFilePath)

    val classes = Seq.newBuilder[String]

    while (entryEnum.hasMoreElements) {
      val jarEntry  = entryEnum.nextElement
      val entryName = jarEntry.getName
      if (entryName.endsWith(".class") && targetPaths.exists(p => entryName.startsWith(p))) {
        val clsName = entryName
          .stripSuffix(".class")
          .replaceAll("\\/", ".")
        classes += clsName
      }
    }

    classes.result()
  }
} 

package de.leanovate.swaggercheck.schema.model.formats

import java.net.{URI, URL}
import java.time.format.DateTimeFormatter
import java.time.temporal.ChronoUnit
import java.time.{Instant, LocalDate}
import java.util.UUID

import de.leanovate.swaggercheck.schema.model.{JsonPath, ValidationResult}

import scala.util.Try

object StringFormats {

  object URLString extends ValueFormat[String] {
    override def validate(path: JsonPath, value: String): ValidationResult =
      if (Try(new URL(value)).isSuccess)
        ValidationResult.success
      else
        ValidationResult.error(s"'$value' is not an url: $path")
  }

  object URIString extends ValueFormat[String] {
    override def validate(path: JsonPath, value: String): ValidationResult =
      if (Try(new URI(value)).isSuccess)
        ValidationResult.success
      else
        ValidationResult.error(s"'$value' is not an uri: $path")
  }

  object UUIDString extends ValueFormat[String] {
    override def validate(path: JsonPath, value: String): ValidationResult =
      if (Try(UUID.fromString(value)).isSuccess)
        ValidationResult.success
      else
        ValidationResult.error(s"'$value' is not an uuid: $path")
  }

  object EmailString extends ValueFormat[String] {
    val emailPattern = """(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])""".r

    override def validate(path: JsonPath, value: String): ValidationResult =
      if (emailPattern.pattern.matcher(value).matches()) {
        ValidationResult.success
      } else {
        ValidationResult.error(s"'$value' is not an email: $path")
      }  }

  object DateString extends ValueFormat[String] {
    override def validate(path: JsonPath, value: String): ValidationResult =
      if (Try(DateTimeFormatter.ISO_DATE.parse(value)).isSuccess)
        ValidationResult.success
      else
        ValidationResult.error(s"'$value' is not a date: $path")
  }

  object DateTimeString extends ValueFormat[String] {
    override def validate(path: JsonPath, value: String): ValidationResult =
      if (Try(DateTimeFormatter.ISO_DATE_TIME.parse(value)).isSuccess)
        ValidationResult.success
      else
        ValidationResult.error(s"'$value' is not a date-time: $path")
  }

  val defaultFormats = Map(
    "url" -> URLString,
    "uri" -> URIString,
    "uuid" -> UUIDString,
    "email" -> EmailString,
    "date" -> DateString,
    "date-time" -> DateTimeString
  )
} 

package de.leanovate.swaggercheck.fixtures.model

import java.net.{URI, URL}
import java.time.temporal.ChronoUnit
import java.time.{Instant, LocalDate}
import java.util.UUID

import de.leanovate.swaggercheck.generators.Generators
import org.scalacheck.{Arbitrary, Gen}
import play.api.libs.json.{Json, OFormat}

import scala.util.Try

case class AnyThing(
                     anUUID: String,
                     anURL: String,
                     anURI: String,
                     anEmail: String,
                     aDate: LocalDate,
                     aDateTime: Instant,
                     anInt32: Int,
                     anInt64: Long,
                     aFloat: Float,
                     aDouble: Double,
                     aBoolean: Boolean,
                     anEnum: String,
                     aMap: Map[String, String]
                   ) {
  def isValid: Boolean = {
    Try {
      UUID.fromString(anUUID)
      new URL(anURL)
      new URI(anURI)
    }.isSuccess && Set("V1", "V2", "V3").contains(anEnum)
  }
}

object AnyThing {
  implicit val jsonFormat: OFormat[AnyThing] = Json.format[AnyThing]

  implicit val arbitrary = Arbitrary(for {
    anUUID <- Gen.uuid.map(_.toString)
    anURL <- Generators.url
    anURI <- Generators.uri
    anEmail <- Generators.email
    aDate <- Arbitrary.arbitrary[Int].map(diff => LocalDate.now().plus(diff, ChronoUnit.DAYS))
    aDateTime <- Arbitrary.arbitrary[Long].map(diff => Instant.now().plus(diff, ChronoUnit.NANOS))
    anInt32 <- Arbitrary.arbitrary[Int]
    anInt64 <- Arbitrary.arbitrary[Long]
    aFloat <- Arbitrary.arbitrary[Float]
    aDouble <- Arbitrary.arbitrary[Double]
    aBoolean <- Arbitrary.arbitrary[Boolean]
    anEnum <- Gen.oneOf("V1", "V2", "V3")
    aMap <- Arbitrary.arbitrary[Map[String, String]]
  } yield AnyThing(anUUID, anURL, anURI, anEmail, aDate, aDateTime, anInt32, anInt64, aFloat, aDouble, aBoolean, anEnum, aMap))
} 

package de.leanovate.swaggercheck.generators

import java.net.{URI, URL}

import de.leanovate.swaggercheck.schema.model.JsonPath
import de.leanovate.swaggercheck.schema.model.formats.StringFormats
import org.scalacheck.Prop.forAll
import org.scalacheck.Properties

import scala.util.Try

object GeneratorsSpecification extends Properties("Generators") {
  property("generate valid urls") = forAll(Generators.url) {
    url =>
      Try(new URL(url)).isSuccess
  }

  property("generate valid uris") = forAll(Generators.uri) {
    url =>
      Try(new URI(url)).isSuccess
  }

  property("generate valid emails") = forAll(Generators.email) {
    email =>
      StringFormats.EmailString.validate(JsonPath(), email).isSuccess
  }
} 

package com.codacy.analysis.cli.configuration

import java.net.URL

import better.files.File
import com.codacy.analysis.core.utils.Implicits._
import org.log4s.{Logger, getLogger}

import scala.util.{Failure, Try}

class Environment(variables: Map[String, String]) {

  private val logger: Logger = getLogger

  def codeDirectoryEnvironmentVariable(): Option[String] = {
    validate("Project directory", "environment variable", "CODACY_CODE")(variables.get("CODACY_CODE"))
  }

  def projectTokenArgument(projectTokenFromArguments: Option[String]): Option[String] = {
    validate("Project token", "argument", "--project-token")(projectTokenFromArguments)
  }

  def projectTokenEnvironmentVariable(): Option[String] = {
    validate("Project token", "environment variable", "CODACY_PROJECT_TOKEN")(variables.get("CODACY_PROJECT_TOKEN"))
  }

  def apiTokenArgument(apiTokenFromArguments: Option[String]): Option[String] = {
    validate("API token", "argument", "--api-token")(apiTokenFromArguments)
  }

  def apiTokenEnvironmentVariable(): Option[String] = {
    validate("API token", "environment variable", "CODACY_API_TOKEN")(variables.get("CODACY_API_TOKEN"))
  }

  def apiBaseUrlArgument(codacyApiBaseURLFromArguments: Option[String]): Option[String] = {
    val apiURL =
      validate("API base URL", "argument", "--codacy-api-base-url")(codacyApiBaseURLFromArguments)
    validateApiBaseUrl(apiURL)
  }

  def apiBaseUrlEnvironmentVariable(): Option[String] = {
    val apiURL =
      validate("API base URL", "environment variable", "CODACY_API_BASE_URL")(variables.get("CODACY_API_BASE_URL"))
    validateApiBaseUrl(apiURL)
  }

  def baseProjectDirectory(directory: Option[File]): File =
    directory.fold(codeDirectoryEnvironmentVariable().map(File(_)).getOrElse(File.currentWorkingDirectory))(dir =>
      if (dir.isDirectory) dir else dir.parent)

  private def validateApiBaseUrl(apiURL: Option[String]): Option[String] = {
    apiURL.flatMap { url =>
      Try(new URL(url)) match {
        case Failure(_) =>
          val error = s"Invalid API base URL: $url"

          val help = if (!url.startsWith("http")) {
            " * Maybe you forgot the http:// or https:// ?"
          }

          logger.warn(s"$error\n$help")
          Option.empty[String]

        case _ =>
          logger.info(s"Using API base URL $url")
          Option(url)
      }
    }
  }

  private def validate(name: String, paramType: String, param: String)(value: Option[String]): Option[String] = {
    value.ifEmpty(logger.info(s"$name not passed through $paramType `$param`")).flatMap {
      case t if t.trim.nonEmpty =>
        logger.info(s"$name found in $paramType `$param`")
        Option(t.trim)
      case _ =>
        logger.warn(s"$name passed through $paramType `$param` is empty")
        Option.empty[String]
    }
  }
} 

package org.apache.openwhisk.standalone

import java.net.{HttpURLConnection, URL}

import akka.http.scaladsl.model.Uri
import com.google.common.base.Stopwatch
import org.apache.openwhisk.utils.retry

import scala.concurrent.duration._

class ServerStartupCheck(uri: Uri, serverName: String) {

  def waitForServerToStart(): Unit = {
    val w = Stopwatch.createStarted()
    retry({
      println(s"Waiting for $serverName server at $uri to start since $w")
      require(getResponseCode() == 200)
    }, 30, Some(1.second))
  }

  private def getResponseCode(): Int = {
    val u = new URL(uri.toString())
    val hc = u.openConnection().asInstanceOf[HttpURLConnection]
    hc.setRequestMethod("GET")
    hc.connect()
    hc.getResponseCode
  }
} 

package net.ladstatt.tictactoe

import java.net.URL
import java.util.ResourceBundle
import javafx.application.Application
import javafx.beans.property.SimpleObjectProperty
import javafx.beans.value.{ChangeListener, ObservableValue}
import javafx.fxml.{FXMLLoader, FXML, Initializable}
import javafx.scene.{Scene, Parent}
import javafx.scene.control.{Label, Button}
import javafx.stage.Stage

import scala.util.control.NonFatal


class TicTacToeApp extends javafx.application.Application {

  private val resource: URL = getClass.getResource("TicTacToeApp.fxml")
  assert(resource != null)
  val loader = new FXMLLoader(resource)

  override def start(stage: Stage): Unit =
    try {
      stage.setTitle("TicTacToe App")
      loader.load[Parent]()
      stage.setScene(new Scene(loader.getRoot[Parent]))
      stage.show()
    } catch {
      case NonFatal(e) => e.printStackTrace()
    }

} 

package akka.stream.integration

import java.io.InputStream
import java.net.URL

import akka.stream.integration.BrokerResources.{ QueueStat, TopicStat }
import org.scalatest.BeforeAndAfterEach

import scala.xml.NodeSeq

trait BrokerResources extends BeforeAndAfterEach { _: TestSpec =>

  def enableClearQueus: Boolean

  private def callBroker(path: String): InputStream = {
    val amqHost = system.settings.config.getString("amq.host")
    val url = new URL(s"http://$amqHost:8161" + path)
    val urlConnection = url.openConnection()
    val basicAuth = "Basic " + new String(java.util.Base64.getUrlEncoder.encode("admin:admin".getBytes()))
    urlConnection.addRequestProperty("Authorization", basicAuth)
    urlConnection.getInputStream
  }

  // communicate with the broker //
  private def getQueueXmlFromBroker: NodeSeq = {
    import scala.xml.XML
    XML.load(callBroker("/admin/xml/queues.jsp"))
  }

  def getTopicXmlFromBroker: NodeSeq = {
    import scala.xml.XML
    XML.load(callBroker("/admin/xml/topics.jsp"))
  }

  def getQueueStats: List[QueueStat] = (for {
    e ← getQueueXmlFromBroker \\ "queue"
    stat ← e \ "stats"
  } yield QueueStat(
    (e \ "@name").text,
    (stat \ "@size").text.toInt,
    (stat \ "@consumerCount").text.toInt,
    (stat \ "@enqueueCount").text.toInt,
    (stat \ "@dequeueCount").text.toInt
  )).toList

  def getTopicStats: List[TopicStat] = (for {
    e ← getTopicXmlFromBroker \\ "topic"
    stat ← e \ "stats"
  } yield TopicStat(
    (e \ "@name").text,
    (stat \ "@size").text.toInt,
    (stat \ "@consumerCount").text.toInt,
    (stat \ "@enqueueCount").text.toInt,
    (stat \ "@dequeueCount").text.toInt
  )).toList

  def purgeQueues(): Unit = {
    def purgeQueue(destinationName: String): InputStream = {
      val path = s"/api/jolokia/exec/org.apache.activemq:brokerName=localhost,destinationName=$destinationName,destinationType=Queue,type=Broker/purge"
      callBroker(path)
    }
    getQueueList.foreach(purgeQueue)
  }

  def getQueueList: List[String] = (for {
    e ← getQueueXmlFromBroker \\ "queue"
  } yield (e \ "@name").text).toList

  def getQueueStatFor(topic: String): Option[QueueStat] =
    getQueueStats.find(_.name contains topic)

  def getQueueMessageCount(topic: String): Option[Int] = for {
    stat ← getQueueStatFor(topic)
  } yield stat.enqueueCount - stat.dequeueCount

  override protected def beforeEach(): Unit = {
    if (enableClearQueus)
      purgeQueues()
    super.beforeEach()
  }
}

object BrokerResources {
  case class QueueStat(name: String, size: Int, consumerCount: Int, enqueueCount: Int, dequeueCount: Int)
  case class TopicStat(name: String, size: Int, consumerCount: Int, enqueueCount: Int, dequeueCount: Int)
} 

package verizon.build

import sbt._
import java.net.URL
import depgraph._
import BlockadeOps._

object Fixtures extends Fixtures
trait Fixtures {
  def load(p: String): URL =
    getClass.getClassLoader.getResource(p)

  def commons(s: String, v: String): sbt.ModuleID =
    s"commons-$s" % s"commons-$s" % v

  val `commons-codec-1.9` = commons("codec", "1.9")
  val `commons-codec-2.9` = commons("codec", "2.9")
  val `commons-codec-1.+` = commons("codec", "1.+")
  val `commons-io-2.2` = commons("io", "2.2")
  val `commons-lang-2.2` = commons("lang", "2.2")
  val `commons-net-3.3` = commons("net", "3.3")
  val `funnel-1.3.71` = "intelmedia.ws.funnel" %% "http" % "1.3.71"
  val `funnel-1.3.+` = "intelmedia.ws.funnel" %% "http" % "1.3.+"


  val `toplevel-has-direct-dep-on-scalaz` = "org.foo" %% "has-direct-dep-on-scalaz" % "1.2.4"
  val `toplevel-has-trans-dep-on-shapeless` = "org.foo" %% "has-trans-dep-on-shapeless" % "1.2.3"
  val `toplevel-has-trans-dep-on-scalaz` = "org.foo" %% "has-trans-dep-on-scalaz" % "1.2.5"
  val `doobie-core-0.2.3` = "org.tpolecat" %% "doobie-core" % "0.2.3"
  val `scalaz-core-7.1.4` = "org.scalaz" %% "scalaz-core" % "7.1.4"
  val `scalaz-core-7.1.5` = "org.scalaz" %% "scalaz-core" % "7.1.5"
  val `scalaz-core-7.0.4` = "org.scalaz" %% "scalaz-core" % "7.0.4"
  val `scalaz-core-7.2.0` = "org.scalaz" %% "scalaz-core" % "7.2.0"
  val `scalaz-effect-7.1.4` = "org.scalaz" %% "scalaz-effect" % "7.1.4"
  val `scalaz-stream-0.8` = "org.scalaz.stream" %% "scalaz-stream" % "0.8"
  val `shapeless-2.2.5` = "com.chuusai" %% "shapeless" % "2.2.5"

  val m0HasScalazDep = Module(toModuleId(`toplevel-has-direct-dep-on-scalaz`))
  val m0HasShapelessTransDep = Module(toModuleId(`toplevel-has-trans-dep-on-shapeless`))
  val m0HasScalazTransDep = Module(toModuleId(`toplevel-has-trans-dep-on-scalaz`))
  val m1 = Module(toModuleId(`doobie-core-0.2.3`))
  val m2 = Module(toModuleId(`scalaz-core-7.1.4`))
  val m3 = Module(
    id = toModuleId(`scalaz-effect-7.1.4`),
    evictedByVersion = Some("notrelevant")
  )
  val m4 = Module(toModuleId(`scalaz-stream-0.8`))
  val m5 = Module(toModuleId(`shapeless-2.2.5`))
  val graphWithNestedShapeless = ModuleGraph(
    nodes = Seq(m5, m3, m4, m0HasShapelessTransDep, m0HasScalazTransDep, m1, m2, m0HasScalazDep),
    edges = Seq(
      m0HasShapelessTransDep.id -> m1.id,
      m0HasScalazTransDep.id -> m1.id,
      m1.id -> m2.id,
      m1.id -> m3.id,
      m1.id -> m4.id,
      m1.id -> m5.id,
      m0HasScalazDep.id -> m2.id
    )
  )
  val graphWithNestedShapelessWithoutEvicted = ModuleGraph(
    nodes = Seq(m5, m4, m0HasShapelessTransDep, m0HasScalazTransDep, m1, m2, m0HasScalazDep),
    edges = Seq(
      m0HasShapelessTransDep.id -> m1.id,
      m0HasScalazTransDep.id -> m1.id,
      m1.id -> m2.id,
      m1.id -> m4.id,
      m1.id -> m5.id,
      m0HasScalazDep.id -> m2.id
    )
  )
} 

package stormlantern.consul.client

import java.net.URL

import org.scalatest._
import org.scalatest.concurrent.{ Eventually, IntegrationPatience, ScalaFutures }
import stormlantern.consul.client.dao.akka.AkkaHttpConsulClient
import stormlantern.consul.client.dao.{ ConsulHttpClient, ServiceRegistration }
import stormlantern.consul.client.discovery.{ ConnectionProvider, ConnectionProviderFactory, ConnectionStrategy, ServiceDefinition }
import stormlantern.consul.client.loadbalancers.RoundRobinLoadBalancer
import stormlantern.consul.client.util.{ ConsulDockerContainer, Logging, TestActorSystem }

import scala.concurrent.Future

class ServiceBrokerIntegrationTest extends FlatSpec with Matchers with ScalaFutures with Eventually with IntegrationPatience with ConsulDockerContainer with TestActorSystem with Logging {

  import scala.concurrent.ExecutionContext.Implicits.global

  "The ServiceBroker" should "provide a usable connection to consul" in withConsulHost { (host, port) ⇒
    withActorSystem { implicit actorSystem ⇒
      val akkaHttpClient = new AkkaHttpConsulClient(new URL(s"http://$host:$port"))
      // Register the HTTP interface
      akkaHttpClient.putService(ServiceRegistration("consul-http", Some("consul-http-1"), address = Some(host), port = Some(port)))
      akkaHttpClient.putService(ServiceRegistration("consul-http", Some("consul-http-2"), address = Some(host), port = Some(port)))
      val connectionProviderFactory = new ConnectionProviderFactory {
        override def create(host: String, port: Int): ConnectionProvider = new ConnectionProvider {
          logger.info(s"Asked to create connection provider for $host:$port")
          val httpClient: ConsulHttpClient = new AkkaHttpConsulClient(new URL(s"http://$host:$port"))
          override def getConnection: Future[Any] = Future.successful(httpClient)
        }
      }
      val connectionStrategy = ConnectionStrategy(ServiceDefinition("consul-http"), connectionProviderFactory, new RoundRobinLoadBalancer)
      val sut = ServiceBroker(actorSystem, akkaHttpClient, Set(connectionStrategy))
      eventually {
        sut.withService("consul-http") { connection: ConsulHttpClient ⇒
          connection.getService("bogus").map(_.resource should have size 0)
        }
        sut
      }
    }
  }
} 

package stormlantern.consul.client

import java.net.URL

import scala.concurrent.duration._
import scala.concurrent._

import akka.actor._
import akka.util.Timeout
import akka.pattern.ask

import stormlantern.consul.client.dao._
import stormlantern.consul.client.dao.akka.AkkaHttpConsulClient
import stormlantern.consul.client.discovery._
import stormlantern.consul.client.election.LeaderInfo
import stormlantern.consul.client.loadbalancers.LoadBalancerActor
import stormlantern.consul.client.util._

class ServiceBroker(serviceBrokerActor: ActorRef, consulClient: ConsulHttpClient)(implicit ec: ExecutionContext) extends RetryPolicy with Logging {

  private[this] implicit val timeout = Timeout(10.seconds)

  def withService[A, B](name: String)(f: A ⇒ Future[B]): Future[B] = {
    logger.info(s"Trying to get connection for service $name")
    serviceBrokerActor.ask(ServiceBrokerActor.GetServiceConnection(name)).mapTo[ConnectionHolder].flatMap { connectionHolder ⇒
      logger.info(s"Received connectionholder $connectionHolder")
      try {
        connectionHolder.connection.flatMap(c ⇒ f(c.asInstanceOf[A]))
      } finally {
        connectionHolder.loadBalancer ! LoadBalancerActor.ReturnConnection(connectionHolder)
      }
    }
  }

  def registerService(registration: ServiceRegistration): Future[Unit] = {
    consulClient.putService(registration).map { serviceId ⇒
      // Add shutdown hook
      val deregisterService = new Runnable {
        override def run(): Unit = consulClient.deleteService(serviceId)
      }
      Runtime.getRuntime.addShutdownHook(new Thread(deregisterService))
    }
  }

  def withLeader[A](key: String)(f: Option[LeaderInfo] ⇒ Future[A]): Future[A] = {
    ???
  }

  def joinElection(key: String): Future[Unit] = {
    ???
  }
}

object ServiceBroker {

  def apply(rootActor: ActorSystem, httpClient: ConsulHttpClient, services: Set[ConnectionStrategy]): ServiceBroker = {
    implicit val ec = ExecutionContext.Implicits.global
    val serviceAvailabilityActorFactory = (factory: ActorRefFactory, service: ServiceDefinition, listener: ActorRef) ⇒
      factory.actorOf(ServiceAvailabilityActor.props(httpClient, service, listener))
    val actorRef = rootActor.actorOf(ServiceBrokerActor.props(services, serviceAvailabilityActorFactory), "ServiceBroker")
    new ServiceBroker(actorRef, httpClient)
  }

  def apply(consulAddress: URL, services: Set[ConnectionStrategy]): ServiceBroker = {
    implicit val rootActor = ActorSystem("reactive-consul")
    val httpClient = new AkkaHttpConsulClient(consulAddress)
    ServiceBroker(rootActor, httpClient, services)
  }

}

case class ServiceUnavailableException(service: String) extends RuntimeException(s"$service service unavailable") 

package stormlantern.consul.example

import java.net.URL

import akka.actor.ActorSystem
import akka.io.IO
import akka.pattern._
import akka.util.Timeout
import spray.can.Http
import spray.json.{ JsString, JsObject }
import stormlantern.consul.client.discovery.{ ConnectionStrategy, ServiceDefinition, ConnectionProvider }
import stormlantern.consul.client.loadbalancers.RoundRobinLoadBalancer
import stormlantern.consul.client.ServiceBroker
import stormlantern.consul.client.DNS

import scala.concurrent.Future
import scala.concurrent.duration._

object Boot extends App {
  implicit val system = ActorSystem("reactive-consul")
  implicit val executionContext = system.dispatcher

  val service = system.actorOf(ReactiveConsulHttpServiceActor.props(), "webservice")

  implicit val timeout = Timeout(5.seconds)

  IO(Http) ? Http.Bind(service, interface = "0.0.0.0", port = 8080)

  def connectionProviderFactory = (host: String, port: Int) ⇒ new ConnectionProvider {
    val client = new SprayExampleServiceClient(new URL(s"http://$host:$port"))
    override def getConnection: Future[Any] = Future.successful(client)
  }
  val connectionStrategy1 = ConnectionStrategy("example-service-1", connectionProviderFactory)
  val connectionStrategy2 = ConnectionStrategy("example-service-2", connectionProviderFactory)

  val services = Set(connectionStrategy1, connectionStrategy2)
  val serviceBroker = ServiceBroker(DNS.lookup("consul-8500.service.consul"), services)

  system.scheduler.schedule(5.seconds, 5.seconds) {
    serviceBroker.withService("example-service-1") { client: SprayExampleServiceClient ⇒
      client.identify
    }.foreach(println)
    serviceBroker.withService("example-service-2") { client: SprayExampleServiceClient ⇒
      client.identify
    }.foreach(println)
  }
} 

package org.apache.spark.sql.hive

import java.net.URL

import org.apache.spark.SparkFunSuite


class ClasspathDependenciesSuite extends SparkFunSuite {
  private val classloader = this.getClass.getClassLoader

  private def assertLoads(classname: String): Unit = {
    val resourceURL: URL = Option(findResource(classname)).getOrElse {
      fail(s"Class $classname not found as ${resourceName(classname)}")
    }

    logInfo(s"Class $classname at $resourceURL")
    classloader.loadClass(classname)
  }

  private def findResource(classname: String): URL = {
    val resource = resourceName(classname)
    classloader.getResource(resource)
  }

  private def resourceName(classname: String): String = {
    classname.replace(".", "/") + ".class"
  }

  private def assertClassNotFound(classname: String): Unit = {
    Option(findResource(classname)).foreach { resourceURL =>
      fail(s"Class $classname found at $resourceURL")
    }

    intercept[ClassNotFoundException] {
      classloader.loadClass(classname)
    }
  }

  test("shaded Protobuf") {
    assertLoads("org.apache.hive.com.google.protobuf.ServiceException")
  }

  test("shaded Kryo") {
    assertLoads("org.apache.hive.com.esotericsoftware.kryo.Kryo")
  }

  test("hive-common") {
    assertLoads("org.apache.hadoop.hive.conf.HiveConf")
  }

  test("hive-exec") {
    assertLoads("org.apache.hadoop.hive.ql.CommandNeedRetryException")
  }

  private val STD_INSTANTIATOR = "org.objenesis.strategy.StdInstantiatorStrategy"

  test("Forbidden Dependencies") {
    assertClassNotFound("com.esotericsoftware.shaded." + STD_INSTANTIATOR)
    assertClassNotFound("org.apache.hive.com.esotericsoftware.shaded." + STD_INSTANTIATOR)
  }

  test("parquet-hadoop-bundle") {
    assertLoads("parquet.hadoop.ParquetOutputFormat")
    assertLoads("parquet.hadoop.ParquetInputFormat")
  }
} 

package org.apache.spark.sql.hive

import java.net.URL

import org.apache.hadoop.hive.conf.HiveConf.ConfVars

import org.apache.spark.SparkConf
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SQLTestUtils}
import org.apache.spark.util.{ChildFirstURLClassLoader, MutableURLClassLoader}

class HiveUtilsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {

  test("newTemporaryConfiguration overwrites listener configurations") {
    Seq(true, false).foreach { useInMemoryDerby =>
      val conf = HiveUtils.newTemporaryConfiguration(useInMemoryDerby)
      assert(conf(ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname) === "")
      assert(conf(ConfVars.METASTORE_EVENT_LISTENERS.varname) === "")
      assert(conf(ConfVars.METASTORE_END_FUNCTION_LISTENERS.varname) === "")
    }
  }

  test("newTemporaryConfiguration respect spark.hadoop.foo=bar in SparkConf") {
    sys.props.put("spark.hadoop.foo", "bar")
    Seq(true, false) foreach { useInMemoryDerby =>
      val hiveConf = HiveUtils.newTemporaryConfiguration(useInMemoryDerby)
      assert(!hiveConf.contains("spark.hadoop.foo"))
      assert(hiveConf("foo") === "bar")
    }
  }

  test("ChildFirstURLClassLoader's parent is null, get spark classloader instead") {
    val conf = new SparkConf
    val contextClassLoader = Thread.currentThread().getContextClassLoader
    val loader = new ChildFirstURLClassLoader(Array(), contextClassLoader)
    try {
      Thread.currentThread().setContextClassLoader(loader)
      HiveUtils.newClientForMetadata(
        conf,
        SparkHadoopUtil.newConfiguration(conf),
        HiveUtils.newTemporaryConfiguration(useInMemoryDerby = true))
    } finally {
      Thread.currentThread().setContextClassLoader(contextClassLoader)
    }
  }

  test("toHiveString correctly handles UDTs") {
    val point = new ExamplePoint(50.0, 50.0)
    val tpe = new ExamplePointUDT()
    assert(HiveUtils.toHiveString((point, tpe)) === "(50.0, 50.0)")
  }
} 

package sbtwhitesource

import java.net.URL

import sbt.Artifact

import org.scalatest.WordSpec
import org.scalatest.Matchers

class BaseActionSpec extends WordSpec with Matchers {
  "The base action" should {
    "merge standard and native jars of the same artifacts" in {
      val nativeUrl = new URL("https://repo1.maven.org/maven2/com/github/jnr/jffi/1.2.16/jffi-1.2.16-native.jar")
      val nativeArtifact: Artifact = Artifact("jffi", "jar", "jar", Some("native"), Vector(), Some(nativeUrl))
      val native = ModuleInfo("com.github", "jffi", "1.2.16", Some((nativeArtifact, null)))
      
      val javaUrl = new URL("https://repo1.maven.org/maven2/com/github/jnr/jffi/1.2.16/jffi-1.2.16.jar")
      val javaArtifact: Artifact = Artifact("jffi", "jar", "jar", None, Vector(), Some(javaUrl))
      val java = ModuleInfo("com.github", "jffi", "1.2.16", Some((javaArtifact, null)))
      
      BaseAction.mergeModuleInfo(native, java) should be(Some(java))
      BaseAction.mergeModuleInfo(java, native) should be(Some(java))
    }

    "merge platform-specific artifacts with matching platform-independent artifacts" in {
      val nativeUrl = new URL("https://repo1.maven.org/maven2/io/netty/netty-transport-native-epoll/4.1.42.Final/netty-transport-native-epoll-4.1.42.Final-linux-x86_64.jar")
      val nativeArtifact: Artifact = Artifact("netty-transport-native-epoll", "jar", "jar", Some("linux-x86_64"), Vector(), Some(nativeUrl))
      val native = ModuleInfo("io.netty", "netty-transport-native-epoll", "4.1.42.Final", Some((nativeArtifact, null)))

      val javaUrl = new URL("https://repo1.maven.org/maven2/io/netty/netty-transport-native-epoll/4.1.42.Final/netty-transport-native-epoll-4.1.42.Final.jar")
      val javaArtifact: Artifact = Artifact("netty-transport-native-epoll", "jar", "jar", None, Vector(), Some(javaUrl))
      val java = ModuleInfo("io.netty", "netty-transport-native-epoll", "4.1.42.Final", Some((javaArtifact, null)))

      BaseAction.mergeModuleInfo(native, java) should be(Some(java))
      BaseAction.mergeModuleInfo(java, native) should be(Some(java))
    }

    "upgrade 'jar' to 'bundle' when both types are present" in {
      val url = new URL("https://repo1.maven.org/maven2/com/example/osgi/fake-osgi-bundle/1.0.0/fake-osgi-bundle-1.0.0.jar")

      val bundleArtifact: Artifact = Artifact("fake-osgi-bundle", "bundle", "jar", None, Vector(), Some(url))
      val bundle = ModuleInfo("com.example.osgi", "fake-osgi-bundle", "1.0.0", Some((bundleArtifact, null)))

      val jarArtifact: Artifact = Artifact("fake-osgi-bundle", "jar", "jar", None, Vector(), Some(url))
      val jar = ModuleInfo("com.example.osgi", "fake-osgi-bundle", "1.0.0", Some((jarArtifact, null)))

      BaseAction.mergeModuleInfo(bundle, jar) should be(Some(bundle))
      BaseAction.mergeModuleInfo(jar, bundle) should be(Some(bundle))
    }
  }
} 

package controllers

import java.net.URL
import javax.inject.Inject

import context.MyExecutionContext
import models.ReadmeForm
import play.api.Logger
import play.api.data.Form
import play.api.data.Forms._
import play.api.mvc._
import readme.TableOfContentHelper
import util.HtmlUtil

import scala.concurrent.Future

class TableOfContentController @Inject()(ec: MyExecutionContext, cc: ControllerComponents)
    extends AbstractController(cc) {

  val logger = Logger(this.getClass)

  implicit val xc: MyExecutionContext = ec

  val userForm = Form(
    mapping(
      "description" -> text,
      "github_url"  -> text
    )(ReadmeForm.apply)(ReadmeForm.unapply)
  )

  val startContent: String =
    """Example :
    |# Title 1
    |## Title 2
    |### Title 3""".stripMargin('|')

  def readme = Action {
    Ok(HtmlUtil.prettify(views.html.readme(startContent)))
  }

  
  private def getGithubReadmeUrl(url: String): String = {
    val githubUrl    = new URL(url)
    val path         = githubUrl.getPath.substring(1)
    val endIndex     = path.indexOf("/", path.indexOf("/") + 1)
    val userNproject = if (endIndex == -1) path else path.substring(0, endIndex)
    s"https://raw.githubusercontent.com/${userNproject}/master/README.md"
  }

  private def readContentFromUrl(mdUrl: String): Future[String] = Future {
    val f = scala.io.Source.fromURL(mdUrl)
    try f.mkString
    finally f.close()
  }

} 

package org.apache.iota.fey

import java.io.FileOutputStream
import java.net.URL
import java.io.File

import com.eclipsesource.schema._
import org.slf4j.LoggerFactory
import play.api.libs.json._
import JSON_PATH._
import java.nio.file.{Files, Paths}

import org.apache.commons.io.IOUtils
import org.apache.commons.codec.binary.Base64
import scala.util.Properties._


  def exceptionOnRun(e: Exception): Unit
}

object HttpBasicAuth {
  val BASIC = "Basic"
  val AUTHORIZATION = "Authorization"

  def encodeCredentials(username: String, password: String): String = {
    new String(Base64.encodeBase64((username + ":" + password).getBytes))
  }

  def getHeader(username: String, password: String): String =
    BASIC + " " + encodeCredentials(username, password)
} 

package org.apache.iota.fey

import java.io.{File, FileOutputStream}
import java.net.URL
import java.nio.file.{Files, Paths}
import com.eclipsesource.schema._
import akka.actor.ActorRef
import com.eclipsesource.schema.SchemaValidator
import org.apache.commons.io.IOUtils
import play.api.libs.json._
import scala.concurrent.duration._
import scala.util.Properties._

abstract class FeyGenericActorReceiver(override val params: Map[String,String] = Map.empty,
                                       override val backoff: FiniteDuration = 1.minutes,
                                       override val connectTo: Map[String,ActorRef] = Map.empty,
                                       override val schedulerTimeInterval: FiniteDuration = 2.seconds,
                                       override val orchestrationName: String = "",
                                       override val orchestrationID: String = "",
                                       override val autoScale: Boolean = false) extends FeyGenericActor{

  private[fey] val feyCore = FEY_CORE_ACTOR.actorRef

  override final def processMessage[T](message: T, sender: ActorRef): Unit = {
    try {
      val jsonString = getJSONString(message)
      if(jsonString != "{}") {
        processJson(jsonString)
      }
      startBackoff()
    }catch{
      case e: Exception => log.error(e, s"Could not process message $message")
    }
  }

  private[fey] def processJson(jsonString: String) = {
    var orchID:String = "None"
    try{
      val orchestrationJSON = Json.parse(jsonString)
      orchID = (orchestrationJSON \ JSON_PATH.GUID).as[String]
      val valid = validJson(orchestrationJSON)
      if(valid && (orchestrationJSON \ JSON_PATH.COMMAND).as[String].toUpperCase != "DELETE"){
        checkForLocation(orchestrationJSON)
      }
      if(valid) {
        feyCore ! FeyCore.ORCHESTRATION_RECEIVED(orchestrationJSON, None)
      }else{
        log.warning(s"Could not forward Orchestration $orchID. Invalid JSON schema")
      }
    } catch {
      case e: Exception =>
        log.error(e, s"Orchestration $orchID could not be forwarded")
    }
  }

  
  def resolveCredentials(credentials: Option[JsObject]):Option[(String, String)] = {
    credentials match {
      case None => None
      case Some(cred) =>
        val user = (cred \ JSON_PATH.JAR_CRED_USER).as[String]
        val password = (cred \ JSON_PATH.JAR_CRED_PASSWORD).as[String]
        Option(envOrElse(user,user), envOrElse(password,password))
    }
  }

} 

package com.webtrends.harness.http

import java.net.{HttpURLConnection, URL}
import java.util.concurrent.TimeUnit
import akka.actor.{Props, ActorSystem}
import akka.testkit.TestKit
import akka.util.Timeout
import com.webtrends.harness.TestKitSpecificationWithJUnit
import com.webtrends.harness.service.messages.CheckHealth
import scala.concurrent.Await
import akka.pattern.ask
import scala.concurrent.duration.FiniteDuration

class InternalHttpSpec extends TestKitSpecificationWithJUnit(ActorSystem("test")) with InternalHttpClient {
  val port = 8123
  val path = "http://127.0.0.1:" + port + "/"
  val httpActor = system.actorOf(Props(classOf[SimpleHttpServer], port))

  // We need to make sure the httpActor has started up before trying to connect.
  implicit val timeout = Timeout(FiniteDuration(5, TimeUnit.SECONDS))
  Await.result(httpActor ? CheckHealth, timeout.duration)

  "Test handlers" should {
    "handle the get path /ping" in {
      val url = new URL(path + "ping")
      val conn = url.openConnection().asInstanceOf[HttpURLConnection]
      val resp = getResponseContent(conn)

      resp.status mustEqual "200"
      resp.content.length must be > 0
      resp.content.substring(0, 5) mustEqual "pong:"
    }
  }

  step {
    TestKit.shutdownActorSystem(system)
  }

} 

package org.jetbrains.sbtidea.download.plugin

import java.io.InputStream
import java.net.URL
import java.nio.file.{Files, Path}

import scala.xml._

case class PluginDescriptor(id: String,
                            name: String,
                            version: String,
                            sinceBuild: String,
                            untilBuild: String,
                            dependsOn: Seq[PluginDescriptor.Dependency] = Seq.empty) {
  def toXMLStr: String = {
    s"""
       |<idea-plugin>
       |  <name>$name</name>
       |  <id>$id</id>
       |  <version>$version</version>
       |  <idea-version since-build="$sinceBuild" until-build="$untilBuild"/>
       |  ${dependsOn.map(dep => s"""<depends optional="${dep.optional}">${dep.id}</depends>""").mkString("\n")}
       |</idea-plugin>
       |""".stripMargin
  }
}

object PluginDescriptor {

  private val OPTIONAL_KEY  = "(optional) "
  private val OPTIONAL_ATTR = "optional"

  final case class Dependency(id: String, optional: Boolean)

  def load(str: String): PluginDescriptor =
    load(XML.withSAXParser(createNonValidatingParser).loadString(str))

  def load(url: URL): PluginDescriptor =
    load(XML.withSAXParser(createNonValidatingParser).load(url))

  def load(path: Path): PluginDescriptor =
    load(XML.withSAXParser(createNonValidatingParser).load(Files.newInputStream(path)))

  def load(stream: InputStream): PluginDescriptor =
    load(XML.withSAXParser(createNonValidatingParser).load(stream))

  //noinspection ExistsEquals : scala 2.10
  def load(xml: Elem): PluginDescriptor = {
    val id      = (xml \\ "id").text
    val version = (xml \\ "version").text
    val name    = (xml \\ "name").text
    val since   = (xml \\ "idea-version").headOption.map(_.attributes("since-build").text).getOrElse("")
    val until   = (xml \\ "idea-version").headOption.map(_.attributes("until-build").text).getOrElse("")
    val dependencies = (xml \\ "depends").map { node =>
      val id        = node.text.replace(OPTIONAL_KEY, "")
      val optional  = node.text.contains(OPTIONAL_KEY) || node.attributes.asAttrMap.get(OPTIONAL_ATTR).exists(_ == "true")
      Dependency(id, optional)
    }
    val idOrName = if (id.isEmpty) name else id
    PluginDescriptor(idOrName, name, version, since, until, dependencies)
  }

  private def createNonValidatingParser = {
    val factory = javax.xml.parsers.SAXParserFactory.newInstance()
    factory.setValidating(false)
    factory.setFeature("http://xml.org/sax/features/validation", false)
    factory.setFeature("http://apache.org/xml/features/nonvalidating/load-dtd-grammar", false)
    factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false)
    factory.setFeature("http://xml.org/sax/features/external-general-entities", false)
    factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false)
    factory.newSAXParser()
  }
} 

package org.jetbrains.sbtidea.download.idea

import java.net.URL
import java.nio.file.{Files, Path}

import org.jetbrains.sbtidea.download.FileDownloader
import org.jetbrains.sbtidea.{PluginLogger, pathToPathExt}
import sbt._
import org.jetbrains.sbtidea.download.api._

import scala.language.postfixOps

abstract class IdeaSources extends IdeaArtifact {
  override type R = IdeaSources
  override protected def usedInstaller: Installer[IdeaSources] = new Installer[IdeaSources] {
    override def isInstalled(art: IdeaSources)(implicit ctx: InstallContext): Boolean =
      ctx.baseDirectory / "sources.zip" exists
    override def downloadAndInstall(art: IdeaSources)(implicit ctx: InstallContext): Unit = {
      val file = FileDownloader(ctx.baseDirectory.getParent).download(art.dlUrl, optional = true)
      Files.move(file, ctx.baseDirectory.resolve("sources.zip"))
      PluginLogger.info(s"${caller.buildInfo.edition.name} sources installed")
    }
  }
}

class IdeaSourcesImpl(override val caller: AbstractIdeaDependency, dlUrlProvider: () => URL) extends IdeaSources {
  override def dlUrl: URL = dlUrlProvider()
}

object IdeaSourcesImpl {
  def apply(caller: AbstractIdeaDependency, dlUrlProvider: () => URL): IdeaSourcesImpl = new IdeaSourcesImpl(caller, dlUrlProvider)
} 

package org.jetbrains.sbtidea.download.idea

import java.net.URL

import org.jetbrains.sbtidea.download.api._

abstract class IdeaDist extends IdeaArtifact {
  override type R = IdeaDist
  override protected def usedInstaller: Installer[IdeaDist] = new IdeaDistInstaller(caller.buildInfo)
}

class IdeaDistImpl(override val caller: AbstractIdeaDependency, dlUrlProvider: () => URL) extends IdeaDist {
  override def dlUrl: URL = dlUrlProvider()
}

object IdeaDistImpl {
  def apply(caller: AbstractIdeaDependency, dlUrlProvider: () => URL): IdeaDistImpl = new IdeaDistImpl(caller, dlUrlProvider)
} 

package org.jetbrains.sbtidea

import java.net.{HttpURLConnection, URL}
import java.nio.file.{Files, Path}

import com.eclipsesource.json.Json
import org.jetbrains.sbtidea.Keys._
import org.jetbrains.sbtidea.pathToPathExt
import sbt._

package object download {

  case class BuildInfo(buildNumber: String, edition: IntelliJPlatform, jbrVersion: Option[String]) {
    override def toString: String = s"BuildInfo($edition-$buildNumber)"
  }

  def withConnection[V](url: URL)(f: => HttpURLConnection => V): V = {
    var connection: HttpURLConnection = null
    try {
      connection = url.openConnection().asInstanceOf[HttpURLConnection]
      f(connection)
    } finally {
      try {
        if (connection != null) connection.disconnect()
      } catch {
        case e: Exception =>
          println(s"Failed to close connection $url: ${e.getMessage}")
      }
    }
  }

  implicit class BuildInfoOps(val buildInfo: BuildInfo) {
    def getActualIdeaBuild(ideaRoot: Path): String = {
      val productInfo = ideaRoot / "product-info.json"
      if (buildInfo.buildNumber.count(_ == '.') < 2 && productInfo.exists) { // most likely some LATEST-EAP-SNAPSHOT kind of version
        try {
          val content = new String(Files.readAllBytes(productInfo))
          val parsed = Json.parse(content)
          parsed.asObject().getString("buildNumber", buildInfo.buildNumber)
        } catch {
          case _: Throwable => buildInfo.buildNumber
        }
      } else buildInfo.buildNumber
    }
  }

} 

package org.jetbrains.sbtidea.download.idea

import java.net.{URI, URL}
import java.nio.file.{Files, Path, Paths}
import java.util.zip.{ZipEntry, ZipInputStream}

import org.jetbrains.sbtidea.download.BuildInfo
import org.jetbrains.sbtidea.packaging.artifact
import org.jetbrains.sbtidea.{Keys, TmpDirUtils}
import org.jetbrains.sbtidea.Keys._
import org.jetbrains.sbtidea.download.jbr.JbrDependency

trait IdeaMock extends TmpDirUtils {
  protected val IDEA_VERSION      = "192.5728.12"
  protected val IDEA_EDITION      = "IU"
  protected val IDEA_DIST         = s"idea$IDEA_EDITION-$IDEA_VERSION.zip"
  protected val IDEA_DIST_PATH    = s"/org/jetbrains/sbtidea/download/$IDEA_DIST"
  protected val IDEA_BUILDINFO: BuildInfo =
    BuildInfo(IDEA_VERSION, Keys.IntelliJPlatform.IdeaUltimate, Some(JbrDependency.VERSION_AUTO))
  protected val IDEA_DEP: IdeaDependency  = IdeaDependency(IDEA_BUILDINFO)
  protected val IDEA_ART: IdeaDist        = IdeaDistImpl(IDEA_DEP, new URL("file:"))

  protected val bundledPlugins: List[Keys.IntellijPlugin] =
    "org.jetbrains.plugins.yaml".toPlugin ::
    "com.intellij.properties".toPlugin :: Nil

  protected def installIdeaMock: Path = {
    val tmpDir      = newTmpDir
    val installDir  = Files.createDirectory(tmpDir.resolve(IDEA_VERSION))
    val stream      = getClass.getResourceAsStream(IDEA_DIST_PATH)
    artifact.using(new ZipInputStream(stream)) { zip =>
      var entry: ZipEntry = zip.getNextEntry
      while (entry != null) {
        val toPath = installDir.resolve(entry.getName)
        if (entry.isDirectory)
          Files.createDirectory(toPath)
        else
          Files.copy(zip, toPath)
        entry = zip.getNextEntry
      }
    }
    installDir
  }

  protected def getDistCopy: Path = Files.copy(getIdeaDistMockPath, newTmpDir.resolve(IDEA_DIST))

  protected def getIdeaDistMockURI: URI = getClass.getResource(IDEA_DIST_PATH).toURI

  protected def getIdeaDistMockPath: Path = Paths.get(getIdeaDistMockURI)
} 

package com.datamountaineer.streamreactor.connect.utils

import java.io.File
import java.net.URL
import java.util.jar.JarFile

import scala.collection.mutable

case class JarManifest(location: URL) {

  val map = mutable.Map.empty[String, String]

  var msg = "unknown"
  try {
    val file = new File(location.toURI)
    if (file.isFile) {
      val jarFile = new JarFile(file)
      val manifest = jarFile.getManifest
      val attributes = manifest.getMainAttributes
      map += "StreamReactor-Version" -> attributes.getValue("StreamReactor-Version")
      map += "Kafka-Version" -> attributes.getValue("Kafka-Version")
      map += "Git-Repo" -> attributes.getValue("Git-Repo")
      map += "Git-Commit-Hash" -> attributes.getValue("Git-Commit-Hash")
      map += "Git-Tag" -> attributes.getValue("Git-Tag")
      map += "StreamReactor-Docs" -> attributes.getValue("StreamReactor-Docs")
    }
  }
  catch {
    case t: Throwable => msg = t.getMessage
  }


  def version(): String = map.getOrElse("StreamReactor-Version", "")

  def printManifest(): String = {
    val msg = "unknown"

    s"""
       |StreamReactor-Version:       ${map.getOrElse("StreamReactor-Version", msg)}
       |Kafka-Version:               ${map.getOrElse("Kafka-Version", msg)}
       |Git-Repo:                    ${map.getOrElse("Git-Repo", msg)}
       |Git-Commit-Hash:             ${map.getOrElse("Git-Commit-Hash", msg)}
       |Git-Tag:                     ${map.getOrElse("Git-Tag", msg)}
       |StreamReactor-Docs:          ${map.getOrElse("StreamReactor-Docs", msg)}
      """.
      stripMargin
  }
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object numbers_validation_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿paths⌿/⌿get⌿double_optional") → 
		Opt(Dbl(TypeMeta(Some("double"), List("max(10.toDouble, true)", "min(0.toDouble, true)", "multipleOf(5.toDouble)"))), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿integer_required") → 
		Intgr(TypeMeta(Some("int32"), List("max(10.toInt, false)", "min(0.toInt, false)", "multipleOf(5.toInt)"))),
	Reference("⌿paths⌿/⌿get⌿integer_optional") → 
		Opt(Intgr(TypeMeta(Some("int32"), List("max(10.toInt, true)", "min(-10.toInt, true)", "multipleOf(5.toInt)"))), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿double_required") → 
		Dbl(TypeMeta(Some("double"), List("max(10.toDouble, false)", "min(2.toDouble, false)", "multipleOf(5.toDouble)"))),
	Reference("⌿paths⌿/⌿get⌿long_optional") → 
		Opt(Lng(TypeMeta(Some("int64"), List("max(10.toLong, true)", "min(10.toLong, true)", "multipleOf(10.toLong)"))), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿float_required") → 
		Flt(TypeMeta(Some("float"), List("max(10.toFloat, true)", "min(10.toFloat, true)", "multipleOf(5.toFloat)"))),
	Reference("⌿paths⌿/⌿get⌿float_optional") → 
		Opt(Flt(TypeMeta(Some("float"), List("max(10.toFloat, false)", "min(1.toFloat, false)", "multipleOf(5.toFloat)"))), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿long_required") → 
		Lng(TypeMeta(Some("int64"), List("max(10.toLong, true)", "min(0.toLong, true)", "multipleOf(5.toLong)"))),
	Reference("⌿paths⌿/⌿get⌿responses⌿200") → 
		Null(TypeMeta(None, List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿float_required")) → Parameter("float_required", Flt(TypeMeta(Some("float"), List("max(10.toFloat, true)", "min(10.toFloat, true)", "multipleOf(5.toFloat)"))), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿double_required")) → Parameter("double_required", Dbl(TypeMeta(Some("double"), List("max(10.toDouble, false)", "min(2.toDouble, false)", "multipleOf(5.toDouble)"))), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿integer_optional")) → Parameter("integer_optional", TypeRef(Reference("⌿paths⌿/⌿get⌿integer_optional")), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿long_required")) → Parameter("long_required", Lng(TypeMeta(Some("int64"), List("max(10.toLong, true)", "min(0.toLong, true)", "multipleOf(5.toLong)"))), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿integer_required")) → Parameter("integer_required", Intgr(TypeMeta(Some("int32"), List("max(10.toInt, false)", "min(0.toInt, false)", "multipleOf(5.toInt)"))), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿float_optional")) → Parameter("float_optional", TypeRef(Reference("⌿paths⌿/⌿get⌿float_optional")), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿double_optional")) → Parameter("double_optional", TypeRef(Reference("⌿paths⌿/⌿get⌿double_optional")), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿long_optional")) → Parameter("long_optional", TypeRef(Reference("⌿paths⌿/⌿get⌿long_optional")), None, None, ".+", encode = true, ParameterPlace.withName("query"))
) 
 def basePath: String =null
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq(
	ApiCall(GET, Path(Reference("⌿")), 
		HandlerCall(
			"numbers_validation.yaml",
			"Numbers_validationYaml",
			instantiate = false,
			"get",parameters = 
			Seq(
				ParameterRef(Reference("⌿paths⌿/⌿get⌿float_required")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿double_required")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿integer_optional")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿long_required")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿integer_required")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿float_optional")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿double_optional")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿long_optional"))
				)
			), 
		Set.empty[MimeType], 
		Set(MimeType("application/json"), MimeType("application/yaml")), 
		Map.empty[String, Seq[Class[Exception]]], 
		TypesResponseInfo(
			Map[Int, ParameterRef](
			200 -> ParameterRef(Reference("⌿paths⌿/⌿get⌿responses⌿200"))
		), None), 
		StateResponseInfo(
				Map[Int, State](
					200 -> Self
			), None), 
		Set.empty[Security.Constraint]))

def packageName: Option[String] = Some("numbers_validation.yaml")

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object additional_properties_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿KeyedArrays") → 
		TypeDef(Reference("⌿definitions⌿KeyedArrays"), 
			Seq(
					Field(Reference("⌿definitions⌿KeyedArrays⌿additionalProperties"), TypeRef(Reference("⌿definitions⌿KeyedArrays⌿additionalProperties")))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿definitions⌿KeyedArrays⌿additionalProperties") → 
		CatchAll(TypeRef(Reference("⌿definitions⌿KeyedArrays⌿additionalProperties⌿CatchAll")), TypeMeta(None, List())),
	Reference("⌿definitions⌿KeyedArrays⌿additionalProperties⌿CatchAll") → 
		Arr(BInt(TypeMeta(None, List())), TypeMeta(None, List()), "csv")
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String = "/api" 
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq()

def packageName: Option[String] = None

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object basic_extension_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿ErrorModel") → 
		TypeDef(Reference("⌿definitions⌿ErrorModel"), 
			Seq(
					Field(Reference("⌿definitions⌿ErrorModel⌿message"), Str(None, TypeMeta(Some("The text of the error message"), List()))),
					Field(Reference("⌿definitions⌿ErrorModel⌿code"), BInt(TypeMeta(Some("The error code"), List("""max(BigInt("600"), false)""", """min(BigInt("100"), false)"""))))
			), TypeMeta(Some("Named types: 2"), List())),
	Reference("⌿definitions⌿ExtendedErrorModel") → 
					AllOf(Reference("⌿definitions⌿ExtendedErrorModel⌿ExtendedErrorModel"), TypeMeta(Some("Schemas: 2"), List()),  Seq(
			TypeRef(Reference("⌿definitions⌿ErrorModel")),
			TypeRef(Reference("⌿definitions⌿ExtendedErrorModel⌿AllOf1"))) , None),
	Reference("⌿definitions⌿ExtendedErrorModel⌿AllOf1") → 
		TypeDef(Reference("⌿definitions⌿ExtendedErrorModel"), 
			Seq(
					Field(Reference("⌿definitions⌿ExtendedErrorModel⌿rootCause"), Str(None, TypeMeta(None, List())))
			), TypeMeta(Some("Named types: 1"), List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String =null
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq()

def packageName: Option[String] = None

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object nested_objects_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿NestedObjects") → 
		TypeDef(Reference("⌿definitions⌿NestedObjects"), 
			Seq(
					Field(Reference("⌿definitions⌿NestedObjects⌿plain"), TypeRef(Reference("⌿definitions⌿NestedObjects⌿plain"))),
					Field(Reference("⌿definitions⌿NestedObjects⌿nested"), TypeRef(Reference("⌿definitions⌿NestedObjects⌿nested")))
			), TypeMeta(Some("Named types: 2"), List())),
	Reference("⌿definitions⌿NestedObjects⌿nested") → 
		Opt(TypeRef(Reference("⌿definitions⌿NestedObjects⌿nested⌿Opt")), TypeMeta(None, List())),
	Reference("⌿definitions⌿NestedObjects⌿plain") → 
		Opt(TypeRef(Reference("⌿definitions⌿NestedObjects⌿plain⌿Opt")), TypeMeta(None, List())),
	Reference("⌿definitions⌿NestedObjects⌿nested⌿Opt") → 
		TypeDef(Reference("⌿definitions⌿NestedObjects⌿nested"), 
			Seq(
					Field(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2"), TypeRef(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2")))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿definitions⌿NestedObjects⌿plain⌿Opt") → 
		TypeDef(Reference("⌿definitions⌿NestedObjects⌿plain"), 
			Seq(
					Field(Reference("⌿definitions⌿NestedObjects⌿plain⌿simple"), Str(None, TypeMeta(None, List("""pattern("""+"""""""""+"""the pattern"""+"""""""""+""".r)"""))))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2") → 
		TypeDef(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2"), 
			Seq(
					Field(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3"), TypeRef(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3")))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3") → 
		Opt(TypeRef(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3⌿Opt")), TypeMeta(None, List())),
	Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3⌿bottom") → 
		Opt(Str(None, TypeMeta(None, List("maxLength(30)", "minLength(3)"))), TypeMeta(None, List())),
	Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3⌿Opt") → 
		TypeDef(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3"), 
			Seq(
					Field(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3⌿bottom"), TypeRef(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3⌿bottom")))
			), TypeMeta(Some("Named types: 1"), List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String = "/api" 
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq()

def packageName: Option[String] = None

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object nested_objects_validation_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿NestedObjects") → 
		TypeDef(Reference("⌿definitions⌿NestedObjects"), 
			Seq(
					Field(Reference("⌿definitions⌿NestedObjects⌿plain"), TypeRef(Reference("⌿definitions⌿NestedObjects⌿plain"))),
					Field(Reference("⌿definitions⌿NestedObjects⌿nested"), TypeRef(Reference("⌿definitions⌿NestedObjects⌿nested")))
			), TypeMeta(Some("Named types: 2"), List())),
	Reference("⌿definitions⌿NestedObjects⌿nested") → 
		Opt(TypeRef(Reference("⌿definitions⌿NestedObjects⌿nested⌿Opt")), TypeMeta(None, List())),
	Reference("⌿definitions⌿NestedObjects⌿plain") → 
		Opt(TypeRef(Reference("⌿definitions⌿NestedObjects⌿plain⌿Opt")), TypeMeta(None, List())),
	Reference("⌿definitions⌿NestedObjects⌿nested⌿Opt") → 
		TypeDef(Reference("⌿definitions⌿NestedObjects⌿nested"), 
			Seq(
					Field(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2"), TypeRef(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2")))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿definitions⌿NestedObjects⌿plain⌿Opt") → 
		TypeDef(Reference("⌿definitions⌿NestedObjects⌿plain"), 
			Seq(
					Field(Reference("⌿definitions⌿NestedObjects⌿plain⌿simple"), Str(None, TypeMeta(None, List("""pattern("""+"""""""""+"""the pattern"""+"""""""""+""".r)"""))))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2") → 
		TypeDef(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2"), 
			Seq(
					Field(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3"), TypeRef(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3")))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3") → 
		Opt(TypeRef(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3⌿Opt")), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿responses⌿200") → 
		Null(TypeMeta(None, List())),
	Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3⌿bottom") → 
		Opt(Str(None, TypeMeta(None, List("maxLength(30)", "minLength(3)"))), TypeMeta(None, List())),
	Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3⌿Opt") → 
		TypeDef(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3"), 
			Seq(
					Field(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3⌿bottom"), TypeRef(Reference("⌿definitions⌿NestedObjects⌿nested⌿nested2⌿nested3⌿bottom")))
			), TypeMeta(Some("Named types: 1"), List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿nestedObject")) → Parameter("nestedObject", TypeRef(Reference("⌿definitions⌿NestedObjects")), None, None, ".+", encode = false, ParameterPlace.withName("body"))
) 
 def basePath: String = "/api" 
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq(
	ApiCall(GET, Path(Reference("⌿")), 
		HandlerCall(
			"nested_objects_validation.yaml",
			"Nested_objects_validationYaml",
			instantiate = false,
			"get",parameters = 
			Seq(
				ParameterRef(Reference("⌿paths⌿/⌿get⌿nestedObject"))
				)
			), 
		Set(MimeType("application/json")), 
		Set(MimeType("application/json")), 
		Map.empty[String, Seq[Class[Exception]]], 
		TypesResponseInfo(
			Map[Int, ParameterRef](
			200 -> ParameterRef(Reference("⌿paths⌿/⌿get⌿responses⌿200"))
		), None), 
		StateResponseInfo(
				Map[Int, State](
					200 -> Self
			), None), 
		Set.empty[Security.Constraint]))

def packageName: Option[String] = Some("nested_objects_validation.yaml")

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object nested_options_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿definitions⌿Basic") → 
		TypeDef(Reference("⌿definitions⌿Basic"), 
			Seq(
					Field(Reference("⌿definitions⌿Basic⌿optional"), TypeRef(Reference("⌿definitions⌿Basic⌿optional")))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿definitions⌿Basic⌿optional") → 
		Opt(TypeRef(Reference("⌿definitions⌿Basic⌿optional⌿Opt")), TypeMeta(None, List())),
	Reference("⌿definitions⌿Basic⌿optional⌿Opt") → 
		TypeDef(Reference("⌿definitions⌿Basic⌿optional"), 
			Seq(
					Field(Reference("⌿definitions⌿Basic⌿optional⌿nested_optional"), TypeRef(Reference("⌿definitions⌿Basic⌿optional⌿nested_optional")))
			), TypeMeta(Some("Named types: 1"), List())),
	Reference("⌿definitions⌿Basic⌿optional⌿nested_optional") → 
		Opt(Str(None, TypeMeta(None, List())), TypeMeta(None, List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](

) 
 def basePath: String = "/api" 
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq()

def packageName: Option[String] = None

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package de.zalando.model
import de.zalando.apifirst.Application._
import de.zalando.apifirst.Domain._
import de.zalando.apifirst.ParameterPlace
import de.zalando.apifirst.naming._
import de.zalando.apifirst.Hypermedia._
import de.zalando.apifirst.Http._
import de.zalando.apifirst.Security
import java.net.URL
import Security._ 
//noinspection ScalaStyle
object string_formats_yaml extends WithModel {
 
 def types = Map[Reference, Type](
	Reference("⌿paths⌿/⌿get⌿base64") → 
		Opt(Base64String(TypeMeta(Some("byte"), List())), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿petId") → 
		BinaryString(TypeMeta(Some("binary"), List())),
	Reference("⌿paths⌿/⌿get⌿date_time") → 
		Opt(DateTime(TypeMeta(Some("date-time"), List())), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿uuid") → 
		Opt(UUID(TypeMeta(Some("UUID"), List())), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿date") → 
		Opt(Date(TypeMeta(Some("date"), List())), TypeMeta(None, List())),
	Reference("⌿paths⌿/⌿get⌿responses⌿200") → 
		Null(TypeMeta(None, List()))
) 
 
 def parameters = Map[ParameterRef, Parameter](
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿date_time")) → Parameter("date_time", TypeRef(Reference("⌿paths⌿/⌿get⌿date_time")), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿date")) → Parameter("date", TypeRef(Reference("⌿paths⌿/⌿get⌿date")), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿base64")) → Parameter("base64", TypeRef(Reference("⌿paths⌿/⌿get⌿base64")), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿uuid")) → Parameter("uuid", TypeRef(Reference("⌿paths⌿/⌿get⌿uuid")), None, None, ".+", encode = true, ParameterPlace.withName("query")),
	ParameterRef(	Reference("⌿paths⌿/⌿get⌿petId")) → Parameter("petId", BinaryString(TypeMeta(Some("binary"), List())), None, None, ".+", encode = false, ParameterPlace.withName("body"))
) 
 def basePath: String =null
 def discriminators: DiscriminatorLookupTable = Map[Reference, Reference](
	)
 def securityDefinitions: SecurityDefinitionsTable = Map[String, Security.Definition](
	
)
def stateTransitions: StateTransitionsTable = Map[State, Map[State, TransitionProperties]]()
def calls: Seq[ApiCall] = Seq(
	ApiCall(GET, Path(Reference("⌿")),
		HandlerCall(
			"string_formats.yaml",
			"String_formatsYaml",
			instantiate = false,
			"get",parameters = 
			Seq(
				ParameterRef(Reference("⌿paths⌿/⌿get⌿date_time")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿date")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿base64")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿uuid")),
				ParameterRef(Reference("⌿paths⌿/⌿get⌿petId"))
				)
			),
		Set.empty[MimeType],
		Set(MimeType("application/json"), MimeType("application/yaml")),
		Map.empty[String, Seq[Class[Exception]]],
		TypesResponseInfo(
			Map[Int, ParameterRef](
			200 -> ParameterRef(Reference("⌿paths⌿/⌿get⌿responses⌿200"))
		), None),
		StateResponseInfo(
				Map[Int, State](
					200 -> Self
			), None),
		Set.empty[Security.Constraint]))

def packageName: Option[String] = Some("string_formats.yaml")

def model = new StrictModel(calls, types, parameters, discriminators, basePath, packageName, stateTransitions, securityDefinitions)
    
} 

package ru.stachek66.tools

import java.io.File
import java.net.URL

import org.junit.runner.RunWith
import org.scalatest.{Ignore, FunSuite}
import org.scalatest.junit.JUnitRunner


@Ignore
class Downloader$Test extends FunSuite {

  test("downloading-something") {

    val hello = new File("hello-test.html")
    val mystem = new File("atmta.binary")

    Downloader.downloadBinaryFile(new URL("http://www.stachek66.ru/"), hello)

    Downloader.downloadBinaryFile(
      new URL("http://download.cdn.yandex.net/mystem/mystem-3.0-linux3.1-64bit.tar.gz"),
      mystem
    )

    Downloader.downloadBinaryFile(
      new URL("http://download.cdn.yandex.net/mystem/mystem-3.1-win-64bit.zip"),
      mystem
    )

    hello.delete
    mystem.delete
  }

  test("download-and-unpack") {
    val bin = new File("atmta.binary.tar.gz")
    val bin2 = new File("executable")

    Decompressor.select.unpack(
      Downloader.downloadBinaryFile(
        new URL("http://download.cdn.yandex.net/mystem/mystem-3.0-linux3.1-64bit.tar.gz"),
        bin),
      bin2
    )

    bin.delete
    bin2.delete
  }
} 

package s4j.scala.chapter13

import java.io._
import java.net.{MalformedURLException, URL}


object Exceptions extends App {

  autocloseExample()

  def example2() = try {
    val url = new URL("http://baddotrobot.com")
    val reader = new BufferedReader(new InputStreamReader(url.openStream))
    try {
      var line = reader.readLine
      while (line != null) {
        line = reader.readLine
        println(line)
      }
    } finally {
      reader.close()
    }
  } catch {
    case _: MalformedURLException => System.out.println("Bad URL")
    case e: IOException => System.out.println("Problem reading data from the web: " + e.getMessage)
  }

  def autoclose[A, B <: Closeable](resource: B)(f: => A): A =
    try {
      f
    } finally {
      resource.close()
    }

  def autocloseExample() = {
    try {
      val url = new URL("http://baddotrobot.com")
      val reader = new BufferedReader(new InputStreamReader(url.openStream))
      autoclose(reader) {
        var line = reader.readLine
        while (line != null) {
          line = reader.readLine
          println(line)
        }
      }
    } catch {
      case _: MalformedURLException => System.out.println("Bad URL")
      case e: IOException => System.out.println("Problem reading data from the web: " + e.getMessage)
    }
  }

  def try_[A, B <: Closeable](resource: B)(f: B => A): A =
    try {
      f(resource)
    } finally {
      resource.close()
    }

  def autocloseExample2() = {
    try {
      val url = new URL("http://baddotrobot.com")
      try_(new BufferedReader(new InputStreamReader(url.openStream))) {
        reader => {
          var line = reader.readLine
          while (line != null) {
            line = reader.readLine
            println(line)
          }
        }
      }
    } catch {
      case _: MalformedURLException => System.out.println("Bad URL")
      case e: IOException => System.out.println("Problem reading data from the web: " + e.getMessage)
    }
  }

} 

package s4j.scala.chapter18

import java.net.{MalformedURLException, URL}

object Extractors {

  object YearsOfCustom {
    def unapply(customer: Customer): Option[Int] = Some(customer.yearsACustomer)
  }


  object UrlExtractor {
    def unapply(string: String): Option[(String, String)] = {
      try {
        val url = new URL(string)
        Some((url.getProtocol, url.getHost))
      } catch {
        case _: MalformedURLException => None
      }
    }
  }


  object DiscountExtractor {
    def unapply(customer: Customer): Option[Discount] = {
      if (customer.yearsACustomer >= 5) Some(Discount(0.5))
      else if (customer.yearsACustomer >= 2) Some(Discount(0.2))
      else if (customer.yearsACustomer >= 1) Some(Discount(0.1))
      else None
    }
  }

} 

package com.kakao.cuesheet.deps

import java.io.{BufferedOutputStream, File, FileOutputStream, IOException}
import java.net.{URL, URLDecoder}
import java.nio.file.{Files, Paths}
import java.util.zip.{ZipEntry, ZipOutputStream}

import com.kakao.mango.io.FileSystems
import com.kakao.mango.logging.Logging
import com.kakao.shaded.guava.io.Files.createTempDir

sealed trait DependencyNode {
  def path: String
}

case class ManagedDependency(group: String, artifact: String, classifier: String = "jar")

case class ManagedDependencyNode(
  path: String,
  group: String,
  artifact: String,
  classifier: String,
  version: String,
  children: Seq[ManagedDependency]
) extends DependencyNode {
  def key = ManagedDependency(group, artifact, classifier)
}

case class DirectoryDependencyNode(path: String) extends DependencyNode with Logging {
  lazy val compressed: UnmanagedDependencyNode = {
    val tmpdir = createTempDir()
    val jar = new File(s"${tmpdir.getAbsolutePath}/local-${tmpdir.getName}.jar")
    val root = Paths.get(path)

    val output = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(jar)))
    var count = 0

    FileSystems.entries(root).foreach { path =>
      if (resourceExtensions.exists(path.toString.endsWith)) {
        val entry = new ZipEntry(root.relativize(path).toString)
        output.putNextEntry(entry)
        try {
          Files.copy(path, output)
          count += 1
        } catch {
          case e: IOException => logger.warn(s"skipping $path due to an IOException: ${e.getMessage}")
        }
        output.closeEntry()
      }
    }

    output.close()

    logger.debug(s"Successfully zipped $count files in $path into $jar")

    UnmanagedDependencyNode(jar.getAbsolutePath)
  }
}

case class JavaRuntimeDependencyNode(path: String) extends DependencyNode
case class UnmanagedDependencyNode(path: String) extends DependencyNode

object DependencyNode {

  val resolver = new ChainedArtifactResolver(
    new IvyPathArtifactResolver,
    new IvyOriginalPathArtifactResolver,
    new MavenPathArtifactResolver,
    new GradlePathArtifactResolver,
    new JavaRuntimeResolver,
    new MavenMetadataArtifactResolver,
    new UnmanagedJarResolver
  )

  def resolve(url: URL): DependencyNode = {
    if (url.getProtocol != "file") {
      throw new IllegalArgumentException("non-file dependency is not supported")
    }

    val path = URLDecoder.decode(url.getFile, "UTF-8")
    val file = new File(path)
    if (file.isDirectory) {
      return DirectoryDependencyNode(file.getAbsolutePath)
    }

    if (!file.isFile || !file.canRead) {
      throw new IllegalArgumentException(s"$path is not a file or readable")
    }

    DependencyNode.resolver.resolve(file.getAbsolutePath) match {
      case Some(node) => node
      case None => throw new IllegalArgumentException(s"Could not determine the dependency of $path")
    }
  }
} 

package com.kakao.cuesheet.deps

import java.net.{URL, URLClassLoader}

import com.kakao.mango.concurrent.KeyedSingletons
import com.kakao.mango.logging.Logging
import com.kakao.mango.reflect.Accessible

import scala.reflect.io.AbstractFile

class DependencyAnalyzer(loader: ClassLoader = getClass.getClassLoader) extends Logging {
  val chain: List[ClassLoader] = DependencyAnalyzer.classLoaderChain(loader)

  lazy val graph = {
    // collect all nodes
    val nodes = for (
      loader <- chain;
      url <- DependencyAnalyzer.components(loader)
    ) yield {
      DependencyNode.resolve(url)
    }
    new DependencyGraph(nodes)
  }
}

object DependencyAnalyzer extends KeyedSingletons[ClassLoader, DependencyAnalyzer] with Logging {

  
  def components(loader: ClassLoader): Seq[URL] = {
    loader match {
      case _ if loader.getClass.getName.startsWith("sun.misc.Launcher$ExtClassLoader") =>
        Nil // ignore extension class loader
      case loader: URLClassLoader =>
        loader.getURLs
      case _ if loader.getClass.getName == "scala.tools.nsc.interpreter.AbstractFileClassLoader" =>
        val root = Accessible.field(loader.getClass, "root")
        Seq(root.get(loader).asInstanceOf[AbstractFile].toURL)
      case _ if loader.getClass.getName == "scala.reflect.internal.util.AbstractFileClassLoader" =>
        val root = Accessible.field(loader.getClass, "root")
        Seq(root.get(loader).asInstanceOf[AbstractFile].toURL)
      case _ if Seq(loader.getClass.getName).exists(c => c.startsWith("xsbt.") || c.startsWith("sbt.")) =>
        Nil // ignore SBT's internal loader
      case _ =>
        throw new RuntimeException("Unknown ClassLoader Type: " + loader.getClass.getName)
    }
  }

  override def newInstance(loader: ClassLoader): DependencyAnalyzer = {
    new DependencyAnalyzer(loader)
  }

  def apply(): DependencyAnalyzer = apply(getClass.getClassLoader)

} 

package docspell.analysis.nlp

import java.net.URL
import java.util.zip.GZIPInputStream

import scala.jdk.CollectionConverters._
import scala.util.Using

import docspell.common._

import edu.stanford.nlp.ie.AbstractSequenceClassifier
import edu.stanford.nlp.ie.crf.CRFClassifier
import edu.stanford.nlp.ling.{CoreAnnotations, CoreLabel}
import org.log4s.getLogger

object StanfordNerClassifier {
  private[this] val logger = getLogger

  lazy val germanNerClassifier  = makeClassifier(Language.German)
  lazy val englishNerClassifier = makeClassifier(Language.English)

  def nerAnnotate(lang: Language)(text: String): Vector[NerLabel] = {
    val nerClassifier = lang match {
      case Language.English => englishNerClassifier
      case Language.German  => germanNerClassifier
    }
    nerClassifier
      .classify(text)
      .asScala
      .flatMap(a => a.asScala)
      .collect(Function.unlift { label =>
        val tag = label.get(classOf[CoreAnnotations.AnswerAnnotation])
        NerTag
          .fromString(Option(tag).getOrElse(""))
          .toOption
          .map(t => NerLabel(label.word(), t, label.beginPosition(), label.endPosition()))
      })
      .toVector
  }

  private def makeClassifier(lang: Language): AbstractSequenceClassifier[CoreLabel] = {
    logger.info(s"Creating ${lang.name} Stanford NLP NER classifier...")
    val ner = classifierResource(lang)
    Using(new GZIPInputStream(ner.openStream())) { in =>
      CRFClassifier.getClassifier(in).asInstanceOf[AbstractSequenceClassifier[CoreLabel]]
    }.fold(throw _, identity)
  }

  private def classifierResource(lang: Language): URL = {
    def check(u: URL): URL =
      if (u == null) sys.error(s"NER model url not found for language ${lang.name}")
      else u

    check(lang match {
      case Language.German =>
        getClass.getResource(
          "/edu/stanford/nlp/models/ner/german.conll.germeval2014.hgc_175m_600.crf.ser.gz"
        )
      case Language.English =>
        getClass.getResource(
          "/edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz"
        )
    })
  }
} 

package examples.templating

import java.net.URL

import com.twitter.finagle.http.Method.Get
import com.twitter.finagle.http.Request
import com.twitter.finagle.http.filter.Cors
import com.twitter.finagle.http.filter.Cors.HttpFilter
import com.twitter.finagle.http.path.Root
import com.twitter.finagle.{Http, Service}
import com.twitter.util.Await
import io.fintrospect.formats.PlainText
import io.fintrospect.renderers.SiteMapModuleRenderer
import io.fintrospect.templating.{MustacheTemplates, RenderView}
import io.fintrospect.{RouteModule, RouteSpec}

object TemplatingApp extends App {

  val devMode = true
  val renderer = if (devMode) MustacheTemplates.HotReload("src/main/resources") else MustacheTemplates.CachingClasspath(".")

  val renderView = new RenderView(PlainText.ResponseBuilder, renderer)

  val module = RouteModule(Root, new SiteMapModuleRenderer(new URL("http://my.cool.app")), renderView)
    .withRoute(RouteSpec().at(Get) / "echo" bindTo Service.mk { rq: Request => MustacheView(rq.uri) })

  println("See the Sitemap description at: http://localhost:8181")

  Await.ready(
    Http.serve(":8181", new HttpFilter(Cors.UnsafePermissivePolicy).andThen(module.toService))
  )
} 

package io.fintrospect.renderers

import java.net.URL

import com.twitter.finagle.http.Method.Get
import com.twitter.finagle.http.path.Path
import com.twitter.finagle.http.{Request, Response, Status}
import io.fintrospect.formats.Xml.ResponseBuilder._
import io.fintrospect.util.ExtractionError
import io.fintrospect.{Security, ServerRoute}

class SiteMapModuleRenderer(baseUrl: URL) extends ModuleRenderer {

  override def badRequest(badParameters: Seq[ExtractionError]): Response = BadRequest(badParameters.toString())

  override def notFound(request: Request): Response = HttpResponse(Status.NotFound).build()

  override def description(basePath: Path, security: Security, routes: Seq[ServerRoute[_, _]]): Response = {
    def buildUrl(route: ServerRoute[_, _]) =
      <url>
        <loc>
          {baseUrl + route.describeFor(basePath)}
        </loc>
      </url>

    Ok(<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
      {routes.filter(_.method == Get).map(buildUrl)}
    </urlset>)
  }
} 

package io.fintrospect

import java.io.File
import java.net.URL

trait ResourceLoader {
  def load(path: String): URL
}

object ResourceLoader {
  
  def Directory(baseDir: String) = new ResourceLoader {
    private val finalBaseDir = if (baseDir.endsWith("/")) baseDir else baseDir + "/"

    override def load(path: String): URL = {
      val f = new File(finalBaseDir, path)
      if (f.exists() && f.isFile) f.toURI.toURL else null
    }
  }
} 

package io.fintrospect.renderers

import java.net.URL

import com.twitter.finagle.Service
import com.twitter.finagle.http.Method._
import com.twitter.finagle.http.Status.{BadRequest, NotFound, Ok}
import com.twitter.finagle.http.path.Root
import com.twitter.finagle.http.{Method, Request, Response}
import com.twitter.util.Future
import io.fintrospect.util.HttpRequestResponseUtil.statusAndContentFrom
import io.fintrospect.{NoSecurity, RouteSpec, ServerRoute}
import org.scalatest.{FunSpec, Matchers}

class SiteMapModuleRendererTest extends FunSpec with Matchers {

  it("renders 400") {
    new SiteMapModuleRenderer(new URL("http://fintrospect.io")).badRequest(Nil).status shouldBe BadRequest
  }

  it("renders 404") {
    new SiteMapModuleRenderer(new URL("http://fintrospect.io")).notFound(Request()).status shouldBe NotFound
  }

  it("should describe only GET endpoints of module as a sitemap") {
    val rsp = new SiteMapModuleRenderer(new URL("http://fintrospect.io")).description(Root / "bob", NoSecurity, Seq(
      endpointFor(Get),
      endpointFor(Post),
      endpointFor(Delete),
      endpointFor(Put),
      endpointFor(Options),
      endpointFor(Connect),
      endpointFor(Head),
      endpointFor(Trace)
    ))

    val (status, content) = statusAndContentFrom(rsp)
    status shouldBe Ok
    content shouldBe <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
      <url>
        <loc>
          http://fintrospect.io/bob/GET
        </loc>
      </url>
    </urlset>.toString()
  }

  private def endpointFor(method: Method): ServerRoute[Request, Response] = {
    RouteSpec().at(method) / method.toString() bindTo Service.mk[Request, Response]((r) => Future(Response()))
  }
} 

package org.codeoverflow.chatoverflow.framework.helper

import java.net.{URL, URLClassLoader}

import org.codeoverflow.chatoverflow.WithLogger


private object PluginClassLoader extends WithLogger {
  val appClassloader: ClassLoader = this.getClass.getClassLoader
  val platformClassloader: ClassLoader = {
    var current = appClassloader
    while (current != null && !current.getClass.getName.contains("ExtClassLoader") && // ExtClassLoader is java < 9
      !current.getClass.getName.contains("PlatformClassLoader")) { // PlatformClassLoader is java >= 9
      current = current.getParent
    }

    if (current != null) {
      current
    } else {
      logger error "Platform classloader couldn't be found. Falling back to normal app classloader."
      appClassloader
    }
  }
} 

package uk.gov.hmrc.http

import java.net.{ConnectException, URL}
import java.util.concurrent.TimeoutException

import com.typesafe.config.Config

import scala.collection.JavaConverters.iterableAsScalaIterableConverter
import scala.concurrent.{ExecutionContext, Future}
import scala.util.matching.Regex

trait HttpVerb extends Request {

  protected def configuration: Option[Config]

  def mapErrors(httpMethod: String, url: String, f: Future[HttpResponse])(
    implicit ec: ExecutionContext): Future[HttpResponse] =
    f.recoverWith {
      case e: TimeoutException => Future.failed(new GatewayTimeoutException(gatewayTimeoutMessage(httpMethod, url, e)))
      case e: ConnectException => Future.failed(new BadGatewayException(badGatewayMessage(httpMethod, url, e)))
    }

  def badGatewayMessage(verbName: String, url: String, e: Exception): String =
    s"$verbName of '$url' failed. Caused by: '${e.getMessage}'"

  def gatewayTimeoutMessage(verbName: String, url: String, e: Exception): String =
    s"$verbName of '$url' timed out with message '${e.getMessage}'"

  lazy val internalHostPatterns: Seq[Regex] = configuration match {
    case Some(config) if config.hasPathOrNull("internalServiceHostPatterns") =>
      config.getStringList("internalServiceHostPatterns").asScala.map(_.r).toSeq
    case _ =>
      Seq("^.*\\.service$".r, "^.*\\.mdtp$".r)
  }

  lazy val userAgentHeader: Seq[(String, String)] = configuration match {
    case Some(config) if config.hasPathOrNull("appName") =>
      Seq("User-Agent" -> config.getString("appName"))
    case _ =>
      Seq.empty
  }

  override def applicableHeaders(url: String)(implicit hc: HeaderCarrier): Seq[(String, String)] = {
    val headers = if (internalHostPatterns.exists(_.pattern.matcher(new URL(url).getHost).matches())) {
      hc.headers
    } else {
      hc.headers.filterNot(hc.otherHeaders.contains(_))
    }

    headers ++ userAgentHeader
  }
} 

package com.testerhome.appcrawler.it

import java.net.URL

import com.testerhome.appcrawler.AppiumSuite
import org.openqa.selenium.Capabilities
import org.openqa.selenium.remote.{DesiredCapabilities, RemoteWebDriver}
import org.scalatest.FunSuite
import scala.collection.JavaConversions._


class TestWebDriverAgent extends AppiumSuite{
  test("use remote webdriver"){
    val capability=new DesiredCapabilities()
    capability.setCapability("app", "/Users/seveniruby/projects/snowball-ios/DerivedData/Snowball/Build/Products/Debug-iphonesimulator/Snowball.app")
    capability.setCapability("bundleId", "com.xueqiu")
    capability.setCapability("fullReset", "true")
    capability.setCapability("noReset", "true")
    capability.setCapability("udid", "4F05E384-FE32-43DE-8539-4DC3E2EBC117")
    capability.setCapability("automationName", "XCUITest")
    capability.setCapability("platformName", "ios")
    capability.setCapability("deviceName", "iPhone Simulator")
    capability.setCapability("bundleId", "com.xueqiu")

    //val url="http://192.168.100.65:7771"
    val url="http://127.0.0.1:4723/wd/hub"
    val driver=new RemoteWebDriver(new URL(url), capability)
    println(driver.getPageSource)
  }


  test("use remote webdriver meituan"){
    val capability=new DesiredCapabilities()
    capability.setCapability("app", "/Users/seveniruby/Downloads/app/waimai.app")
    capability.setCapability("bundleId", "com.meituan.iToGo.ep")
    //capability.setCapability("fullReset", false)
    //capability.setCapability("noReset", true)
    //capability.setCapability("udid", "4F05E384-FE32-43DE-8539-4DC3E2EBC117")
    capability.setCapability("automationName", "XCUITest")
    capability.setCapability("platformName", "ios")
    capability.setCapability("deviceName", "iPhone 6")
    capability.setCapability("platformVersion", "10.2")
    capability.setCapability("autoAcceptAlerts", true)
    //capability.setCapability("webDriverAgentUrl", "http://172.18.118.90:8100/")

    //val url="http://192.168.100.65:7771"
    //val url="http://127.0.0.1:8100"
    val url="http://127.0.0.1:4730/wd/hub"
    val driver=new RemoteWebDriver(new URL(url), capability)

    while(true){
      Thread.sleep(2000)
      println(driver.getPageSource)
    }

  }

  test("use remote webdriver xueqiu"){
    val capability=new DesiredCapabilities()
    capability.setCapability("app", "/Users/seveniruby/projects/snowball-ios/DerivedData/Snowball/Build/Products/Debug-iphonesimulator/Snowball.app")
    capability.setCapability("bundleId", "com.xueqiu")
    capability.setCapability("fullReset", "false")
    capability.setCapability("noReset", "true")
    //capability.setCapability("udid", "4F05E384-FE32-43DE-8539-4DC3E2EBC117")
    capability.setCapability("automationName", "XCUITest")
    capability.setCapability("platformName", "ios")
    capability.setCapability("deviceName", "iPhone Simulator")
    capability.setCapability("bundleId", "com.xueqiu")
    capability.setCapability("autoAcceptAlerts", true)


    //val url="http://192.168.100.65:7771"
    //val url="http://127.0.0.1:8100"
    val url="http://127.0.0.1:4730/wd/hub"
    val driver=new RemoteWebDriver(new URL(url), capability)

    while(true){
      Thread.sleep(2000)
      driver.findElementsByXPath("//*").foreach(e=>{
        println(s"tag=${e.getTagName} text=${e.getText}")
      })
      println(driver.getPageSource)
      println("==============")
    }

  }
} 

package com.testerhome.appcrawler.it

import java.net.URL

import org.openqa.selenium.chrome.{ChromeOptions, ChromeDriver}
import org.openqa.selenium.remote.{RemoteWebDriver, DesiredCapabilities}
import org.scalatest.FunSuite
import collection.JavaConversions._


class TestNW extends FunSuite{
  test("test nw"){

    System.setProperty("webdriver.chrome.driver",
      "/Users/seveniruby/projects/nwjs/ics4_debug_nw0.14.7/chromedriver")
    val options=new ChromeOptions()
    options.addArguments("nwapp=/Users/seveniruby/projects/nwjs/ics4_debug_nw0.14.7/app")
    val driver=new ChromeDriver(options)
    println(driver.getPageSource)
    Thread.sleep(2000)
    driver.findElementsByXPath("//label").foreach(x=>{
      println(x.getTagName)
      println(x.getLocation)
      println(x.getText)
      println("text()="+x.getAttribute("text()"))
      println("text="+x.getAttribute("text"))
      println("value="+x.getAttribute("value"))
      println("name="+x.getAttribute("name"))
      println("id="+x.getAttribute("id"))
      println("class="+x.getAttribute("class"))
      println("type="+x.getAttribute("type"))
      println("placeholder="+x.getAttribute("placeholder"))
      println("============")
    })
    driver.findElementByXPath("//label[contains(., 'selectedRegion')]").click()

    //driver.quit()

  }

  test("test nw remote"){
    val options=new ChromeOptions()
    options.addArguments("nwapp=/Users/seveniruby/projects/nwjs/ics4_debug_nw0.14.7/app")
    val url="http://10.3.2.65:4444/wd/hub"

    val dc = DesiredCapabilities.chrome()
    dc.setCapability(ChromeOptions.CAPABILITY, options)

    val driver=new RemoteWebDriver(new URL(url), dc)
    println(driver.getPageSource)
    Thread.sleep(2000)
    driver.findElementsByXPath("//label").foreach(x=>{
      println(x.getTagName)
      println(x.getLocation)
      println(x.getText)
      println("text()="+x.getAttribute("text()"))
      println("text="+x.getAttribute("text"))
      println("value="+x.getAttribute("value"))
      println("name="+x.getAttribute("name"))
      println("id="+x.getAttribute("id"))
      println("class="+x.getAttribute("class"))
      println("type="+x.getAttribute("type"))
      println("placeholder="+x.getAttribute("placeholder"))
      println("============")
    })
    driver.findElementByXPath("//label[contains(., 'selectedRegion')]").click()

    //driver.quit()

  }

} 

package com.testerhome.appcrawler.it

import java.net.URL

import com.sun.jdi.connect.spi.TransportService.Capabilities
import com.testerhome.appcrawler.driver.AppiumClient
import io.appium.java_client.android.AndroidDriver
import io.appium.java_client.remote.{AndroidMobileCapabilityType, MobileCapabilityType}
import org.openqa.selenium.WebElement
import org.openqa.selenium.remote.DesiredCapabilities
import org.scalatest.FunSuite

import scala.io.Source


class TestAppium extends FunSuite{
  val a=new AppiumClient()
  test("appium success"){
    a.start()
    println(Source.fromURL("http://127.0.0.1:4723/wd/hub/sessions").mkString)
    a.stop()
  }

  test("single session"){
    val capa=new DesiredCapabilities()
    capa.setCapability(AndroidMobileCapabilityType.APP_PACKAGE, "com.xueqiu.android")
    capa.setCapability(AndroidMobileCapabilityType.APP_ACTIVITY, ".view.WelcomeActivityAlias")
    capa.setCapability(MobileCapabilityType.DEVICE_NAME, "demo")
    val driver=new AndroidDriver[WebElement](new URL("http://127.0.0.1:4723/wd/hub/"), capa)


  }
} 

package com.testerhome.appcrawler.it

import java.net.URL

import com.testerhome.appcrawler.AppCrawler
import io.appium.java_client.android.{AndroidDriver, AndroidElement}
import org.openqa.selenium.remote.DesiredCapabilities
import org.scalatest.FunSuite

class TestXueQiu extends FunSuite{
  val capability=new DesiredCapabilities()
  capability.setCapability("app", "")
  capability.setCapability("appPackage", "com.tencent.mm")
  capability.setCapability("appActivity", ".ui.LauncherUI")
  capability.setCapability("deviceName", "emulator-5554")
  capability.setCapability("fastReset", "false")
  capability.setCapability("fullReset", "false")
  capability.setCapability("noReset", "true")
  capability.setCapability("unicodeKeyboard", "true")
  capability.setCapability("resetKeyboard", "true")
  capability.setCapability("automationName", "appium")

  test("all app "){
    capability.setCapability("app", "")
    capability.setCapability("appPackage", "com.xueqiu.android")
    capability.setCapability("appActivity", ".view.WelcomeActivityAlias")
    val driver=new AndroidDriver[AndroidElement](new URL("http://127.0.0.1:4723/wd/hub"), capability)

  }

  test("appcrawler"){
    AppCrawler.main(Array("-c", "src/test/scala/com/testerhome/appcrawler/it/xueqiu_private.yml",
      "-o", s"/tmp/xueqiu/${System.currentTimeMillis()}", "--verbose"
    )
    )
  }

  test("appcrawler base example"){
    AppCrawler.main(Array("-c", "src/test/scala/com/testerhome/appcrawler/it/xueqiu_base.yml",
      "-o", s"/tmp/xueqiu/${System.currentTimeMillis()}", "--verbose"
    )
    )
  }

} 

package com.testerhome.appcrawler.it

import java.net.URL

import io.appium.java_client.android.AndroidDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.remote.DesiredCapabilities
import org.scalatest._

import scala.collection.JavaConversions._


class TestTesterHome extends FunSuite with BeforeAndAfterAll with BeforeAndAfterEach with Matchers {

  val capabilities=new DesiredCapabilities()
  capabilities.setCapability("deviceName", "emulator-5554")
  capabilities.setCapability("app", "/Users/seveniruby/Downloads/app-release.apk_1.1.0.apk")
  capabilities.setCapability("appPackage", "com.testerhome.nativeandroid")
  capabilities.setCapability("appActivity", ".views.MainActivity")
  capabilities.setCapability("unicodeKeyboard", "true")

  var driver=new AndroidDriver[WebElement](new URL("http://127.0.0.1:4723/wd/hub/"), capabilities)

  override def beforeEach(): Unit = {
    capabilities.setCapability("app", "")
    driver=new AndroidDriver[WebElement](new URL("http://127.0.0.1:4723/wd/hub/"), capabilities)
    Thread.sleep(3000)
    verbose()

  }

  def verbose(): Unit ={
    println()
    println(driver.currentActivity())
    println(driver.getPageSource)
  }

  test("招聘"){
    driver.findElementByXPath("//*[@content-desc='Open navigation drawer']").click()
    driver.findElementByXPath("//*[@text='招聘']").click()
    driver.getContextHandles.foreach(println)
    verbose()
    driver.findElementsByXPath("//*[@text='欢迎报名第三届中国移动互联网测试开发大会']").size() should be >=1
  }
  test("精华帖"){
    driver.findElementByXPath("//*[@content-desc='Open navigation drawer']").click()
    driver.findElementByXPath("//*[@text='社区']").click()
    //等待动画切换完成
    Thread.sleep(3000)
    driver.findElementByXPath("//*[@text='精华']").click()
    driver.findElementByXPath("//*[contains(@text, '王者荣耀')]").click()
    driver.findElementByXPath("//*[contains(@text, '评论')]").click()
    driver.findElementsByXPath("//*[@text='恒温']").size() should be >=1
  }

  override def afterEach(): Unit = {
    driver.quit()
  }
} 

package com.testerhome.appcrawler.it

import java.net.URL

import io.appium.java_client.android.AndroidDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.remote.DesiredCapabilities
import org.scalatest._
import scala.collection.JavaConversions._


class TestJianShu extends FunSuite with BeforeAndAfterAll with BeforeAndAfterEach with Matchers {

  val capabilities=new DesiredCapabilities()
  capabilities.setCapability("deviceName", "emulator-5554")
  capabilities.setCapability("appPackage", "com.jianshu.haruki")
  capabilities.setCapability("appActivity", "com.baiji.jianshu.account.SplashScreenActivity")
  capabilities.setCapability("unicodeKeyboard", "true")

  var driver=new AndroidDriver[WebElement](new URL("http://127.0.0.1:4723/wd/hub/"), capabilities)

  override def beforeAll(): Unit ={
    capabilities.setCapability("app", "/Users/seveniruby/Downloads/Jianshu-2.3.1-17051515-1495076675.apk")
    driver=new AndroidDriver[WebElement](new URL("http://127.0.0.1:4723/wd/hub/"), capabilities)
    Thread.sleep(3000)
    verbose()
  }

  override def beforeEach(): Unit = {
    capabilities.setCapability("app", "")
    driver=new AndroidDriver[WebElement](new URL("http://127.0.0.1:4723/wd/hub/"), capabilities)
    Thread.sleep(3000)
    verbose()

  }

  def verbose(): Unit ={
    println()
    println(driver.currentActivity())
    println(driver.getPageSource)
  }

  test("绕过登陆"){
    driver.findElementByXPath("//*[@text='跳过']").click()
    driver.findElementById("iv_close").click()
    driver.findElementsByXPath("//*[@text='登录']").size() should be >= 1
  }

  test("错误密码登录"){
    driver.findElementByXPath("//*[@text='跳过']").click()
    driver.findElementByXPath("//*[@text='已有帐户登录']").click()
    driver.findElementByXPath("//*[@text='手机或邮箱']").sendKeys("[email protected]")
    driver.findElementByXPath("//*[@password='true']").sendKeys("wrong")
    driver.findElementByXPath("//*[@text='登录']").click()
    verbose()
    driver.findElementsByXPath("//*[contains(@text, '错误')]").size() should be >= 1
  }

  test("随便看看"){
    driver.findElementByXPath("//*[@text='跳过']").click()
    driver.findElementByXPath("//*[@text='随便看看']").click()
    verbose()
    driver.findElementsByXPath("//*[contains(@resource-id, 'tag_flow_layout')]//*[contains(name(),'TextView')]").foreach(tag => {
      tag.click()
      Thread.sleep(1000)
      driver.findElementsByXPath("//*[@text='关注']").size() should be >=1
      driver.navigate().back()
    })
  }

  override def afterEach(): Unit = {
    driver.quit()
  }
} 

package ch.renku.acceptancetests.model

import java.net.URL
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter

import ch.renku.acceptancetests.generators.Generators.Implicits._
import ch.renku.acceptancetests.generators.Generators._
import ch.renku.acceptancetests.model.users.UserCredentials
import eu.timepit.refined.api.Refined
import eu.timepit.refined.collection.NonEmpty

object projects {

  final case class ProjectIdentifier(
      namespace: String Refined NonEmpty,
      slug:      String Refined NonEmpty
  )

  final case class ProjectDetails(
      title:       String Refined NonEmpty,
      description: String Refined NonEmpty,
      readmeTitle: String
  )

  final case class ProjectUrl(value: String) {
    override lazy val toString: String = value
  }

  object ProjectUrl {

    implicit class ProjectUrlOps(projectUrl: ProjectUrl)(implicit userCredentials: UserCredentials) {
      import ch.renku.acceptancetests.tooling.UrlEncoder.urlEncode

      lazy val addGitCredentials: String = {
        val protocol = new URL(projectUrl.value).getProtocol
        projectUrl.value
          .replace(
            s"$protocol://",
            s"$protocol://${urlEncode(userCredentials.username.value)}:${urlEncode(userCredentials.password.value)}@"
          )
      }
    }
  }

  object ProjectDetails {

    def generate: ProjectDetails = {
      val now         = LocalDateTime.now()
      val desc        = prefixParagraph("An automatically generated project for testing: ").generateOne
      val readmeTitle = s"test${now.format(DateTimeFormatter.ofPattern("yyyyMMddHHmm_ss"))}"
      ProjectDetails(Refined.unsafeApply(s"test_${now.format(DateTimeFormatter.ofPattern("yyyy_MM_dd_HHmm_ss"))}"),
                     desc,
                     readmeTitle)
    }

    def generateHandsOnProject(captureScreenshots: Boolean): ProjectDetails =
      if (captureScreenshots) {
        val readmeTitle = "flights tutorial"
        ProjectDetails(Refined.unsafeApply(readmeTitle), Refined.unsafeApply("A renku tutorial project."), readmeTitle)
      } else
        generate

    implicit class TitleOps(title: String Refined NonEmpty) {
      lazy val toPathSegment: String = title.value.replace(" ", "-")
    }

  }

} 

// scrimage-core, scrimage-filters

import com.sksamuel.scrimage._, filter._
import java.io.{File, FileInputStream}
import java.net.URL
import java.nio.file.{Files, Paths}
import scala.util.Try

// Download image to cache
val dest = Paths.get("/tmp/scastie/lanzarote.jpg")
if (!Files.exists(dest)) {
  Files.createDirectories(dest.getParent)
  val url = new URL("https://github.com/sksamuel/scrimage/blob/master/scrimage-core/src/test/resources/lanzarote.jpg?raw=true")
  Try(url.openStream()).foreach(src => Files.copy(src, dest))
}
val image = Image.fromStream(new FileInputStream(new File("/tmp/scastie/lanzarote.jpg")))
val small = image.scaleToWidth(200)


toBase64(small)

toBase64(small.filter(SepiaFilter)) 

package scalaclean.model.impl

import scalaclean.model._
import java.io.File
import java.net.{URL, URLClassLoader}
import java.nio.file.{Files, Path, Paths}
import java.util.Properties
import java.util.concurrent.ConcurrentHashMap

import scalafix.v1.SymbolInformation

import scala.meta.internal.symtab.{GlobalSymbolTable, SymbolTable}
import scala.meta.io.{AbsolutePath, Classpath}

object Project {

  import org.scalaclean.analysis.PropertyNames._

  def apply(propsPath: Path, projects: ProjectSet): Project = {
    val props = new Properties()
    println("PropsPath = " + propsPath)
    props.load(Files.newBufferedReader(propsPath))
    val classpathValue = props.getProperty(prop_classpath)
    val outputPath = props.getProperty(prop_outputDir)
    val elementsFilePath = props.getProperty(prop_elementsFile)
    val relationshipsFilePath = props.getProperty(prop_relationshipsFile)
    val extensionsFilePath = props.getProperty(prop_extensionsFile)
    val src = props.getProperty(prop_src)
    val srcBuildBase = props.getProperty(prop_srcBuildBase)
    val srcFiles = props.getProperty(prop_srcFiles, "").split(File.pathSeparatorChar).toSet
    val srcRoots = props.getProperty(prop_srcRoots).split(File.pathSeparatorChar).toList.sortWith((s1, s2) => s1.length > s1.length || s1 < s2).map(AbsolutePath(_))
    println("srcRoots = " + srcRoots)
    assert(classpathValue ne null, props.keys)
    assert(outputPath ne null, props.keys)

    val classPath = Classpath.apply(classpathValue)

    new Project(projects, classPath, outputPath, src, srcRoots, srcBuildBase, elementsFilePath, relationshipsFilePath, extensionsFilePath, srcFiles)
  }
}

class Project private(
                       val projects: ProjectSet, val classPath: Classpath, val outputPath: String, val src: String, val srcRoots: List[AbsolutePath], val srcBuildBase: String,
                       elementsFilePath: String, relationshipsFilePath: String, extensionsFilePath: String,
                       val srcFiles: Set[String]) {
  def symbolTable: SymbolTable = GlobalSymbolTable(classPath, includeJdk = true)

  lazy val classloader: ClassLoader = new URLClassLoader(Array(new URL("file:" + outputPath + "/")), null)

  private val infos = new ConcurrentHashMap[LegacyElementId, SymbolInformation]()

  def symbolInfo(viewedFrom: ElementModelImpl, symbol: LegacyElementId): SymbolInformation = {
    infos.computeIfAbsent(symbol,
      s => //any doc in the project would do though
        viewedFrom.source.doc.info(s.symbol).orNull)
  }

  def read: (Vector[ElementModelImpl], BasicRelationshipInfo) = ModelReader.read(this, elementsFilePath, relationshipsFilePath, extensionsFilePath)

  private val sourcesMap = new ConcurrentHashMap[String, SourceData]()

  def source(name: String): SourceData = {
    sourcesMap.computeIfAbsent(name, p => SourceData(this, Paths.get(p)))
  }

} 

package mouse

import java.net.{MalformedURLException, URI, URISyntaxException, URL}

import cats.Eq
import cats.syntax.all._
import mouse.string._

class StringJvmTests extends MouseSuite {

  test("parseFloat") {
    "123.1".parseFloat should ===(123.1f.asRight[NumberFormatException])
  }

  test("parseURL") {
    implicit val urlEq: Eq[URL] = Eq.fromUniversalEquals
    implicit val malformedURLExceptionEq: Eq[MalformedURLException] =
      new Eq[MalformedURLException] {
        override def eqv(x: MalformedURLException, y: MalformedURLException): Boolean =
          x.getMessage == y.getMessage
      }

    "http://example.com".parseURL should ===(new URL("http://example.com").asRight[MalformedURLException])

    "blah".parseURL should ===(new MalformedURLException("no protocol: blah").asLeft)
  }

  test("parseURI") {
    implicit val urlEq: Eq[URI] = Eq.fromUniversalEquals
    implicit val malformedURIExceptionEq: Eq[URISyntaxException] =
      new Eq[URISyntaxException] {
        override def eqv(x: URISyntaxException, y: URISyntaxException): Boolean =
          x.getMessage == y.getMessage
      }

    "http://example.com".parseURI should ===(new URI("http://example.com").asRight[URISyntaxException])

    "invalid uri".parseURI should ===(new URISyntaxException("invalid uri", "Illegal character in path at index 7").asLeft)
  }

} 

import java.net.URL

object Helpers {
  def generatePomExtra(scmUrl: String, scmConnection: String,
                       developerId: String, developerName: String): xml.NodeSeq =
    <scm>
      <url>{ scmUrl }</url>
      <connection>{ scmConnection }</connection>
    </scm>
    <developers>
      <developer>
        <id>{ developerId }</id>
        <name>{ developerName }</name>
      </developer>
    </developers>
} 

package net.virtualvoid.optimizer

import java.io.File
import java.net.URL

sealed trait DownloadKind extends Product {
  def extraInfo: String = ""
}
case object Failed extends DownloadKind
case class GetURLInfo(isAvailable: Boolean) extends DownloadKind {
  import Console._
  override def extraInfo: String =
    if (isAvailable) s"${GREEN}AVAILABLE$RESET"
    else s"${RED}MISSING$RESET"
}
case class DownloadURL(to: File, size: Long) extends DownloadKind {
  override def extraInfo: String = f"-> $to%s $size%d"
}
case object OpenStreamURL extends DownloadKind
case class NetworkAccess(url: URL, kind: DownloadKind, startNanos: Long, endNanos: Long) extends Span {
  import Console._
  def color: String = url.getProtocol match {
    case "file"           ⇒ GREEN
    case "jar"            ⇒ YELLOW
    case "http" | "https" ⇒ CYAN
    case _                ⇒ BLUE
  }

  override def toString: String =
    f"$lastedMillis%5d ms ${kind.productPrefix}%-15s $color%s$url%s$RESET ${kind.extraInfo}"
} 

package com.marcospereira.play.i18n

import java.net.URL
import java.util.Properties
import javax.inject.{ Inject, Singleton }

import com.typesafe.config.ConfigFactory
import play.api.http.HttpConfiguration
import play.api.i18n._
import play.api.inject.Module
import play.api.{ Configuration, Environment }
import play.utils.Resources

import scala.collection.JavaConverters._

@Singleton
class HoconMessagesApiProvider @Inject() (
  environment: Environment,
  config: Configuration,
  langs: Langs,
  httpConfiguration: HttpConfiguration
)
    extends DefaultMessagesApiProvider(environment, config, langs, httpConfiguration) {

  override lazy val get: MessagesApi = {
    new DefaultMessagesApi(
      loadAllMessages,
      langs,
      langCookieName = langCookieName,
      langCookieSecure = langCookieSecure,
      langCookieHttpOnly = langCookieHttpOnly,
      httpConfiguration = httpConfiguration
    )
  }

  override protected def loadMessages(file: String): Map[String, String] = {
    getResources(file)
      .filterNot(url => Resources.isDirectory(environment.classLoader, url)).reverse
      .map(getMessages)
      .foldLeft(Map.empty[String, String]) { _ ++ _ }
  }

  override protected def loadAllMessages: Map[String, Map[String, String]] = {
    langs.availables.map(_.code).map { lang =>
      (lang, loadMessages(s"messages.$lang.conf"))
    }.toMap ++ Map(
      "default" -> loadMessages("messages.conf"),
      "default.play" -> loadMessages("messages.default")
    )
  }

  override protected def joinPaths(first: Option[String], second: String) = first match {
    case Some(parent) => new java.io.File(parent, second).getPath
    case None => second
  }

  private def getResources(file: String): List[URL] = {
    environment.classLoader.getResources(joinPaths(messagesPrefix, file)).asScala.toList
  }

  private def getMessages(url: URL): Map[String, String] = {
    // messages.default is bundled with play and it is a properties file
    val config = if (url.toString.endsWith("messages.default")) {
      ConfigFactory.parseProperties(getProperties(url))
    } else {
      ConfigFactory.parseURL(url)
    }

    config.resolve().entrySet().asScala
      .map(e => e.getKey -> String.valueOf(e.getValue.unwrapped()))
      .toMap
  }

  private def getProperties(url: URL): Properties = {
    val properties = new Properties()
    val input = url.openStream()
    try {
      properties.load(input)
    } finally {
      input.close()
    }
    properties
  }
}


trait HoconI18nComponents extends I18nComponents {

  def environment: Environment
  def configuration: Configuration
  def httpConfiguration: HttpConfiguration
  def langs: Langs

  override lazy val messagesApi: MessagesApi = new HoconMessagesApiProvider(environment, configuration, langs, httpConfiguration).get
} 

package pureconfig

import java.net.URL

import com.typesafe.config.{ ConfigOrigin, ConfigOriginFactory }

import scala.reflect.ClassTag
import org.scalatest._
import org.scalatest.matchers.{ MatchResult, Matcher }
import pureconfig.error._
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

trait ConfigReaderMatchers { this: AnyFlatSpec with Matchers =>

  def failWith(reason: FailureReason): Matcher[ConfigReader.Result[Any]] =
    matchPattern { case Left(ConfigReaderFailures(ConvertFailure(`reason`, _, _))) => }

  def failWith(
    reason: FailureReason,
    path: String,
    origin: Option[ConfigOrigin] = None): Matcher[ConfigReader.Result[Any]] =
    be(Left(ConfigReaderFailures(ConvertFailure(reason, origin, path))))

  def failWith(failure: ConfigReaderFailure): Matcher[ConfigReader.Result[Any]] =
    be(Left(ConfigReaderFailures(failure)))

  def failWithType[Reason <: FailureReason: ClassTag]: Matcher[ConfigReader.Result[Any]] =
    matchPattern { case Left(ConfigReaderFailures(ConvertFailure(_: Reason, _, _))) => }

  def failWithType[Failure <: ConfigReaderFailure: ClassTag](implicit dummy: DummyImplicit): Matcher[ConfigReader.Result[Any]] =
    matchPattern { case Left(ConfigReaderFailures(_: Failure)) => }

  def failLike(pf: PartialFunction[ConfigReaderFailure, MatchResult]) =
    new Matcher[ConfigReader.Result[Any]] with Inside with PartialFunctionValues {

      def apply(left: ConfigReader.Result[Any]): MatchResult = {
        inside(left) { case Left(ConfigReaderFailures(failure)) => pf.valueAt(failure) }
      }
    }

  def stringConfigOrigin(line: Int) =
    Some(ConfigOriginFactory.newSimple("String").withLineNumber(line))

  def urlConfigOrigin(url: URL, line: Int): Option[ConfigOrigin] =
    Some(ConfigOriginFactory.newURL(url).withLineNumber(line))

  val emptyConfigOrigin: Option[ConfigOrigin] =
    Some(ConfigOriginFactory.newSimple())
} 

package pureconfig

import java.net.URL

import com.typesafe.config.{ ConfigFactory, ConfigValueType }
import org.scalatest.{ EitherValues, Inside }
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
import pureconfig.error._
import pureconfig.generic.auto._


class ConfigReaderFailureOriginSuite extends BaseSuite with EitherValues with Inside {
  "Loading configuration from files" should "show proper error locations when loading a single file" in {
    import pureconfig.syntax._
    case class Conf(a: Int, b: String, c: Int)

    val workingDir = getClass.getResource("/").getFile
    val file = "conf/configFailureOrigin/single/a.conf"
    val conf = ConfigFactory.load(file).root()

    inside(conf.get("conf").to[Conf].left.value.toList) {
      case List(
        ConvertFailure(
          KeyNotFound("a", _),
          Some(origin1),
          ""),
        ConvertFailure(
          WrongType(ConfigValueType.STRING, valueTypes),
          Some(origin2),
          "c")
        ) =>
        origin1.filename() should endWith(file)
        origin1.url() shouldBe new URL("file", "", workingDir + file)
        origin1.lineNumber() shouldBe 1

        origin2.filename() should endWith(file)
        origin2.url() shouldBe new URL("file", "", workingDir + file)
        origin2.lineNumber() shouldBe 3
        valueTypes should contain only ConfigValueType.NUMBER

    }

    inside(conf.get("other-conf").to[Conf].left.value.toList) {
      case List(
        ConvertFailure(
          KeyNotFound("a", _),
          Some(origin1),
          ""),
        ConvertFailure(
          KeyNotFound("b", _),
          Some(origin2),
          ""),
        ConvertFailure(
          WrongType(ConfigValueType.STRING, valueTypes2),
          Some(origin3),
          "c")
        ) =>
        origin1.filename() should endWith(file)
        origin1.url shouldBe new URL("file", "", workingDir + file)
        origin1.lineNumber shouldBe 7

        origin2.filename() should endWith(file)
        origin2.url shouldBe new URL("file", "", workingDir + file)
        origin2.lineNumber shouldBe 7

        origin3.filename() should endWith(file)
        origin3.url shouldBe new URL("file", "", workingDir + file)
        origin3.lineNumber shouldBe 9
        valueTypes2 should contain only ConfigValueType.NUMBER
    }
  }

  it should "show proper error location when loading from multiple files" in {
    import pureconfig.syntax._
    case class Conf(a: Int, b: String, c: Int)

    val workingDir = getClass.getResource("/").getFile
    val file1 = "conf/configFailureOrigin/multiple/a.conf"
    val file2 = "conf/configFailureOrigin/multiple/b.conf"
    val conf = ConfigFactory.load(file1).withFallback(ConfigFactory.load(file2)).root()

    inside(conf.get("conf").to[Conf].left.value.toList) {
      case List(ConvertFailure(
        WrongType(ConfigValueType.STRING, valueTypes),
        Some(origin),
        "a")) =>
        valueTypes should contain only ConfigValueType.NUMBER
        origin.url() shouldBe new URL("file", "", workingDir + file2)
        origin.lineNumber() shouldBe 2

    }
  }
} 

package coursier.cache.protocol

import java.io.File
import java.net.{URL, URLConnection, URLStreamHandler, URLStreamHandlerFactory}

import coursier.test.HandmadeMetadata

class TestprotocolHandler extends URLStreamHandlerFactory {

  def createURLStreamHandler(protocol: String): URLStreamHandler = new URLStreamHandler {
    protected def openConnection(url: URL): URLConnection = {
      val f = new File(HandmadeMetadata.repoBase, "http/abc.com" + url.getPath)
      val resURLOpt = Option(f.toURI.toURL)

      resURLOpt match {
        case None =>
          new URLConnection(url) {
            def connect() = throw new NoSuchElementException(f.getAbsolutePath)
          }
        case Some(resURL) =>
          resURL.openConnection()
      }
    }
  }
}

object TestprotocolHandler {
  val protocol = "testprotocol"

  // get this namespace via a macro?
  val expectedClassName = s"coursier.cache.protocol.${protocol.capitalize}Handler"
  assert(classOf[TestprotocolHandler].getName == expectedClassName)
} 

package coursier.cli.resolve

import java.net.{URL, URLDecoder}

import cats.data.{Validated, ValidatedNel}
import cats.implicits._
import coursier.core.{Configuration, Dependency, Exclusions, Module, ModuleName, Organization}
import coursier.parse.{DependencyParser, JavaOrScalaDependency, JavaOrScalaModule}

object Dependencies {


  
  def handleDependencies(
    rawDependencies: Seq[String]
  ): ValidatedNel[String, List[(JavaOrScalaDependency, Map[String, String])]] =
    rawDependencies
      .map { s =>
        DependencyParser.javaOrScalaDependencyParams(s) match {
          case Left(error) => Validated.invalidNel(error)
          case Right(d) => Validated.validNel(List(d))
        }
      }
      .toList
      .flatSequence

  def withExtraRepo(
    rawDependencies: Seq[String],
    extraDependencies: Seq[(JavaOrScalaDependency, Map[String, String])]
  ): Either[Throwable, (List[JavaOrScalaDependency], Map[(JavaOrScalaModule, String), URL])] =
    handleDependencies(rawDependencies) match {
      case Validated.Valid(l) =>

        val l0 = l ++ extraDependencies

        val deps = l0.map(_._1)

        val extraRepo =
          // Any dependencies with URIs should not be resolved with a pom so this is a
          // hack to add all the deps with URIs to the FallbackDependenciesRepository
          // which will be used during the resolve
          l0.flatMap {
            case (dep, extraParams) =>
              extraParams.get("url").map { url =>
                (dep.module, dep.version) -> new URL(URLDecoder.decode(url, "UTF-8"))
              }
          }.toMap

        Right((deps, extraRepo))

      case Validated.Invalid(err) =>
        Left(new ResolveException(
          "Error processing dependencies:\n" +
            err.toList.map("  " + _).mkString("\n")
        ))
    }

  def addExclusions(
    dep: Dependency,
    perModuleExclude: Map[Module, Set[Module]],
  ): Dependency =
    perModuleExclude.get(dep.module) match {
      case None => dep
      case Some(exclusions) =>
        dep.withExclusions(
          Exclusions.minimize(dep.exclusions ++ exclusions.map(m => (m.organization, m.name)))
        )
    }

  def addExclusions(
    deps: Seq[Dependency],
    perModuleExclude: Map[Module, Set[Module]],
  ): Seq[Dependency] =
    deps.map { dep =>
      addExclusions(dep, perModuleExclude)
    }

} 

package org.example.books.entities

import scala.util.{Failure, Success, Try}
import java.net.URL

case class Book(id: Long,
                title: String,
                authors: List[String],
                imageUrl: Option[URL]) {

  def toPrettyString: String =
    s"[$id] $title ${authors.mkString("{", ", ", "}")}"
}

object Book {

  def parse(row: Map[String, String]): Try[Book] =
    for {
      id <- parseLong(row, "goodreads_book_id")
      title <- parseString(row, "title")
      authors <- parseStrings(row, "authors")
    } yield {
      // optional fields
      val imageUrl = parseURL(row, "image_url").toOption

      Book(id, title, authors, imageUrl)
    }

  private def parseLong(row: Map[String, String],
                        key: String): Try[Long] =
    parseAs(row, key, _.toLong)

  private def parseString(row: Map[String, String],
                          key: String): Try[String] =
    parseAs(row, key, x => x)

  private def parseStrings(row: Map[String, String],
                           key: String): Try[List[String]] =
    parseAs(row, key, _.split(",").map(_.trim).toList)

  private def parseURL(row: Map[String, String],
                       key: String): Try[URL] =
    parseAs(row, key, s => new URL(s))

  private def parseAs[T](row: Map[String, String],
                         key: String, parser: String => T): Try[T] =
    for {
      value <- getValue(row, key)
      t <- Try(parser(value))
    } yield t

  private def getValue(row: Map[String, String],
                       key: String): Try[String] =
    row.get(key) match {
      case Some(value) => Success(value)
      case None => Failure(new IllegalArgumentException(
        s"Couldn't find column $key in row - row was $row"))
    }

} 

package better.files

import java.net.{URL, URLClassLoader}

import better.files.test_pkg.ResourceSpecHelper

final class ResourceSpec extends CommonSpec {
  implicit val charset = java.nio.charset.StandardCharsets.US_ASCII
  val testFileText     = "This is the test-file.txt file."
  val altTestFileText  = "This is the another-test-file.txt file."
  val testFile         = "better/files/test-file.txt"
  val testFileRel      = "test-file.txt"
  val testFileAltRel   = "another-test-file.txt"
  val testFileFromCL   = "files/test-file.txt"

  "Resource" can "look up from the context class loader" in {
    assert(Resource.asStream(testFile).get.asString() startsWith testFileText)
  }

  it can "look up from a specified class loader" in {
    val clURL = new URL(Resource.my.getUrl("ResourceSpec.class"), "../")
    assert(clURL.toExternalForm endsWith "/")
    assert(Resource.from(new URLClassLoader(Array(clURL))).getAsString(testFileFromCL) startsWith testFileText)
  }

  it can "look up from the call site" in {
    assert(Resource.my.asStream(testFileRel).get.asString() startsWith testFileText)
    // This tests that Resource.my uses the correct call site when called from outside the better.files package.
    assert((new ResourceSpecHelper).openTestStream().asString() startsWith altTestFileText)
  }

  it can "look up from a statically-known type" in {
    assert(Resource.at[ResourceSpec].getAsString(testFileRel) startsWith testFileText)
    assert(Resource.at[Resource.type].getAsString(testFileRel) startsWith testFileText)
  }

  it can "look up from a java.lang.Class" in {
    assert(Resource.at(Class.forName("better.files.File")).getAsString(testFileRel) startsWith testFileText)
  }

  it can "look up a file in another package" in {
    assert(Resource.at[ResourceSpecHelper].getAsString(testFileAltRel) startsWith altTestFileText)
  }

  it should "require a concrete type" in {
    """def foo[T] = better.files.Resource.at[T].asStream("foo")""" shouldNot typeCheck
  }

  it should "fetch root url" in {
    assert(Option(Resource.getUrl()).isDefined)
  }

  it should "work with using util" in {
    File.usingTemporaryFile() { file =>
      file.appendText("hello world")
      val lines = using(file.newInputStream) { is =>
        is.lines.toList
      }
      assert(lines === "hello world" :: Nil)
    }
  }

  it should "close multiple resources" in {
    def emit(dir: File, partitions: Int, lines: Int) = {
      for {
        writers <- Vector.tabulate(partitions)(i => (dir / s"partition-$i.csv").newPrintWriter()).autoClosed
        line    <- 1 to lines
      } writers(line % partitions).println(line)
    }

    File.usingTemporaryDirectory() { dir =>
      val lines = 1000
      emit(dir = dir, partitions = 5, lines = lines)
      val expected = dir.list(filter = _.extension.contains(".csv")).flatMap(_.lines).map(_.toInt).toSet
      assert((1 to lines).forall(expected))
    }
  }
} 

package cool.graph.system.mutactions.internal.validations

import java.net.{MalformedURLException, URL}

import cool.graph.shared.errors.{UserAPIErrors, UserInputErrors}

object URLValidation {
  def getAndValidateURL(functionName: String, input: Option[String]): String = {
    input match {
      case None =>
        throw UserAPIErrors.InvalidValue("Url")

      case Some(url) =>
        try {
          val trimmedString = url.trim
          new URL(trimmedString)
          trimmedString
        } catch {
          case _: MalformedURLException => throw UserInputErrors.FunctionHasInvalidUrl(functionName, url)
        }
    }
  }
} 

package com.programmaticallyspeaking.ncd.infra

import java.io.File
import java.net.{URI, URL}


final class ScriptURL private[infra](private val uri: URI) {
  import ScriptURL._

  def toFile: File = new File(uri)

  def isFile: Boolean = uri.getScheme == "file"

  override def equals(other: Any): Boolean = other match {
    case that: ScriptURL if isFile && that.isFile => toFile == that.toFile
    case that: ScriptURL => uri == that.uri
    case _ => false
  }

  override def hashCode(): Int = {
    if (isFile) toFile.hashCode() else uri.hashCode()
  }

  override def toString: String = uri.toString

  def resolve(pathLike: String): ScriptURL = {
    if (looksLikeRelativePath(pathLike))
      new ScriptURL(uri.resolve(pathLike))
    else
      ScriptURL.create(pathLike)
  }
}

object ScriptURL {
  private[ScriptURL] def looksLikeRelativePath(x: String) =
    x.length > 0 && x(0) != '/' && !x.lift(1).contains(':')

  private def isAbsoluteUnixOrWindowsFilePath(x: String) =
    x.startsWith("/") || (x.lift(1).contains(':') && x.indexWhere(c => c == '\\' || c == '/') > 1)

  def create(url: URL): ScriptURL = new ScriptURL(url.toURI)

  def create(something: String): ScriptURL = {
    val uri = if (isAbsoluteUnixOrWindowsFilePath(something)) {
      val withUnixSlashes = something.replace("\\", "/")
      val uriPart = if (withUnixSlashes.startsWith("/")) withUnixSlashes else "/" + withUnixSlashes

      new URI("file", "", uriPart, null)
    } else if (something.startsWith("jar:")) {
      // Just drop the 'jar:' prefix. We keep the ! character so that the JAR file itself becomes
      // sort of a special folder.
      return create(something.substring(4))
    } else if (something.startsWith("file:") || something.startsWith("eval:")) {
      // Assume this is something resembling an URL already, e.g. file:/foo/bar,
      // but we don't know how many slashes there are.
      var (scheme, rest) = something.span(_ != ':')
      rest = rest.substring(1) // skip the leading :
      val slashCount = rest.prefixLength(_ == '/')
      new URI(scheme, "", "/" + rest.substring(slashCount), null)
    } else if (something.contains("..")) {
      throw new IllegalArgumentException(s"Cannot create ScriptURL from path/URL with '..' ($something)")
    } else {
      val withUnixSlashes = something.replace("\\", "/")
      new URI(withUnixSlashes)
    }
    normalized(new ScriptURL(uri))
  }

  private def normalized(url: ScriptURL): ScriptURL = {
    val normalized = url.uri.normalize()
    if (url.uri.getPath != normalized.getPath) {
      // normalization was necessary
      create(normalized.toString)
    } else url // no normalization necessary
  }
}

object FileScriptURL {
  def unapply(x: Any): Option[File] = x match {
    case s: ScriptURL if s.isFile => Some(s.toFile)
    case _ => None
  }
} 

package com.programmaticallyspeaking.ncd.testing

import java.io.File
import java.net.URL

import com.programmaticallyspeaking.ncd.chrome.net.FilePublisher
import com.programmaticallyspeaking.ncd.host._
import com.programmaticallyspeaking.ncd.messaging.{Observable, SerializedSubject}

import scala.concurrent.Future
import scala.concurrent.duration.FiniteDuration
import scala.util.Try

object FakeFilePublisher extends FilePublisher {
  override def publish(file: File): URL = new URL("http://localhost/no/such/file")
}

object FakeScriptHost extends ScriptHost {
  val eventSubject = new SerializedSubject[ScriptEvent]

  override def evaluateOnStackFrame(stackFrameId: String, expression: String): Try[ValueNode] = Try(notImpl)
  override def removeBreakpointById(id: String): Unit = notImpl
  override def resume(): Unit = notImpl
  override def reset(): Unit = notImpl
  override def findScript(id: ScriptIdentity): Option[Script] = None
  override def events: Observable[ScriptEvent] = eventSubject
  override def scripts: Seq[Script] = Seq.empty
  override def setBreakpoint(id: ScriptIdentity, location: ScriptLocation, options: BreakpointOptions): Breakpoint = notImpl
  override def getBreakpointLocations(id: ScriptIdentity, from: ScriptLocation, to: Option[ScriptLocation]): Seq[ScriptLocation] = Seq.empty
  override def step(stepType: StepType): Unit = notImpl
  override def pauseOnBreakpoints(): Unit = notImpl
  override def ignoreBreakpoints(): Unit = notImpl
  override def getObjectProperties(objectId: ObjectId, onlyOwn: Boolean, onlyAccessors: Boolean): Seq[(String, types.ObjectPropertyDescriptor)] = Seq.empty
  override def pauseOnExceptions(pauseType: ExceptionPauseType): Unit = notImpl
  
  private def notImpl: Nothing = throw new UnsupportedOperationException("FakeScriptHost is not complete")

  override def restartStackFrame(stackFrameId: String): Seq[StackFrame] = notImpl

  override def startProfiling(samplingInterval: FiniteDuration): Unit = notImpl
  override def stopProfiling(): ProfilingData = notImpl

  override def pauseAtNextStatement(): Unit = notImpl

  override def setSkipAllPauses(skip: Boolean): Unit = notImpl

  override def compileScript(script: String, url: String, persist: Boolean): Future[Option[Script]] = Future.failed(notImpl)

  override def runCompiledScript(scriptId: String): Try[ValueNode] = notImpl

  override def warnings: Seq[String] = Seq.empty

  override def callFunctionOn(stackFrameId: String, thisObject: Option[ObjectId], functionDeclaration: String, arguments: Seq[ObjectId]): Try[ValueNode] = Try(notImpl)
} 

package edu.neu.coe.csye7200


object Map2 {

  //	def map2[A,B,C](a: Option[A], b: Option[B])(f: (A, B) => C): Option[C] =
  //	  a match {
  //			case Some(aa) => b match {
  //			case Some(bb) => Some(f(aa,bb))
  //			case _ => None
  //		}
  //		case _ => None
  //	}

  def map[B, C](b: Option[B])(f: (B) => C): Option[C] =
    b match {
      case Some(bb) => Some(f(bb))
      case _ => None
    }

  def flatMap[A, B](a: Option[A])(f: (A) => Option[B]): Option[B] =
    a match {
      case Some(aa) => f(aa)
      case _ => None
    }

  def map2b[A, B, C](a: Option[A], b: Option[B])(f: (A, B) => C): Option[C] =
    flatMap(a)(aa => map(b)(bb => f(aa, bb)))

  def map2a[A, B, C](a: Option[A], b: Option[B])(f: (A, B) => C): Option[C] =
    a flatMap (aa => b map (bb => f(aa, bb)))

  def map2[A, B, C](a: Option[A], b: Option[B])(f: (A, B) => C): Option[C] =
    for {
      aa <- a
      bb <- b
    } yield f(aa, bb)

  def map3[A, B, C, D](a: Option[A], b: Option[B], c: Option[C])(f: (A, B, C) => D): Option[D] =
    for {
      aa <- a
      bb <- b
      cc <- c
    } yield f(aa, bb, cc)

  def map2right[A, B, C, S](a: Either[S, A], b: Either[S, B])(f: (A, B) => C): Either[S, C] =
    for {
      aa <- a.right
      bb <- b.right
    } yield f(aa, bb)

  def map2leftRight[A, B, C, S](a: Either[S, A], b: Either[S, B])(fr: (A, B) => C)(fl: (S, S) => S): Either[S, C] =
    (a, b) match {
      case (Left(aa), Left(bb)) => Left(fl(aa, bb))
      case _ => map2right(a, b)(fr)
    }
}

object ReadURL {

  import java.net.URL

  import scala.io.Source
  import scala.util._

  def getURLContent(url: String): Try[Iterator[String]] =
    for {
      url <- Try(new URL(url))
      connection <- Try(url.openConnection())
      is <- Try(connection.getInputStream)
      source = Source.fromInputStream(is)
    } yield source.getLines()

  def wget(args: Array[String]): Unit = {
    val maybePages = for {
      arg <- args
      x = getURLContent(arg)
    } yield x
    for {
      Success(p) <- maybePages
      l <- p
    } println(l)
  }

  def main(args: Array[String]): Unit = {
    println(s"web reader: ${args.toList}")
    wget(args)

    val ok: Either[String, Int] = Right(1)
    val notOk: Either[String, Int] = Left("bad")
    val result = Map2.map2right(ok, notOk) {
      _ + _
    }
    print(result)
  }
} 

package edu.neu.coe.csye7200.cache

import java.net.URL

import org.scalatest.concurrent.{Futures, ScalaFutures}
import org.scalatest.{FlatSpec, Matchers}

import scala.concurrent.Future
import scala.concurrent.ExecutionContext.Implicits.global
import scala.util.{Random, Try}

class CacheSpec extends FlatSpec with Matchers with Futures with ScalaFutures {

  behavior of "apply"

  val random = Random

  def lookupStock(k: String): Future[Double] = Future {
    random.setSeed(k.hashCode)
    random.nextInt(1000) / 100.0
  }

  it should "work" in {
    val cache = MyCache[String,Double](lookupStock)
    val xf: Future[Double] = cache("MSFT")
    whenReady(xf) { u => u should matchPattern { case x: Double =>  } }
    xf.value.get.get shouldBe 3.64
  }
} 

package edu.neu.coe.csye7200.hedge_fund.rss

import java.net.URL
import scala.language.postfixOps
import java.util.Date

case class RssUrl(url: URL) {
  override def toString = "RSS: " + url.toString
}

trait RssFeed {
  val link: String
  val title: String
  val desc: String
  val items: Seq[RssItem]
  override def toString = title + "\n" + desc + "\n**"

  def latest = items sortWith ((a, b) => a.date.compareTo(b.date) > 0) head
}

case class AtomRssFeed(title: String, link: String, desc: String, items: Seq[RssItem]) extends RssFeed
case class XmlRssFeed(title: String, link: String, desc: String, language: String, items: Seq[RssItem]) extends RssFeed

case class RssItem(title: String, link: String, desc: String, date: Date, guid: String) {
  override def toString = date + " " + title
} 

package edu.neu.coe.csye7200.cache

import java.net.URL

import org.scalatest.concurrent.{Futures, ScalaFutures}
import org.scalatest.{FlatSpec, Matchers}

import scala.concurrent.Future
import scala.concurrent.ExecutionContext.Implicits.global
import scala.util.{Random, Try}

class CacheSpec extends FlatSpec with Matchers with Futures with ScalaFutures {

  behavior of "apply"

  val random = Random

  def lookupStock(k: String): Future[Double] = Future {
    random.setSeed(k.hashCode)
    random.nextInt(1000) / 100.0
  }

  it should "work" in {
    val cache = MyCache[String,Double](lookupStock)
    val xf: Future[Double] = cache("MSFT")
    whenReady(xf) { u => u should matchPattern { case x: Double =>  } }
    xf.value.get.get shouldBe 3.64
  }
} 

package edu.neu.coe.csye7200.asstwc

import java.net.URL

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent._
import scala.concurrent.duration._
import scala.io.Source
import scala.language.postfixOps
import scala.util._
import scala.xml.Node


object WebCrawler extends App {

  def getURLContent(u: URL): Future[String] = {
    for {
      source <- Future(Source.fromURL(u))
    } yield source mkString
  }

  def wget(u: URL): Future[Seq[URL]] = {
    // TO BE IMPLEMENTED implement. 16 points. Hint: write as a for-comprehension, using the constructor new URL(URL,String) to get the appropriate URL for relative links
    def getURLs(ns: Node): Seq[URL] = ???
    def getLinks(g: String): Try[Seq[URL]] =
      for (n <- HTMLParser.parse(g) recoverWith { case f => Failure(new RuntimeException(s"parse problem with URL $u: $f")) })
        yield getURLs(n)
    // TO BE IMPLEMENTED implement. 9 points. Hint: write as a for-comprehension, using getURLContent (above) and getLinks above. You might also need MonadOps.asFuture
    ???
  }

  def wget(us: Seq[URL]): Future[Seq[Either[Throwable, Seq[URL]]]] = {
    val us2 = us.distinct take 10
    // TO BE IMPLEMENTED implement the rest of this, based on us2 instead of us. 15 points.
    // Hint: Use wget(URL) (above). MonadOps.sequence and Future.sequence are also available to you to use.
    ???
  }

  def crawler(depth: Int, args: Seq[URL]): Future[Seq[URL]] = {
    def inner(urls: Seq[URL], depth: Int, accum: Seq[URL]): Future[Seq[URL]] =
      if (depth > 0)
        for (us <- MonadOps.flattenRecover(wget(urls), { x => System.err.println(x) }); r <- inner(us, depth - 1, accum ++: urls)) yield r
      else
        Future.successful(accum)
    inner(args, depth, Nil)
  }

  println(s"web reader: ${args.toList}")
  val urls = for (arg <- args toList) yield Try(new URL(arg))
  val s = MonadOps.sequence(urls)
  s match {
    case Success(z) =>
      println(s"invoking crawler on $z")
      val f = crawler(2, z)
      Await.ready(f, Duration("60 second"))
      for (x <- f) println(s"Links: $x")
    case Failure(z) => println(s"failure: $z")
  }
} 

// Copyright (C) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See LICENSE in project root for information.

package com.microsoft.ml.spark.cognitive.split2

import java.net.{URI, URL}

import com.microsoft.ml.spark.Secrets
import com.microsoft.ml.spark.cognitive.{SpeechResponse, SpeechToText}
import com.microsoft.ml.spark.core.test.fuzzing.{TestObject, TransformerFuzzing}
import org.apache.commons.compress.utils.IOUtils
import org.apache.spark.ml.util.MLReadable
import org.apache.spark.sql.{DataFrame, Row}
import org.scalactic.Equality

trait SpeechKey {
  lazy val speechKey = sys.env.getOrElse("SPEECH_API_KEY", Secrets.SpeechApiKey)
}

class SpeechToTextSuite extends TransformerFuzzing[SpeechToText]
  with SpeechKey {

  import session.implicits._

  val region = "eastus"
  val resourcesDir = System.getProperty("user.dir") + "/src/test/resources/"
  val uri = new URI(s"https://$region.api.cognitive.microsoft.com/sts/v1.0/issuetoken")
  val language = "en-us"
  val profanity = "masked"
  val format = "simple"

  lazy val stt = new SpeechToText()
    .setSubscriptionKey(speechKey)
    .setLocation(region)
    .setOutputCol("text")
    .setAudioDataCol("audio")
    .setLanguage("en-US")

  lazy val audioBytes: Array[Byte] = {
    IOUtils.toByteArray(new URL("https://mmlspark.blob.core.windows.net/datasets/Speech/test1.wav").openStream())
  }

  lazy val df: DataFrame = Seq(
    Tuple1(audioBytes)
  ).toDF("audio")

  override lazy val dfEq = new Equality[DataFrame] {
    override def areEqual(a: DataFrame, b: Any): Boolean =
      baseDfEq.areEqual(a.drop("audio"), b.asInstanceOf[DataFrame].drop("audio"))
  }

  override def testSerialization(): Unit = {
    tryWithRetries(Array(0, 100, 100, 100, 100))(super.testSerialization)
  }

  
  def jaccardSimilarity(s1: String, s2: String): Double = {
    val a = Set(s1)
    val b = Set(s2)
    a.intersect(b).size.toDouble / (a | b).size.toDouble
  }

  test("Basic Usage") {
    val toObj: Row => SpeechResponse = SpeechResponse.makeFromRowConverter
    val result = toObj(stt.setFormat("simple")
      .transform(df).select("text")
      .collect().head.getStruct(0))
    result.DisplayText.get.contains("this is a test")
  }

  test("Detailed Usage") {
    val toObj = SpeechResponse.makeFromRowConverter
    val result = toObj(stt.setFormat("detailed")
      .transform(df).select("text")
      .collect().head.getStruct(0))
    result.NBest.get.head.Display.contains("this is a test")
  }

  override def testObjects(): Seq[TestObject[SpeechToText]] =
    Seq(new TestObject(stt, df))

  override def reader: MLReadable[_] = SpeechToText
} 

package uk.gov.hmrc.audit.handler

import java.net.URL

import akka.stream.Materializer
import org.slf4j.{Logger, LoggerFactory}
import play.api.inject.ApplicationLifecycle
import play.api.libs.json.JsValue
import uk.gov.hmrc.audit.HandlerResult
import uk.gov.hmrc.audit.HandlerResult.{Failure, Rejected, Success}

import scala.concurrent.{ExecutionContext, Future}
import scala.concurrent.duration.Duration

class DatastreamHandler(
  scheme        : String,
  host          : String,
  port          : Integer,
  path          : String,
  connectTimeout: Duration,
  requestTimeout: Duration,
  userAgent     : String,
  materializer  : Materializer,
  lifecycle     : ApplicationLifecycle
) extends HttpHandler(
  endpointUrl    = new URL(s"$scheme://$host:$port$path"),
  userAgent      = userAgent,
  connectTimeout = connectTimeout,
  requestTimeout = requestTimeout,
  materializer   = materializer,
  lifecycle      = lifecycle
) with AuditHandler {

  private val logger: Logger = LoggerFactory.getLogger(getClass)

  override def sendEvent(event: JsValue)(implicit ec: ExecutionContext): Future[HandlerResult] =
    sendEvent(event, retryIfMalformed = true)

  private def sendEvent(event: JsValue, retryIfMalformed: Boolean)(implicit ec: ExecutionContext): Future[HandlerResult] =
    sendHttpRequest(event).flatMap {
      case HttpResult.Response(status) =>
        Future.successful(status match {
          case 204 => Success
          case 400 => logger.warn("Malformed request rejected by Datastream")
                      Rejected
          case 413 => logger.warn("Too large request rejected by Datastream")
                      Rejected
          case _   => logger.error(s"Unknown return value $status")
                      Failure
        })
      case HttpResult.Malformed =>
        if (retryIfMalformed) {
          logger.warn("Malformed response on first request, retrying")
          sendEvent(event, retryIfMalformed = false)
        } else {
          logger.warn("Malformed response on second request, failing")
          Future.successful(Failure)
        }
      case HttpResult.Failure(msg, exceptionOption) =>
        exceptionOption match {
          case None     => logger.error(msg)
          case Some(ex) => logger.error(msg, ex)
        }
        Future.successful(Failure)
    }
} 

package uk.gov.hmrc.audit.handler

import java.io.IOException
import java.net.URL
import java.util.concurrent.TimeoutException

import akka.stream.Materializer
import org.slf4j.{Logger, LoggerFactory}
import play.api.inject.ApplicationLifecycle
import play.api.libs.json.JsValue

import scala.concurrent.{ExecutionContext, Future}
import scala.concurrent.duration.Duration


sealed trait HttpResult
object HttpResult {
  case class Response(statusCode: Int) extends HttpResult
  case object Malformed extends HttpResult
  case class Failure(msg: String, nested: Option[Throwable] = None) extends Exception(msg, nested.orNull) with HttpResult
}

abstract class HttpHandler(
  endpointUrl      : URL,
  userAgent        : String,
  connectTimeout   : Duration,
  requestTimeout   : Duration,
  materializer     : Materializer,
  lifecycle        : ApplicationLifecycle
) {
  private val logger: Logger = LoggerFactory.getLogger(getClass)

  val HTTP_STATUS_CONTINUE = 100

  val wsClient: WSClient = {
    implicit val m = materializer
    val wsClient = WSClient(connectTimeout, requestTimeout, userAgent)
    lifecycle.addStopHook { () =>
      logger.info("Closing play-auditing http connections...")
      wsClient.close()
      Future.successful(())
    }
    wsClient
  }

  def sendHttpRequest(event: JsValue)(implicit ec: ExecutionContext): Future[HttpResult] =
    try {
      logger.debug(s"Sending audit request to URL ${endpointUrl.toString}")

      wsClient.url(endpointUrl.toString)
        .post(event)
        .map { response =>
          val httpStatusCode = response.status
          logger.debug(s"Got status code : $httpStatusCode")
          response.body
          logger.debug("Response processed and closed")

          if (httpStatusCode >= HTTP_STATUS_CONTINUE) {
            logger.info(s"Got status code $httpStatusCode from HTTP server.")
            HttpResult.Response(httpStatusCode)
          } else {
            logger.warn(s"Malformed response (status $httpStatusCode) returned from server")
            HttpResult.Malformed
          }
        }.recover {
          case e: TimeoutException =>
            HttpResult.Failure("Error opening connection, or request timed out", Some(e))
          case e: IOException =>
            HttpResult.Failure("Error opening connection, or request timed out", Some(e))
        }
    } catch {
      case t: Throwable =>
        Future.successful(HttpResult.Failure("Error sending HTTP request", Some(t)))
    }
} 

package org.argus.jawa.core.compiler.compile.io

import java.io.{Closeable, FileInputStream, FileOutputStream, InputStream, OutputStream, File => JavaFile}
import java.io.{BufferedInputStream, BufferedOutputStream, InputStreamReader, OutputStreamWriter}
import java.io.{BufferedReader, BufferedWriter}
import java.util.zip.GZIPInputStream
import java.net.URL
import java.nio.channels.FileChannel
import java.nio.charset.Charset
import java.util.jar.{JarFile, JarInputStream, JarOutputStream}
import java.util.zip.{GZIPOutputStream, ZipEntry, ZipFile, ZipInputStream, ZipOutputStream}

import ErrorHandling.translate

import scala.reflect.{Manifest => SManifest}

abstract class Using[Source, T]
{
  protected def open(src: Source): T
  def apply[R](src: Source)(f: T => R): R =
  {
    val resource = open(src)
    try { f(resource) }
    finally { close(resource) }
  }
  protected def close(out: T): Unit
}
abstract class WrapUsing[Source, T](implicit srcMf: SManifest[Source], targetMf: SManifest[T]) extends Using[Source, T]
{
  protected def label[S](m: SManifest[S]): String = m.runtimeClass.getSimpleName
  protected def openImpl(source: Source): T
  protected final def open(source: Source): T =
    translate("Error wrapping " + label(srcMf) + " in " + label(targetMf) + ": ") { openImpl(source) }
}
trait OpenFile[T] extends Using[JavaFile, T]
{
  protected def openImpl(file: JavaFile): T
  protected final def open(file: JavaFile): T =
  {
    val parent = file.getParentFile
    if(parent != null)
      IO.createDirectory(parent)
    openImpl(file)
  }
}
object Using
{
  def wrap[Source, T<: Closeable](openF: Source => T)(implicit srcMf: SManifest[Source], targetMf: SManifest[T]): Using[Source,T] =
    wrap(openF, closeCloseable)
  def wrap[Source, T](openF: Source => T, closeF: T => Unit)(implicit srcMf: SManifest[Source], targetMf: SManifest[T]): Using[Source,T] =
    new WrapUsing[Source, T]
    {
      def openImpl(source: Source): T = openF(source)
      def close(t: T): Unit = closeF(t)
    }

  def resource[Source, T <: Closeable](openF: Source => T): Using[Source,T] =
    resource(openF, closeCloseable)
  def resource[Source, T](openF: Source => T, closeF: T => Unit): Using[Source,T] =
    new Using[Source,T]
    {
      def open(s: Source): T = openF(s)
      def close(s: T): Unit = closeF(s)
    }
  def file[T <: Closeable](openF: JavaFile => T): OpenFile[T] = file(openF, closeCloseable)
  def file[T](openF: JavaFile => T, closeF: T => Unit): OpenFile[T] =
    new OpenFile[T]
    {
      def openImpl(file: JavaFile): T = openF(file)
      def close(t: T): Unit = closeF(t)
    }
  private def closeCloseable[T <: Closeable]: T => Unit = _.close()

  def bufferedOutputStream: Using[OutputStream, BufferedOutputStream] = wrap((out: OutputStream) => new BufferedOutputStream(out) )
  def bufferedInputStream: Using[InputStream, BufferedInputStream] = wrap((in: InputStream) => new BufferedInputStream(in) )
  def fileOutputStream(append: Boolean = false): OpenFile[BufferedOutputStream] = file(f => new BufferedOutputStream(new FileOutputStream(f, append)))
  def fileInputStream: OpenFile[BufferedInputStream] = file(f => new BufferedInputStream(new FileInputStream(f)))
  def urlInputStream: Using[URL, BufferedInputStream] = resource((u: URL) => translate("Error opening " + u + ": ")(new BufferedInputStream(u.openStream)))
  def fileOutputChannel: OpenFile[FileChannel] = file(f => new FileOutputStream(f).getChannel)
  def fileInputChannel: OpenFile[FileChannel] = file(f => new FileInputStream(f).getChannel)
  def fileWriter(charset: Charset = IO.utf8, append: Boolean = false): OpenFile[BufferedWriter] =
    file(f => new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f, append), charset)) )
  def fileReader(charset: Charset): OpenFile[BufferedReader] = file(f => new BufferedReader(new InputStreamReader(new FileInputStream(f), charset)) )
  def urlReader(charset: Charset): Using[URL, BufferedReader] = resource((u: URL) => new BufferedReader(new InputStreamReader(u.openStream, charset)))
  def jarFile(verify: Boolean): OpenFile[JarFile] = file(f => new JarFile(f, verify), (_: JarFile).close())
  def zipFile: OpenFile[ZipFile] = file(f => new ZipFile(f), (_: ZipFile).close())
  def streamReader: Using[(InputStream, Charset), InputStreamReader] = wrap{ (_: (InputStream, Charset)) match { case (in, charset) => new InputStreamReader(in, charset) } }
  def gzipInputStream: Using[InputStream, GZIPInputStream] = wrap((in: InputStream) => new GZIPInputStream(in, 8192) )
  def zipInputStream: Using[InputStream, ZipInputStream] = wrap((in: InputStream) => new ZipInputStream(in))
  def zipOutputStream: Using[OutputStream, ZipOutputStream] = wrap((out: OutputStream) => new ZipOutputStream(out))
  def gzipOutputStream: Using[OutputStream, GZIPOutputStream] = wrap((out: OutputStream) => new GZIPOutputStream(out, 8192), (_: GZIPOutputStream).finish())
  def jarOutputStream: Using[OutputStream, JarOutputStream] = wrap((out: OutputStream) => new JarOutputStream(out))
  def jarInputStream: Using[InputStream, JarInputStream] = wrap((in: InputStream) => new JarInputStream(in))
  def zipEntry(zip: ZipFile): Using[ZipEntry, InputStream] = resource((entry: ZipEntry) =>
    translate("Error opening " + entry.getName + " in " + zip + ": ") { zip.getInputStream(entry) } )
} 

package org.argus.jawa.core.classpath

import java.io.File
import java.net.URL

import FileUtils.AbstractFileOps
import org.argus.jawa.core.io.{AbstractFile, FileZipArchive}


trait ZipArchiveFileLookup[FileEntryType <: ClassRepClasspathEntry] extends FlatClasspath {
  val zipFile: File

  assert(zipFile != null, "Zip file in ZipArchiveFileLookup cannot be null")

  override def asURLs: Seq[URL] = Seq(zipFile.toURI.toURL)
  override def asClasspathStrings: Seq[String] = Seq(zipFile.getPath)

  private val archive = new FileZipArchive(zipFile)

  override private[jawa] def packages(inPackage: String): Seq[PackageEntry] = {
    val prefix = PackageNameUtils.packagePrefix(inPackage)
    for {
      dirEntry <- findDirEntry(inPackage).toSeq
      entry <- dirEntry.iterator if entry.isPackage
    } yield PackageEntryImpl(prefix + entry.name)
  }

  protected def files(inPackage: String): Seq[FileEntryType] =
    for {
      dirEntry <- findDirEntry(inPackage).toSeq
      entry <- dirEntry.iterator if isRequiredFileType(entry)
    } yield createFileEntry(entry)

  override private[jawa] def list(inPackage: String): FlatClasspathEntries = {
    val foundDirEntry = findDirEntry(inPackage)

    foundDirEntry map { dirEntry =>
      val pkgBuf = collection.mutable.ArrayBuffer.empty[PackageEntry]
      val fileBuf = collection.mutable.ArrayBuffer.empty[FileEntryType]
      val prefix = PackageNameUtils.packagePrefix(inPackage)

      for (entry <- dirEntry.iterator) {
        if (entry.isPackage)
          pkgBuf += PackageEntryImpl(prefix + entry.name)
        else if (isRequiredFileType(entry))
          fileBuf += createFileEntry(entry)
      }
      FlatClasspathEntries(pkgBuf, fileBuf)
    } getOrElse FlatClasspathEntries(Seq.empty, Seq.empty)
  }

  private def findDirEntry(pkg: String) = {
    val dirName = s"${FileUtils.dirPath(pkg)}/"
    archive.allDirs.get(dirName)
  }

  protected def createFileEntry(file: FileZipArchive#Entry): FileEntryType
  protected def isRequiredFileType(file: AbstractFile): Boolean
} 

package com.dimafeng.testcontainers

import java.net.URL

import org.testcontainers.containers.{NginxContainer => JavaNginxContainer}

case class NginxContainer(
  customContent: Option[String] = None
) extends SingleContainer[JavaNginxContainer[_]] {

  override val container: JavaNginxContainer[_] = {
    val c = new JavaNginxContainer()
    customContent.foreach(c.withCustomContent)
    c
  }

  def baseUrl(scheme: String, port: Int): URL = container.getBaseUrl(scheme, port)
}

object NginxContainer {

  case class Def(
    customContent: Option[String] = None
  ) extends ContainerDef {

    override type Container = NginxContainer

    override def createContainer(): NginxContainer = {
      new NginxContainer(
        customContent
      )
    }
  }

} 

package com.dimafeng.testcontainers

import java.net.URL

import org.testcontainers.containers.{SolrContainerConfiguration, SolrContainer => JavaSolrContainer}

case class SolrContainer(
  dockerImageName: String = SolrContainer.defaultDockerImageName,
  zookeeper: Boolean = SolrContainer.defaultConfig.isZookeeper,
  collectionName: String = SolrContainer.defaultConfig.getCollectionName,
  configurationName: String = SolrContainer.defaultConfig.getConfigurationName,
  configuration: URL = SolrContainer.defaultConfig.getSolrConfiguration,
  schema: URL = SolrContainer.defaultConfig.getSolrSchema
) extends SingleContainer[JavaSolrContainer] {

  override val container: JavaSolrContainer = {
    val c = new JavaSolrContainer(dockerImageName)

    c.withZookeeper(zookeeper)
    c.withCollection(collectionName)
    c.withConfiguration(configurationName, configuration)
    c.withSchema(schema)

    c
  }

  def solrPort: Int = container.getSolrPort

  def zookeeperPort: Int = container.getZookeeperPort
}

object SolrContainer {

  val defaultImage = "solr"
  val defaultTag = "8.3.0"
  val defaultDockerImageName = s"$defaultImage:$defaultTag"
  val defaultConfig = new SolrContainerConfiguration()

  case class Def(
    dockerImageName: String = SolrContainer.defaultDockerImageName,
    zookeeper: Boolean = SolrContainer.defaultConfig.isZookeeper,
    collectionName: String = SolrContainer.defaultConfig.getCollectionName,
    configurationName: String = SolrContainer.defaultConfig.getConfigurationName,
    configuration: URL = SolrContainer.defaultConfig.getSolrConfiguration,
    schema: URL = SolrContainer.defaultConfig.getSolrSchema
  ) extends ContainerDef {

    override type Container = SolrContainer

    override def createContainer(): SolrContainer = new SolrContainer(
      dockerImageName,
      zookeeper,
      collectionName,
      configurationName,
      configuration,
      schema
    )
  }
} 

package com.dimafeng.testcontainers

import java.io.File
import java.net.URL
import java.util.Optional

import com.dimafeng.testcontainers.lifecycle.TestLifecycleAware
import org.openqa.selenium.WebDriver
import org.openqa.selenium.remote.{DesiredCapabilities, RemoteWebDriver}
import org.scalatest.Suite
import org.testcontainers.containers.BrowserWebDriverContainer
import org.testcontainers.lifecycle.TestDescription

trait SeleniumTestContainerSuite extends ForEachTestContainer {
  self: Suite =>

  def desiredCapabilities: DesiredCapabilities

  def recordingMode: (BrowserWebDriverContainer.VncRecordingMode, File) = null

  val container = SeleniumContainer(desiredCapabilities, recordingMode)

  implicit def webDriver: WebDriver = container.webDriver
}

class SeleniumContainer(desiredCapabilities: Option[DesiredCapabilities] = None,
                        recordingMode: Option[(BrowserWebDriverContainer.VncRecordingMode, File)] = None)
  extends SingleContainer[BrowserWebDriverContainer[_]] with TestLifecycleAware {
  require(desiredCapabilities.isDefined, "'desiredCapabilities' is required parameter")

  override val container: BrowserWebDriverContainer[_] = new BrowserWebDriverContainer()
  desiredCapabilities.foreach(container.withDesiredCapabilities)
  recordingMode.foreach(Function.tupled(container.withRecordingMode))

  def password: String = container.getPassword

  def port: Int = container.getPort

  def seleniumAddress: URL = container.getSeleniumAddress

  def vncAddress: String = container.getVncAddress

  def webDriver: RemoteWebDriver = container.getWebDriver

  override def afterTest(description: TestDescription, throwable: Option[Throwable]): Unit = {
    val javaThrowable: Optional[Throwable] = throwable match {
      case Some(error) => Optional.of(error)
      case None => Optional.empty()
    }
    container.afterTest(description, javaThrowable)
  }
}

object SeleniumContainer {
  def apply(desiredCapabilities: DesiredCapabilities = null, recordingMode: (BrowserWebDriverContainer.VncRecordingMode, File) = null): SeleniumContainer =
    new SeleniumContainer(Option(desiredCapabilities), Option(recordingMode))
} 

package com.dimafeng.testcontainers.integration

import java.net.URL

import com.dimafeng.testcontainers.{GenericContainer, SingleContainer}
import com.dimafeng.testcontainers.lifecycle.and
import com.dimafeng.testcontainers.scalatest.TestContainersForAll
import org.scalatest.FlatSpec
import org.testcontainers.containers.wait.strategy.Wait

import scala.io.Source

class GenericContainerDefSpec extends FlatSpec with TestContainersForAll {

  import GenericContainerDefSpec._

  override type Containers = CompatibleGenericContainer and NotCompatibleGenericContainer

  override def startContainers(): Containers = {
    val compatible = CompatibleGenericContainer.Def().start()
    val notCompatible = NotCompatibleGenericContainer.Def().start()
    compatible and notCompatible
  }

  "GenericContainer.Def" should "be able to work through compatible and not compatible constructors" in withContainers {
    case compatible and notCompatible =>
      val expectedText = "If you see this page, the nginx web server is successfully installed"
      assert(
        compatible.rootPage.contains(expectedText) &&
        notCompatible.rootPage.contains(expectedText)
      )
  }
}
object GenericContainerDefSpec {

  private val port = 80

  private def createUrl(container: SingleContainer[_]) = {
    new URL(s"http://${container.containerIpAddress}:${container.mappedPort(port)}/")
  }

  private def urlToString(url: URL) = {
    Source.fromInputStream(url.openConnection().getInputStream).mkString
  }

  class CompatibleGenericContainer extends GenericContainer(
    dockerImage = "nginx:latest",
    exposedPorts = Seq(port),
    waitStrategy = Some(Wait.forHttp("/"))
  ) {
    def rootUrl: URL = createUrl(this)
    def rootPage: String = urlToString(rootUrl)
  }
  object CompatibleGenericContainer {
    case class Def() extends GenericContainer.Def[CompatibleGenericContainer](
      new CompatibleGenericContainer()
    )
  }

  class NotCompatibleGenericContainer(underlying: GenericContainer) extends GenericContainer(underlying) {
    def rootUrl: URL = createUrl(this)
    def rootPage: String = urlToString(rootUrl)
  }
  object NotCompatibleGenericContainer {
    case class Def() extends GenericContainer.Def[NotCompatibleGenericContainer](
      new NotCompatibleGenericContainer(GenericContainer(
        dockerImage = "nginx:latest",
        exposedPorts = Seq(port),
        waitStrategy = Wait.forHttp("/")
      ))
    )
  }
} 

package com.dimafeng.testcontainers.integration

import java.net.URL

import com.dimafeng.testcontainers.{FixedHostPortGenericContainer, ForAllTestContainer}
import org.scalatest.FlatSpec
import org.testcontainers.containers.wait.Wait

import scala.io.Source

class FixedHostPortContainerSpec extends FlatSpec with ForAllTestContainer {
  override val container = FixedHostPortGenericContainer("nginx:latest",
    waitStrategy = Wait.forHttp("/"),
    exposedHostPort = 8090,
    exposedContainerPort = 80
  )

  "FixedHostPortGenericContainer" should "start nginx and expose 8090 port on host" in {
    assert(container.mappedPort(80) == 8090)
    assert(Source.fromInputStream(
      new URL(s"http://${container.containerIpAddress}:${container.mappedPort(80)}/").openConnection().getInputStream
    ).mkString.contains("If you see this page, the nginx web server is successfully installed"))
  }
} 

package com.dimafeng.testcontainers.integration

import java.net.URL

import com.dimafeng.testcontainers.{ForAllTestContainer, GenericContainer}
import org.scalatest.FlatSpec
import org.testcontainers.containers.wait.strategy.Wait
import org.testcontainers.images.builder.ImageFromDockerfile

import scala.io.Source

class GenericContainerSpec extends FlatSpec with ForAllTestContainer {
  override val container = GenericContainer("nginx:latest",
    exposedPorts = Seq(80),
    waitStrategy = Wait.forHttp("/")
  )

  "GenericContainer" should "start nginx and expose 80 port" in {
    assert(Source.fromInputStream(
      new URL(s"http://${container.containerIpAddress}:${container.mappedPort(80)}/").openConnection().getInputStream
    ).mkString.contains("If you see this page, the nginx web server is successfully installed"))
  }
}

class GenericContainerDockerFileSpec extends GenericContainerSpec {
  private val imageFromDockerfile = new ImageFromDockerfile().withFileFromClasspath("Dockerfile", "generic-container-dockerfile")
  override val container = GenericContainer(imageFromDockerfile,
    exposedPorts = Seq(80),
    waitStrategy = Wait.forHttp("/")
  )
} 

package is.hail.utils

import java.net.URL
import java.io.OutputStream
import java.io.InputStream
import java.net.HttpURLConnection
import is.hail.utils._
import java.nio.charset.StandardCharsets
import org.apache.commons.io.output.ByteArrayOutputStream


object HTTPClient {
  def post[T](
    url: String,
    contentLength: Int,
    writeBody: OutputStream => Unit,
    readResponse: InputStream => T = (_: InputStream) => (),
    chunkSize: Int = 0
  ): T = {
    val conn = new URL(url).openConnection().asInstanceOf[HttpURLConnection]
    conn.setRequestMethod("POST")
    if (chunkSize > 0)
      conn.setChunkedStreamingMode(chunkSize)
    conn.setDoOutput(true);
    conn.setRequestProperty("Content-Length", Integer.toString(contentLength))
    using(conn.getOutputStream())(writeBody)
    assert(200 <= conn.getResponseCode() && conn.getResponseCode() < 300,
      s"POST ${url} ${conn.getResponseCode()} ${using(conn.getErrorStream())(fullyReadInputStreamAsString)}")
    val result = using(conn.getInputStream())(readResponse)
    conn.disconnect()
    result
  }

  def get[T](
    url: String,
    readResponse: InputStream => T
  ): T = {
    val conn = new URL(url).openConnection().asInstanceOf[HttpURLConnection]
    conn.setRequestMethod("GET")
    assert(200 <= conn.getResponseCode() && conn.getResponseCode() < 300,
      s"GET ${url} ${conn.getResponseCode()} ${using(conn.getErrorStream())(fullyReadInputStreamAsString)}")
    val result = using(conn.getInputStream())(readResponse)
    conn.disconnect()
    result
  }

  def delete(
    url: String,
    readResponse: InputStream => Unit = (_: InputStream) => ()
  ): Unit = {
    val conn = new URL(url).openConnection().asInstanceOf[HttpURLConnection]
    conn.setRequestMethod("DELETE")
    assert(200 <= conn.getResponseCode() && conn.getResponseCode() < 300,
      s"DELETE ${url} ${conn.getResponseCode()} ${using(conn.getErrorStream())(fullyReadInputStreamAsString)}")
    val result = using(conn.getInputStream())(readResponse)
    conn.disconnect()
    result
  }

  private[this] def fullyReadInputStreamAsString(is: InputStream): String =
    using(new ByteArrayOutputStream()) { baos =>
      drainInputStreamToOutputStream(is, baos)
      new String(baos.toByteArray(), StandardCharsets.UTF_8)
    }
} 

package cmwell.util.loading

import java.io.{File, InputStream}
import java.net.{URL, URLClassLoader}

// scalastyle:off
import sun.misc.CompoundEnumeration
// scalastyle:on
import scala.util.{Failure, Success, Try}


class ChildFirstURLClassLoader(urls: Array[URL], parent: ClassLoader, except: Seq[String] = Seq())
    extends URLClassLoader(urls, parent) {

  protected override def loadClass(name: String, resolve: Boolean): Class[_] = {
    def tryFind(findAction: => Class[_]): Option[Class[_]] = Try(findAction) match {
      case Failure(e: ClassNotFoundException) => None
      case Failure(e)                         => throw e
      case Success(c)                         => Some(c)
    }

    def loadLocally = if (except.exists(name.startsWith)) None else tryFind(findClass(name))
    def loadFromParent = if (getParent == null) None else tryFind(getParent.loadClass(name))

    val alreadyLoaded = findLoadedClass(name)
    if (alreadyLoaded != null) {
      alreadyLoaded
    } else {

      val `class` = loadLocally.getOrElse(loadFromParent.orNull)

      if (resolve)
        resolveClass(`class`)
      `class`
    }
  }

  override def getResource(name: String): URL = findResource(name) match {
    case null => super.getResource(name)
    case u    => u
  }

  override def getResources(name: String): java.util.Enumeration[URL] = {
    val parent = getParent
    val localUrls = findResources(name)
    val parentUrls: java.util.Enumeration[URL] =
      if (parent != null) parent.getResources(name) else java.util.Collections.emptyEnumeration()
    new CompoundEnumeration(Array(localUrls, parentUrls))
  }

  override def getResourceAsStream(name: String): InputStream = {
    getResource(name) match {
      case null => null
      case url =>
        Try(url.openStream) match {
          case Success(x) => x
          case Failure(_) => null
        }
    }
  }
}

object ChildFirstURLClassLoader {
  def loadClassFromJar[T](className: String, jarPath: String, commonPackageNames:String, excludes: Seq[String] = Seq()): T =
    Loader(jarPath, excludes :+ commonPackageNames).load(className)

  case class Loader(jarPath: String, excludes: Seq[String] = Seq()) {
    val urls = if(new java.io.File(jarPath).isFile) Array(new File(jarPath).toURI.toURL) else Array[URL](new URL(jarPath))
    private val cl =
      new ChildFirstURLClassLoader(urls, this.getClass.getClassLoader, excludes)
    def load[T](className: String) = cl.loadClass(className).newInstance.asInstanceOf[T]
  }

} 

package cmwell.dc.stream.algo

import java.net.URL

import cmwell.dc.LazyLogging
import cmwell.dc.stream.MessagesTypesAndExceptions.AlgoData
import play.api.libs.json.{JsArray, JsDefined, JsLookupResult, JsObject, JsString}

object DDPCAlgorithmJsonParser extends LazyLogging {


  def extractAlgoInfo(f: JsLookupResult):AlgoData = {
  val algoClass = f \ "algoClass" match {
      case JsDefined(JsArray(seq))
        if seq.length == 1 && seq.head.isInstanceOf[JsString] =>
        seq.head.as[String]
    }

    val algoJarUrl = f \ "algoJarUrl" match {
      case JsDefined(JsArray(seq))
        if seq.length == 1 && seq.head.isInstanceOf[JsString] =>
        seq.head.as[String]
    }

    val params = f \ "algoParams" match {
      case JsDefined(JsArray(seq)) =>
        seq.collect {
          case JsString(rule) => rule.split("->") match {
            case Array(source, target) => (source, target)
          }
        }.toMap
      case _ => Map.empty[String, String]
    }
    val url = new URL(algoJarUrl)
    if(url.getHost != "localhost" || !url.getPath.startsWith("/meta"))
      throw new IllegalArgumentException(s"Host is not localhost or url doesn't in /meta, url=$algoJarUrl")
    AlgoData(algoClass, algoJarUrl, params)
  }
} 

package cmwell.tools.neptune.export

import java.net.{URL, URLDecoder, URLEncoder}
import java.time.Instant

import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.DefaultHttpClient
import org.apache.http.util.EntityUtils
import org.slf4j.LoggerFactory

object CmWellConsumeHandler {

  protected lazy val logger = LoggerFactory.getLogger("cm_well_consumer")
  val maxRetry = 5

  private val sleepTimeout = 10000

  def bulkConsume(cluster: String, position: String, format: String, updateMode:Boolean, retryCount:Int= 0): CloseableHttpResponse = {
    val withMeta = if(updateMode) "&with-meta" else ""
    val url = "http://" + cluster + "/_bulk-consume?position=" + position + "&format=" + format + withMeta
    val client = new DefaultHttpClient
    client.setHttpRequestRetryHandler(new CustomHttpClientRetryHandler())
    val get = new HttpGet(url)
    logger.info("Going to bulk consume,url= " + url)
    val response = client.execute(get)
    val statusCode = response.getStatusLine.getStatusCode
    if (statusCode != 200 && statusCode != 204) {
      if(statusCode == 503) {
        logger.error("Failed to bulk consume, error status code=" + statusCode + "response entity=" + EntityUtils.toString(response.getEntity) + ".Going to retry...")
        Thread.sleep(sleepTimeout)
        bulkConsume(cluster, position, format, updateMode)
      }
      else{
        if (retryCount < maxRetry) {
          logger.error("Failed to bulk consume, error status code=" + statusCode + "response entity=" + EntityUtils.toString(response.getEntity) + ".Going to retry...,retry count=" + retryCount)
          Thread.sleep(sleepTimeout)
          bulkConsume(cluster, position, format, updateMode, retryCount + 1)
        } else {
          throw new Throwable("Failed to consume from cm-well, error code status=" + statusCode + ", response entity=" + EntityUtils.toString(response.getEntity))
        }
      }
    }
    response
  }

  def retrivePositionFromCreateConsumer(cluster: String, lengthHint: Int, qp: Option[String], updateMode:Boolean, automaticUpdateMode:Boolean, toolStartTime:Instant, retryCount:Int = 0): String = {
    val withDeletedParam = if(updateMode || automaticUpdateMode) "&with-deleted" else ""
    //initial mode
    val qpTillStartTime = if(!updateMode && !automaticUpdateMode)  URLEncoder.encode(",system.lastModified<") + toolStartTime.toString else ""
    //automatic update mode
    val qpAfterStartTime = if(!updateMode && automaticUpdateMode) URLEncoder.encode(",system.lastModified>>" )+ toolStartTime.toString else ""
    val createConsumerUrl = "http://" + cluster + "/?op=create-consumer&qp=-system.parent.parent_hierarchy:/meta/" + qp.getOrElse("") + qpTillStartTime + qpAfterStartTime + "&recursive&length-hint=" + lengthHint + withDeletedParam
    logger.info("create-consumer-url=" + createConsumerUrl)
    val get = new HttpGet(createConsumerUrl)
    val client = new DefaultHttpClient
    client.setHttpRequestRetryHandler(new CustomHttpClientRetryHandler())
    val response = client.execute(get)
    val res = response.getAllHeaders.find(_.getName == "X-CM-WELL-POSITION").map(_.getValue).getOrElse("")
    logger.info("create-Consumer http status=" + response.getStatusLine.getStatusCode)
    val statusCode = response.getStatusLine.getStatusCode
    if (statusCode != 200) {
      if(statusCode == 503){
        logger.error("Failed to retrieve position via create-consumer api,error status code=" + statusCode + ", response entity=" + EntityUtils.toString(response.getEntity) + ".Going to retry...")
        Thread.sleep(sleepTimeout)
        retrivePositionFromCreateConsumer(cluster, lengthHint, qp, updateMode, automaticUpdateMode, toolStartTime)
      }else {
        if (retryCount < maxRetry) {
          logger.error("Failed to retrieve position via create-consumer api,error status code=" + statusCode + ", response entity=" + EntityUtils.toString(response.getEntity) + ".Going to retry..., retry count=" + retryCount)
          Thread.sleep(sleepTimeout)
          retrivePositionFromCreateConsumer(cluster, lengthHint, qp, updateMode, automaticUpdateMode, toolStartTime, retryCount+1)
        }
        else {
          throw new Throwable("Failed to consume from cm-well, error code status=" + statusCode + ", response entity=" + EntityUtils.toString(response.getEntity))
        }
      }
    }
    res
  }


} 

package org.scalawebtest.core.browser

import java.io.File
import java.net.URL

import org.openqa.selenium.chrome.ChromeDriverService
import org.scalatest.ConfigMap
import org.scalawebtest.core.Configurable

object ChromeDriverServiceRunner extends Configurable {
  //before test suite
  private val driverServiceUrlProperty = "webdriver.chrome.driver.service.url"
  private val driverProperty = "webdriver.chrome.driver"

  var internalServiceOrPort: Either[ChromeDriverService, URL] = _

  def assertInitialized(configMap: ConfigMap): URL = {
    serviceOrPort(configMap) match {
      case Left(s) => s.getUrl
      case Right(u) => u
    }
  }

  private def serviceOrPort(configMap: ConfigMap): Either[ChromeDriverService, URL] = {
    val runningChromeDriverServicePort = configFor[URL](configMap)(driverServiceUrlProperty)
    if (internalServiceOrPort == null) {
      internalServiceOrPort = runningChromeDriverServicePort match {
        case Some(url) =>
          println(s"Not taking any action regarding ChromeDriverService, because it is managed outside of ScalaWebTest and access was provided via $driverServiceUrlProperty property.")
          Right(url)
        case None =>
          val chromeDriverPath = requiredConfigFor[String](configMap)(driverProperty)

          val service = new ChromeDriverService.Builder()
            .usingDriverExecutable(new File(chromeDriverPath))
            .usingAnyFreePort.build

          service.start()
          println(s"Started ChromeDriverService from path $chromeDriverPath")
          Left(service)
      }
    }
    internalServiceOrPort
  }

  //after test suite
  sys addShutdownHook {
    //an empty ConfigMap is used, because we can not access the real ConfigMap in the shutdownHook,
    //but also because we do need information from the ConfigMap during shutdown.
    serviceOrPort(ConfigMap.empty) match {
      case Left(s) =>
        s.stop()
        println("Stopped ChromeDriverService")
      case Right(p) =>
        println(s"Not taking any action regarding ChromeDriverService, because it is managed outside of ScalaWebTest and access was provided via $driverServiceUrlProperty property.")
    }
  }
} 

package org.scalawebtest.core.browser

import java.net.URL

import org.openqa.selenium.Capabilities
import org.openqa.selenium.chrome.ChromeOptions
import org.openqa.selenium.remote.RemoteWebDriver
import org.scalatest.ConfigMap
import org.scalawebtest.core.configuration._
import org.scalawebtest.core.{Configurable, IntegrationSpec}

import scala.jdk.CollectionConverters._


trait SeleniumChrome extends Configurable {
  self: IntegrationSpec =>

  override val loginConfig = new LoginConfiguration with SeleniumChromeConfiguration

  override val config = new Configuration with SeleniumChromeConfiguration

  //this imports the ChromeDriverServiceRunner object, and therefore allows it to execute commands before and after the test suite was run
  private val driverServiceRunner = ChromeDriverServiceRunner

  override def prepareWebDriver(configMap: ConfigMap): Unit = {
    val driverServiceUrl = driverServiceRunner.assertInitialized(configMap)
    val chromeArguments =
      configFor[String](configMap)("webdriver.chrome.arguments").map(_.split(',').toList)
        .getOrElse(List("--no-sandbox", "--headless"))
        .asJava

    webDriver = new ChromeRemoteWebDriver(
      driverServiceUrl,
      new ChromeOptions().addArguments(chromeArguments)
    )
  }

  class ChromeRemoteWebDriver(remoteAddress: URL, capabilities: Capabilities) extends RemoteWebDriver(remoteAddress, capabilities) {
    override def getPageSource: String = {
      super.getPageSource
        .replaceFirst("""<html.*><head.*></head><body.*><pre style="word-wrap: break-word; white-space: pre-wrap;">""", "")
        .replaceFirst("""</pre></body></html>""", "")
    }
  }

  trait SeleniumChromeConfiguration extends BaseConfiguration with WebDriverName with SeleniumBrowserConfiguration {
    override val webDriverName: String = classOf[SeleniumChrome].getCanonicalName
  }

} 

package com.localytics.sbt.dynamodb

import java.io.FileInputStream
import java.net.URL
import java.util.zip.GZIPInputStream
import java.util.zip.ZipFile

import sbt.File
import sbt.Keys._

import scala.concurrent.duration.Duration
import scala.sys.process._
import scala.util.Try

object DeployDynamoDBLocal {

  private[dynamodb] def validJar(file: File): Boolean = Try(new ZipFile(file)).isSuccess

  private[dynamodb] def validGzip(file: File): Boolean = Try(new GZIPInputStream(new FileInputStream(file)).read()).isSuccess

  def apply(ver: String, url: Option[String], targetDir: File, downloadIfOlderThan: Duration, streamz: TaskStreams): File = {
    val targz = new File(targetDir, s"dynamodb_local_$ver.tar.gz")
    val jar = new File(targetDir, "DynamoDBLocal.jar")

    def isStale(file: File) = ver == "latest" && System.currentTimeMillis - file.lastModified() > downloadIfOlderThan.toMillis

    if (!targetDir.exists()) {
      streamz.log.info(s"Creating DynamoDB Local directory $targetDir")
      targetDir.mkdirs()
    }
    if (!targz.exists() || isStale(targz) || !validGzip(targz)) {
      val remoteFile = url.getOrElse(s"https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_$ver.tar.gz")
      streamz.log.info(s"Downloading targz from [$remoteFile] to [${targz.getAbsolutePath}]")
      (new URL(remoteFile) #> targz).!!
    }
    if (!validGzip(targz)) sys.error(s"Invalid gzip file at [${targz.getAbsolutePath}]")
    if (!jar.exists() || !validJar(jar)) {
      streamz.log.info(s"Extracting jar from [${targz.getAbsolutePath}] to [${jar.getAbsolutePath}]")
      Process(Seq("tar", "xzf", targz.getName), targetDir).!!
    }
    if (!validJar(jar)) sys.error(s"Invalid jar file at [${jar.getAbsolutePath}]")
    jar
  }

} 

package org.apache.spark.sql.hive

import java.net.URL

import org.apache.spark.SparkFunSuite


class ClasspathDependenciesSuite extends SparkFunSuite {
  private val classloader = this.getClass.getClassLoader

  private def assertLoads(classname: String): Unit = {
    val resourceURL: URL = Option(findResource(classname)).getOrElse {
      fail(s"Class $classname not found as ${resourceName(classname)}")
    }

    logInfo(s"Class $classname at $resourceURL")
    classloader.loadClass(classname)
  }

  private def findResource(classname: String): URL = {
    val resource = resourceName(classname)
    classloader.getResource(resource)
  }

  private def resourceName(classname: String): String = {
    classname.replace(".", "/") + ".class"
  }

  private def assertClassNotFound(classname: String): Unit = {
    Option(findResource(classname)).foreach { resourceURL =>
      fail(s"Class $classname found at $resourceURL")
    }

    intercept[ClassNotFoundException] {
      classloader.loadClass(classname)
    }
  }

  test("shaded Protobuf") {
    assertLoads("org.apache.hive.com.google.protobuf.ServiceException")
  }

  test("shaded Kryo") {
    assertLoads("org.apache.hive.com.esotericsoftware.kryo.Kryo")
  }

  test("hive-common") {
    assertLoads("org.apache.hadoop.hive.conf.HiveConf")
  }

  test("hive-exec") {
    assertLoads("org.apache.hadoop.hive.ql.CommandNeedRetryException")
  }

  private val STD_INSTANTIATOR = "org.objenesis.strategy.StdInstantiatorStrategy"

  test("Forbidden Dependencies") {
    assertClassNotFound("com.esotericsoftware.shaded." + STD_INSTANTIATOR)
    assertClassNotFound("org.apache.hive.com.esotericsoftware.shaded." + STD_INSTANTIATOR)
  }

  test("parquet-hadoop-bundle") {
    assertLoads("parquet.hadoop.ParquetOutputFormat")
    assertLoads("parquet.hadoop.ParquetInputFormat")
  }
} 

package org.apache.spark.util

import java.net.{URL, URLClassLoader}
import java.util.Enumeration

import scala.collection.JavaConverters._


private[spark] class ChildFirstURLClassLoader(urls: Array[URL], parent: ClassLoader)
  extends MutableURLClassLoader(urls, null) {

  private val parentClassLoader = new ParentClassLoader(parent)

  override def loadClass(name: String, resolve: Boolean): Class[_] = {
    try {
      super.loadClass(name, resolve)
    } catch {
      case e: ClassNotFoundException =>
        parentClassLoader.loadClass(name, resolve)
    }
  }

  override def getResource(name: String): URL = {
    val url = super.findResource(name)
    val res = if (url != null) url else parentClassLoader.getResource(name)
    res
  }

  override def getResources(name: String): Enumeration[URL] = {
    val childUrls = super.findResources(name).asScala
    val parentUrls = parentClassLoader.getResources(name).asScala
    (childUrls ++ parentUrls).asJavaEnumeration
  }

  override def addURL(url: URL) {
    super.addURL(url)
  }

} 

package org.apache.spark.deploy

import java.net.URL

import scala.collection.mutable
import scala.io.Source

import org.apache.spark.{LocalSparkContext, SparkContext, SparkFunSuite}
import org.apache.spark.scheduler.{SparkListener, SparkListenerExecutorAdded}
import org.apache.spark.scheduler.cluster.ExecutorInfo
import org.apache.spark.util.SparkConfWithEnv

class LogUrlsStandaloneSuite extends SparkFunSuite with LocalSparkContext {

  
  private val WAIT_TIMEOUT_MILLIS = 10000

  test("verify that correct log urls get propagated from workers") {
    sc = new SparkContext("local-cluster[2,1,1024]", "test")

    val listener = new SaveExecutorInfo
    sc.addSparkListener(listener)

    // Trigger a job so that executors get added
    sc.parallelize(1 to 100, 4).map(_.toString).count()

    sc.listenerBus.waitUntilEmpty(WAIT_TIMEOUT_MILLIS)
    listener.addedExecutorInfos.values.foreach { info =>
      assert(info.logUrlMap.nonEmpty)
      // Browse to each URL to check that it's valid
      info.logUrlMap.foreach { case (logType, logUrl) =>
        val html = Source.fromURL(logUrl).mkString
        assert(html.contains(s"$logType log page"))
      }
    }
  }

  test("verify that log urls reflect SPARK_PUBLIC_DNS (SPARK-6175)") {
    val SPARK_PUBLIC_DNS = "public_dns"
    val conf = new SparkConfWithEnv(Map("SPARK_PUBLIC_DNS" -> SPARK_PUBLIC_DNS)).set(
      "spark.extraListeners", classOf[SaveExecutorInfo].getName)
    sc = new SparkContext("local-cluster[2,1,1024]", "test", conf)

    // Trigger a job so that executors get added
    sc.parallelize(1 to 100, 4).map(_.toString).count()

    sc.listenerBus.waitUntilEmpty(WAIT_TIMEOUT_MILLIS)
    val listeners = sc.listenerBus.findListenersByClass[SaveExecutorInfo]
    assert(listeners.size === 1)
    val listener = listeners(0)
    listener.addedExecutorInfos.values.foreach { info =>
      assert(info.logUrlMap.nonEmpty)
      info.logUrlMap.values.foreach { logUrl =>
        assert(new URL(logUrl).getHost === SPARK_PUBLIC_DNS)
      }
    }
  }
}

private[spark] class SaveExecutorInfo extends SparkListener {
  val addedExecutorInfos = mutable.Map[String, ExecutorInfo]()

  override def onExecutorAdded(executor: SparkListenerExecutorAdded) {
    addedExecutorInfos(executor.executorId) = executor.executorInfo
  }
} 

package org.apache.spark.deploy.master.ui

import java.io.DataOutputStream
import java.net.{HttpURLConnection, URL}
import java.nio.charset.StandardCharsets
import java.util.Date

import scala.collection.mutable.HashMap

import org.mockito.Mockito.{mock, times, verify, when}
import org.scalatest.BeforeAndAfterAll

import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
import org.apache.spark.deploy.DeployMessages.{KillDriverResponse, RequestKillDriver}
import org.apache.spark.deploy.DeployTestUtils._
import org.apache.spark.deploy.master._
import org.apache.spark.rpc.{RpcEndpointRef, RpcEnv}


class MasterWebUISuite extends SparkFunSuite with BeforeAndAfterAll {

  val conf = new SparkConf
  val securityMgr = new SecurityManager(conf)
  val rpcEnv = mock(classOf[RpcEnv])
  val master = mock(classOf[Master])
  val masterEndpointRef = mock(classOf[RpcEndpointRef])
  when(master.securityMgr).thenReturn(securityMgr)
  when(master.conf).thenReturn(conf)
  when(master.rpcEnv).thenReturn(rpcEnv)
  when(master.self).thenReturn(masterEndpointRef)
  val masterWebUI = new MasterWebUI(master, 0)

  override def beforeAll() {
    super.beforeAll()
    masterWebUI.bind()
  }

  override def afterAll() {
    masterWebUI.stop()
    super.afterAll()
  }

  test("kill application") {
    val appDesc = createAppDesc()
    // use new start date so it isn't filtered by UI
    val activeApp = new ApplicationInfo(
      new Date().getTime, "app-0", appDesc, new Date(), null, Int.MaxValue)

    when(master.idToApp).thenReturn(HashMap[String, ApplicationInfo]((activeApp.id, activeApp)))

    val url = s"http://localhost:${masterWebUI.boundPort}/app/kill/"
    val body = convPostDataToString(Map(("id", activeApp.id), ("terminate", "true")))
    val conn = sendHttpRequest(url, "POST", body)
    conn.getResponseCode

    // Verify the master was called to remove the active app
    verify(master, times(1)).removeApplication(activeApp, ApplicationState.KILLED)
  }

  test("kill driver") {
    val activeDriverId = "driver-0"
    val url = s"http://localhost:${masterWebUI.boundPort}/driver/kill/"
    val body = convPostDataToString(Map(("id", activeDriverId), ("terminate", "true")))
    val conn = sendHttpRequest(url, "POST", body)
    conn.getResponseCode

    // Verify that master was asked to kill driver with the correct id
    verify(masterEndpointRef, times(1)).ask[KillDriverResponse](RequestKillDriver(activeDriverId))
  }

  private def convPostDataToString(data: Map[String, String]): String = {
    (for ((name, value) <- data) yield s"$name=$value").mkString("&")
  }

  
  private def sendHttpRequest(
      url: String,
      method: String,
      body: String = ""): HttpURLConnection = {
    val conn = new URL(url).openConnection().asInstanceOf[HttpURLConnection]
    conn.setRequestMethod(method)
    if (body.nonEmpty) {
      conn.setDoOutput(true)
      conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded")
      conn.setRequestProperty("Content-Length", Integer.toString(body.length))
      val out = new DataOutputStream(conn.getOutputStream)
      out.write(body.getBytes(StandardCharsets.UTF_8))
      out.close()
    }
    conn
  }
} 

package org.apache.spark.util

import java.net.{URLClassLoader, URL}
import java.util.Enumeration
import java.util.concurrent.ConcurrentHashMap

import scala.collection.JavaConversions._


  private val locks = new ConcurrentHashMap[String, Object]()

  override def loadClass(name: String, resolve: Boolean): Class[_] = {
    var lock = locks.get(name)
    if (lock == null) {
      val newLock = new Object()
      lock = locks.putIfAbsent(name, newLock)
      if (lock == null) {
        lock = newLock
      }
    }

    lock.synchronized {
      try {
        super.loadClass(name, resolve)
      } catch {
        case e: ClassNotFoundException =>
          parentClassLoader.loadClass(name, resolve)
      }
    }
  }

  override def getResource(name: String): URL = {
    val url = super.findResource(name)
    val res = if (url != null) url else parentClassLoader.getResource(name)
    res
  }

  override def getResources(name: String): Enumeration[URL] = {
    val urls = super.findResources(name)
    val res =
      if (urls != null && urls.hasMoreElements()) {
        urls
      } else {
        parentClassLoader.getResources(name)
      }
    res
  }

  override def addURL(url: URL) {
    super.addURL(url)
  }

} 

package org.apache.spark.deploy

import java.net.URL

import scala.collection.JavaConversions._
import scala.collection.mutable
import scala.io.Source

import org.apache.spark.scheduler.cluster.ExecutorInfo
import org.apache.spark.scheduler.{SparkListenerExecutorAdded, SparkListener}
import org.apache.spark.{LocalSparkContext, SparkConf, SparkContext, SparkFunSuite}

class LogUrlsStandaloneSuite extends SparkFunSuite with LocalSparkContext {

  
  private val WAIT_TIMEOUT_MILLIS = 10000

  test("verify that correct log urls get propagated from workers") {
    sc = new SparkContext("local-cluster[2,1,512]", "test")

    val listener = new SaveExecutorInfo
    sc.addSparkListener(listener)

    // Trigger a job so that executors get added
    sc.parallelize(1 to 100, 4).map(_.toString).count()

    sc.listenerBus.waitUntilEmpty(WAIT_TIMEOUT_MILLIS)
    listener.addedExecutorInfos.values.foreach { info =>
      assert(info.logUrlMap.nonEmpty)
      // Browse to each URL to check that it's valid
      info.logUrlMap.foreach { case (logType, logUrl) =>
        val html = Source.fromURL(logUrl).mkString
        assert(html.contains(s"$logType log page"))
      }
    }
  }

  test("verify that log urls reflect SPARK_PUBLIC_DNS (SPARK-6175)") {
    val SPARK_PUBLIC_DNS = "public_dns"
    class MySparkConf extends SparkConf(false) {
      override def getenv(name: String): String = {
        if (name == "SPARK_PUBLIC_DNS") SPARK_PUBLIC_DNS
        else super.getenv(name)
      }

      override def clone: SparkConf = {
        new MySparkConf().setAll(getAll)
      }
    }
    val conf = new MySparkConf().set(
      "spark.extraListeners", classOf[SaveExecutorInfo].getName)
    sc = new SparkContext("local-cluster[2,1,512]", "test", conf)

    // Trigger a job so that executors get added
    sc.parallelize(1 to 100, 4).map(_.toString).count()

    sc.listenerBus.waitUntilEmpty(WAIT_TIMEOUT_MILLIS)
    val listeners = sc.listenerBus.findListenersByClass[SaveExecutorInfo]
    assert(listeners.size === 1)
    val listener = listeners(0)
    listener.addedExecutorInfos.values.foreach { info =>
      assert(info.logUrlMap.nonEmpty)
      info.logUrlMap.values.foreach { logUrl =>
        assert(new URL(logUrl).getHost === SPARK_PUBLIC_DNS)
      }
    }
  }
}

private[spark] class SaveExecutorInfo extends SparkListener {
  val addedExecutorInfos = mutable.Map[String, ExecutorInfo]()

  override def onExecutorAdded(executor: SparkListenerExecutorAdded) {
    addedExecutorInfos(executor.executorId) = executor.executorInfo
  }
} 

package io.getquill.context.spark.examples

import java.io.File
import java.net.URL
import scala.language.postfixOps
import scala.sys.process._
import org.apache.spark.sql.SparkSession
import io.getquill.Ord
import io.getquill.QuillSparkContext._

case class User(
  id:          String,
  login:       String,
  gravatar_id: String,
  url:         String,
  avatar_url:  String
)

case class Repo(
  id:   String,
  name: String,
  url:  String
)

case class Activity(
  id:         String,
  `type`:     String,
  actor:      User,
  repo:       Repo,
  created_at: String,
  org:        User
)

object GithubExample extends App {

  val files =
    for {
      year <- 2017 to 2017
      month <- 10 to 10
      day <- 22 to 22
      hour <- 0 to 23
    } yield "%04d-%02d-%02d-%02d".format(year, month, day, hour)

  files.par.foreach { name =>
    val file = new File(s"$name.json.gz")
    if (!file.exists()) {
      println(s"downloading missing file $name")
      new URL(s"http://data.githubarchive.org/$name.json.gz") #> new File(s"$name.json.gz") !!
    }
  }

  implicit val sqlContext =
    SparkSession
      .builder()
      .master("local[*]")
      .appName("spark test")
      .getOrCreate()
      .sqlContext

  import sqlContext.implicits._

  val activities = liftQuery(sqlContext.read.json(files.map(n => s"$n.json.gz"): _*).as[Activity])

  val topStargazers = quote {
    activities
      .groupBy(_.actor)
      .map {
        case (actor, list) => (actor.login, list.size)
      }.sortBy {
        case (login, size) => size
      }(Ord.desc)
  }

  val topProjects = quote {
    activities
      .filter(_.`type` == "WatchEvent")
      .groupBy(_.repo)
      .map {
        case (repo, list) => (repo.name, list.size)
      }.sortBy {
        case (repoName, size) => size
      }(Ord.desc)
  }

  println(run(topStargazers).show())
  println(run(topProjects).show())
} 

package ai.deepsense.workflowmanager.migration

import java.net.URL
import java.util.UUID

import scala.concurrent.duration._
import scala.concurrent.{Await, ExecutionContext, Future}
import scala.language.postfixOps

import akka.actor.ActorSystem

import ai.deepsense.commons.utils.{Logging, Version}
import ai.deepsense.models.workflows.Workflow
import ai.deepsense.workflowmanager.storage.WorkflowStorage
import ai.deepsense.workflowmanager.versionconverter.VersionConverter

class Migration1_3To1_4 private (
    val datasourcemanagerUrl: URL,
    val workflowStorage: WorkflowStorage,
    val actorSystem: ActorSystem)
    extends Logging
    with SeahorseDbMigration {

  implicit val ec: ExecutionContext = actorSystem.dispatcher

  override val previousVersion = Version(1, 3, 0)
  override val targetVersion = Version(1, 4, 0)

  def migrate(): Future[Unit] = {

    val updatedWorkflowsAndNewDatasourcesFut: Future[Seq[MigrationResult]] = for {
      workflows <- workflowStorage.getAllRaw

    } yield {
      for {
        (id, raw) <- workflows.toSeq if isConvertible(id, raw)
      } yield {
        logger.info(s"Found version 1.3.x workflow: $id - will perform conversion to current version 1.4")
        val (rawWorkflow, newDatasources) = VersionConverter.convert13to14(raw.workflow, raw.ownerId, raw.ownerName)
        MigrationResult(
          id,
          UUID.fromString(raw.ownerId),
          raw.ownerName,
          rawWorkflow,
          newDatasources)
      }
    }

    for {
      migrationFutures <- updatedWorkflowsAndNewDatasourcesFut.map(commitMigrationsToDb)
    } yield {
      for {
        migration <- migrationFutures
      } {

        migration.onFailure {
          case t => logger.error("Unable to migrate workflow", t)
        }

        Await.ready(migration, Duration.Inf)
        logger.info(s"Migration to ${targetVersion.humanReadable} finished")
      }
    }
  }

}

object Migration1_3To1_4 {
  def apply(datasourcemanagerUrl: URL, workflowStorage: WorkflowStorage, actorSystem: ActorSystem): Migration1_3To1_4 =
    new Migration1_3To1_4(datasourcemanagerUrl, workflowStorage, actorSystem)
} 

package ai.deepsense.workflowmanager.versionconverter

import java.net.URL
import java.util.UUID

import scala.util.{Try, Success, Failure}

import ai.deepsense.api.datasourcemanager.ApiClient
import ai.deepsense.api.datasourcemanager.client.DefaultApi
import ai.deepsense.api.datasourcemanager.model.DatasourceParams
import ai.deepsense.commons.utils.Logging

class DatasourcesClient(datasourceServerAddress: URL, userId: UUID, userName: String) extends Logging {

  private val client = {
    val apiClient = new ApiClient()
    apiClient.setAdapterBuilder(
      apiClient.getAdapterBuilder.baseUrl(datasourceServerAddress.toString))
    apiClient.createService(classOf[DefaultApi])
  }

  def insertDatasource(uuid: UUID, datasourceParams: DatasourceParams): Try[Unit] = {
    val response = client.putDatasource(userId.toString, userName, uuid.toString, datasourceParams).execute()
    logger.info(s"Adding datasource, userId = $userId, userName = $userName," +
      s"uuid = $uuid, params = $datasourceParams")
    if (response.isSuccessful) { Success {
      logger.info(s"Successfully added datasource; body = ${response.body()}")
    }} else {
      Failure(new RuntimeException(
        s"There was a problem with adding datasource," +
          s"code: ${response.code()}, error body: ${response.errorBody().string()}."
      ))
    }
  }
} 

package ai.deepsense.workflowmanager

import java.net.URL

import scala.concurrent.duration.Duration
import scala.concurrent.{Await, ExecutionContext, Future}

import akka.actor.ActorSystem
import com.google.inject.name.Names
import com.google.inject.{Guice, Key, Stage}

import ai.deepsense.commons.rest.RestServer
import ai.deepsense.commons.utils.Logging
import ai.deepsense.sparkutils.AkkaUtils
import ai.deepsense.workflowmanager.migration.{Migration1_3To1_4, SeahorseDbMigration}
import ai.deepsense.workflowmanager.storage.WorkflowStorage


object WorkflowManagerApp extends App with Logging {

  val insecure: Boolean = args.headOption.forall("insecure".equalsIgnoreCase)

  try {
    FlywayMigration.run()

    val injector = Guice.createInjector(Stage.PRODUCTION, new WorkflowManagerAppModule(insecure))
    val actorSystem = injector.getInstance(classOf[ActorSystem])
    implicit val ec: ExecutionContext = actorSystem.dispatcher

    val datasourceUrl = new URL(injector.getInstance(Key.get(classOf[String],
      Names.named("datasource-server.address"))))


    val migrationFut =
    for {
      _ <- SeahorseDbMigration.waitForDatasourceManager(datasourceUrl, actorSystem)
      _ <- SeahorseDbMigration.migrate(
        datasourceUrl,
        injector.getInstance(classOf[WorkflowStorage]),
        actorSystem)
    } yield ()

    migrationFut.onFailure {
      case t => logger.error("Migration 1.3 to 1.4 failed", t)
    }

    Await.ready(migrationFut, Duration.Inf)

    injector.getInstance(classOf[RestServer]).start()
    AkkaUtils.awaitTermination(actorSystem)
  } catch {
    case e: Exception =>
      logger.error("Application context creation failed", e)
      System.exit(1)
  }

} 

package ai.deepsense.workflowmanager.client

import java.net.URL
import java.util.UUID

import scala.concurrent.Future
import scala.language.postfixOps

import akka.actor.ActorSystem
import akka.util.Timeout
import spray.client.pipelining._
import spray.http._
import spray.json.RootJsonFormat

import ai.deepsense.commons.json.envelope.{Envelope, EnvelopeJsonFormat}
import ai.deepsense.commons.rest.client.RestClient
import ai.deepsense.commons.utils.Logging
import ai.deepsense.models.json.workflow.WorkflowInfoJsonProtocol
import ai.deepsense.models.workflows.{Workflow, WorkflowInfo, WorkflowWithVariables}
import ai.deepsense.workflowmanager.model.{WorkflowDescription, WorkflowDescriptionJsonProtocol}

class WorkflowManagerClient(
    override val apiUrl: URL,
    mandatoryUserId: UUID,
    mandatoryUserName: String,
    override val credentials: Option[HttpCredentials])(
    implicit override val as: ActorSystem,
    override val timeout: Timeout)
  extends RestClient
  with WorkflowInfoJsonProtocol
  with WorkflowDescriptionJsonProtocol
  with Logging {

  override def userId: Option[UUID] = Some(mandatoryUserId)
  override def userName: Option[String] = Some(mandatoryUserName)

  implicit private val envelopeWorkflowIdJsonFormat =
    new EnvelopeJsonFormat[Workflow.Id]("workflowId")


  def fetchWorkflows(): Future[List[WorkflowInfo]] = {
    fetchResponse[List[WorkflowInfo]](Get(endpointPath("")))
  }

  def fetchWorkflow(id: Workflow.Id)(implicit workflowJsonFormat: RootJsonFormat[Workflow]): Future[Workflow] = {
    fetchResponse[Workflow](Get(endpointPath(s"$id")))
  }

  def fetchWorkflowInfo(id: Workflow.Id): Future[WorkflowInfo] = {
    fetchResponse[WorkflowInfo](Get(endpointPath(s"$id/info")))
  }

  def cloneWorkflow(workflowId: Workflow.Id,
      workflowDescription: WorkflowDescription):
  Future[Workflow.Id] = {
    fetchResponse[Envelope[Workflow.Id]](Post(
      endpointPath(s"$workflowId/clone"),
      workflowDescription
    )).map(_.content)
  }

  def deleteWorkflow(workflowId: Workflow.Id): Future[Unit] = {
    fetchHttpResponse(Delete(endpointPath(s"$workflowId"))).map(_ => ())
  }

  def uploadWorkflow(workflow: Workflow)
    (implicit rootJsonWorkflow: RootJsonFormat[Workflow]): Future[Workflow.Id] = {
    uploadWorkflow(rootJsonWorkflow.write(workflow).toString())
  }

  def downloadWorkflow(workflowId: Workflow.Id)
    (implicit jsonFormat: RootJsonFormat[WorkflowWithVariables]): Future[Option[WorkflowWithVariables]] = {
    fetchResponse[Option[WorkflowWithVariables]](Get(endpointPath(s"$workflowId/download?export-datasources=true")))
  }

  def uploadWorkflow(workflow: String): Future[Workflow.Id] = {
    fetchResponse[Envelope[Workflow.Id]](Post(
      endpointPath(s"upload"),
      MultipartFormData(Seq(BodyPart(HttpEntity(workflow), "workflowFile"))))).map(_.content)
  }
} 

package ai.deepsense.workflowmanager.client

import java.net.URL
import java.util.UUID

import akka.actor.ActorSystem
import akka.util.Timeout
import spray.client.pipelining._
import spray.http._

import scala.concurrent.{ExecutionContext, Future}
import scala.language.postfixOps

import ai.deepsense.commons.models.ClusterDetails
import ai.deepsense.commons.rest.ClusterDetailsJsonProtocol
import ai.deepsense.commons.rest.client.RestClient
import ai.deepsense.commons.utils.Logging

class PresetsClient(
    override val apiUrl: URL,
    mandatoryUserId: UUID,
    mandatoryUserName: String,
    override val credentials: Option[HttpCredentials])(
    implicit override val as: ActorSystem,
    override val timeout: Timeout)
  extends RestClient with ClusterDetailsJsonProtocol with Logging {

  override def userId: Option[UUID] = Some(mandatoryUserId)
  override def userName: Option[String] = Some(mandatoryUserName)

  def fetchPreset(id: Long): Future[Option[ClusterDetails]] = {
    fetchResponse[Option[ClusterDetails]](Get(endpointPath(s"$id")))
  }
} 

package ai.deepsense.commons.rest.client

import java.net.URL
import java.util.UUID

import scala.concurrent.{ExecutionContext, Future}

import akka.io.IO
import akka.pattern.ask
import spray.can.Http
import spray.can.Http.HostConnectorInfo
import spray.client.pipelining._
import spray.http.{HttpCredentials, HttpRequest, HttpResponse}
import spray.httpx.unmarshalling.FromResponseUnmarshaller

trait RestClient extends RestClientImplicits {
  import RestClient._

  def apiUrl: URL
  def userId: Option[UUID]
  def userName: Option[String]
  def credentials: Option[HttpCredentials]
  implicit override val ctx: ExecutionContext = as.dispatcher

  private def hostConnectorFut(): Future[HostConnectorInfo] = {
    (IO(Http) ? Http.HostConnectorSetup(apiUrl.getHost, port = apiUrl.getPort)).mapTo[HostConnectorInfo]
  }

  private def sendReceivePipeline() : Future[HttpRequest => Future[HttpResponse]] = {
    for {
      HostConnectorInfo(hostConnector, _) <- hostConnectorFut()
    } yield {
      credentials.map(addCredentials).getOrElse[RequestTransformer](identity) ~>
        userId.map(id => addHeader(UserIdHeader, id.toString)).getOrElse[RequestTransformer](identity) ~>
        userName.map(addHeader(UserNameHeader, _)).getOrElse[RequestTransformer](identity) ~>
        sendReceive(hostConnector)
    }
  }

  private def unmarshalPipeline[U: FromResponseUnmarshaller](): Future[HttpRequest => Future[U]] = {
    for {
      sr <- sendReceivePipeline()
    } yield {
      sr ~> unmarshal[U]
    }
  }

  def fetchResponse[U : FromResponseUnmarshaller](
      req: HttpRequest
  ): Future[U] = {
    unmarshalPipeline().flatMap(_(req))
  }

  def fetchHttpResponse(req: HttpRequest) : Future[HttpResponse] = {
    sendReceivePipeline().flatMap(_(req))
  }

  def endpointPath(endpoint: String): String = {
    new URL(apiUrl, endpoint).getFile
  }
}


object RestClient {
  val UserIdHeader = "X-Seahorse-UserId"
  val UserNameHeader = "X-Seahorse-UserName"
} 

package ai.deepsense.commons.rest.client.datasources

import java.net.URL
import java.util.UUID

import ai.deepsense.api.datasourcemanager.ApiClient
import ai.deepsense.api.datasourcemanager.client.DefaultApi
import ai.deepsense.api.datasourcemanager.model.Datasource
import ai.deepsense.commons.utils.Logging

class DatasourceRestClient(
    datasourceServerAddress: URL,
    userId: String)
  extends DatasourceClient
  with Logging {

  private val client = {
    val apiClient = new ApiClient()
    apiClient.setAdapterBuilder(
      apiClient.getAdapterBuilder.baseUrl(datasourceServerAddress.toString))
    apiClient.createService(classOf[DefaultApi])
  }

  def getDatasource(uuid: UUID): Option[Datasource] = {
    val response = client.getDatasource(userId, uuid.toString).execute()
    if (response.isSuccessful) {
      Some(response.body)
    } else {
      None
    }
  }
}

class DatasourceRestClientFactory(
    datasourceServerAddress: URL,
    userId: String) extends DatasourceClientFactory {

  override def createClient: DatasourceRestClient = {
    new DatasourceRestClient(datasourceServerAddress, userId)
  }
} 

package ai.deepsense.commons.rest.client

import java.net.URL

import scala.concurrent._
import scala.concurrent.duration._
import scala.language.postfixOps

import akka.actor._
import akka.util.Timeout
import spray.client.pipelining._
import spray.http.{HttpResponse, StatusCodes}
import spray.httpx.SprayJsonSupport

import ai.deepsense.commons.json.NotebookRestClientProtocol._
import ai.deepsense.commons.models.Id
import ai.deepsense.commons.rest.client.req.NotebookClientRequest
import ai.deepsense.commons.utils.Logging


case class NotebookHttpException(
    httpResponse: HttpResponse,
    msg: String,
    cause: Throwable = null)
  extends Exception(msg, cause)


class NotebookRestClient(
    notebooksServerAddress: URL,
    workflowId: Id,
    nodeId: Id,
    pollInterval: FiniteDuration,
    retryCountLimit: Int
)(implicit override val as: ActorSystem)
  extends Logging with RestClient with SprayJsonSupport {

  def apiUrl: java.net.URL = new URL(notebooksServerAddress, "/jupyter/")
  def credentials: Option[spray.http.HttpCredentials] = None
  def userId: Option[java.util.UUID] = None
  def userName: Option[String] = None

  implicit val timeout: Timeout = 70 minutes

  private val filenameExtension = "html"

  private val postPath = endpointPath("HeadlessNotebook")
  private val getPath = endpointPath(s"HeadlessNotebook/${workflowId}_$nodeId.$filenameExtension")

  private val poller = NotebookPoller(this, pollInterval, retryCountLimit, workflowId, nodeId, getPath)

  def pollForNotebookData(): Future[Array[Byte]] = poller.tryWork

  def generateNotebookData(language: String): Future[HttpResponse] = {
    val req = NotebookClientRequest(workflowId, nodeId, language)
    fetchHttpResponse(Post(postPath, req)).flatMap { resp => resp.status match {
      case StatusCodes.Success(_) => Future.successful(resp)
      case statusCode => Future.failed(NotebookHttpException(resp,
        s"Notebook server responded with $statusCode when asked to generate notebook data"
      ))
    }}
  }

  def generateAndPollNbData(language: String): Future[Array[Byte]] = {
    generateNotebookData(language).flatMap(_ => pollForNotebookData())
  }

  def toFactory: NotebooksClientFactory =
    new NotebooksClientFactory(notebooksServerAddress, pollInterval, retryCountLimit)

}

class NotebooksClientFactory(notebooksServerAddress: URL, pollInterval: FiniteDuration, retryCountLimit: Int)
  (implicit system: ActorSystem) {
  def createNotebookForNode(workflow: Id, node: Id): NotebookRestClient = {
    new NotebookRestClient(notebooksServerAddress, workflow, node, pollInterval, retryCountLimit)
  }
} 

package ai.deepsense.deeplang

import java.io.File
import java.net.{URL, URLClassLoader}

import ai.deepsense.commons.utils.LoggerForCallerClass
import ai.deepsense.deeplang.catalogs.DCatalog
import ai.deepsense.deeplang.catalogs.spi.CatalogRegistrant
import ai.deepsense.deeplang.catalogs.spi.CatalogRegistrar.DefaultCatalogRegistrar
import ai.deepsense.deeplang.refl.CatalogScanner
import org.apache.spark.SparkContext


class CatalogRecorder private (jars: Seq[URL]) {
  def withDir(jarsDir: File): CatalogRecorder = {
    val additionalJars =
      if (jarsDir.exists && jarsDir.isDirectory) {
        jarsDir.listFiles().toSeq.filter(f => f.isFile && f.getName.endsWith(".jar"))
      } else {
        Seq.empty
      }
    withJars(additionalJars)
  }

  def withJars(additionalJars: Seq[File]): CatalogRecorder = {
    new CatalogRecorder(jars ++ additionalJars.map(_.toURI.toURL))
  }

  def withSparkContext(sparkContext: SparkContext): CatalogRecorder = {
    new CatalogRecorder(jars ++ sparkContext.jars.map(new URL(_)))
  }

  lazy val catalogs: DCatalog = {
    val registrar = new DefaultCatalogRegistrar()
    val loader = URLClassLoader.newInstance(jars.toArray, getClass.getClassLoader)
    CatalogRegistrant.load(registrar, loader)
    new CatalogScanner(jars).register(registrar)
    registrar.catalog
  }
}

object CatalogRecorder {
  val logger = LoggerForCallerClass()

  def fromDir(dir: File): CatalogRecorder = {
    new CatalogRecorder(Seq.empty).withDir(dir)
  }
  def fromJars(jars: Seq[URL]): CatalogRecorder = {
    new CatalogRecorder(jars)
  }
  def fromSparkContext(sparkContext: SparkContext): CatalogRecorder = {
    new CatalogRecorder(Seq.empty).withSparkContext(sparkContext)
  }

  lazy val resourcesCatalogRecorder: CatalogRecorder = {
    fromDir(Config.jarsDir)
  }
} 

package ai.deepsense.deeplang.refl

import java.io.File
import java.net.{URL, URLClassLoader}

import ai.deepsense.commons.utils.Logging
import ai.deepsense.deeplang.catalogs.SortPriority
import ai.deepsense.deeplang.catalogs.spi.{CatalogRegistrant, CatalogRegistrar}
import ai.deepsense.deeplang.{DOperation, DOperationCategories, TypeUtils}
import org.reflections.Reflections
import org.reflections.util.ConfigurationBuilder

import scala.collection.JavaConversions._


  override def register(registrar: CatalogRegistrar): Unit = {
    logger.info(
      s"Scanning registrables. Following jars will be scanned: ${jarsUrls.mkString(";")}.")
    val scanned = scanForRegistrables().iterator
    val priorities = SortPriority.sdkInSequence
    for {
      (registrable, priority) <- scanned.zip(priorities)
    } {
      logger.debug(s"Trying to register class $registrable")
      registrable match {
        case DOperationMatcher(doperation) => registerDOperation(registrar, doperation, priority)
        case other => logger.warn(s"Only DOperation can be `@Register`ed. '$other' not supported.")
      }
    }
  }

  private def scanForRegistrables(): Set[Class[_]] = {

    val urls = thisJarURLOpt ++ jarsUrls

    if (urls.nonEmpty) {

      val configBuilder = ConfigurationBuilder
        .build(urls.toSeq: _*)
        .addClassLoader(getClass.getClassLoader)
        .setExpandSuperTypes(false)

      if (jarsUrls.nonEmpty) {
        configBuilder.addClassLoader(URLClassLoader.newInstance(jarsUrls.toArray, getClass.getClassLoader))
      }

      new Reflections(configBuilder).getTypesAnnotatedWith(classOf[Register]).toSet
    } else {
      Set()
    }

  }

  private lazy val thisJarURLOpt: Option[URL] = {
    val jarRegex = """jar:(file:.*\.jar)!.*""".r

    val url = getClass.getClassLoader.getResource(
      getClass.getCanonicalName.replaceAll("\\.", File.separator) + ".class")

    url.toString match {
      case jarRegex(jar) => Some(new URL(jar))
      case _ => None
    }
  }

  private def registerDOperation(
    registrar: CatalogRegistrar,
    operation: Class[DOperation],
    priority: SortPriority
  ): Unit = TypeUtils.constructorForClass(operation) match {
    case Some(constructor) =>
      registrar.registerOperation(
        DOperationCategories.UserDefined,
        () => TypeUtils.createInstance[DOperation](constructor),
        priority
      )
    case None => logger.error(
      s"Class $operation could not be registered." +
        "It needs to have parameterless constructor"
    )
  }

  class AssignableFromExtractor[T](targetClass: Class[T]) {
    def unapply(clazz: Class[_]): Option[Class[T]] = {
      if (targetClass.isAssignableFrom(clazz)) {
        Some(clazz.asInstanceOf[Class[T]])
      } else {
        None
      }
    }
  }

  object DOperationMatcher extends AssignableFromExtractor(classOf[DOperation])

} 

package ai.deepsense.workflowexecutor

import java.net.URL

case class ExecutionParams(
    workflowFilename: Option[String] = None,
    outputDirectoryPath: Option[String] = None,
    extraVars: Map[String, String] = Map.empty,
    interactiveMode: Boolean = false,
    messageQueueHost: Option[String] = None,
    messageQueuePort: Option[Int] = None,
    messageQueueUser: Option[String] = None,
    messageQueuePass: Option[String] = None,
    customCodeExecutorsPath: Option[String] = None,
    pythonBinaryPath: Option[String] = None,
    workflowId: Option[String] = None,
    wmAddress: Option[String] = None,
    wmUsername: Option[String] = None,
    wmPassword: Option[String] = None,
    mailParams: MailParams = MailParams(),
    notebookServerAddress: Option[URL] = None,
    datasourceServerAddress: Option[URL] = None,
    depsZip: Option[String] = None,
    userId: Option[String] = None,
    tempPath: Option[String] = None)

// It's a separate case class because in Scala 2.10 case classes cannot have more than 22 params.
case class MailParams(
    mailServerHost: Option[String] = None,
    mailServerPort: Option[Int] = None,
    mailServerUser: Option[String] = None,
    mailServerPassword: Option[String] = None,
    mailServerSender: Option[String] = None) 

package ai.deepsense.seahorse.scheduling.schedule

import java.net.URL
import java.util.concurrent.TimeUnit

import akka.util.Timeout
import spray.http.BasicHttpCredentials

import scala.concurrent.Future
import scala.concurrent.duration.FiniteDuration

import ai.deepsense.commons.models.ClusterDetails
import ai.deepsense.commons.utils.LoggerForCallerClass
import ai.deepsense.seahorse.scheduling.SchedulingManagerConfig
import ai.deepsense.workflowmanager.client.PresetsClient

private[schedule] object PresetsApi {
  private[this] val logger = LoggerForCallerClass()
  private[this] val presetsConfig = SchedulingManagerConfig.config.getConfig("presets-manager")
  private[this] lazy val presetsClient = {
    val url = new URL(presetsConfig.getString("url") + "/v1/presets/")
    logger.info(s"Creating presets client with url $url")
    val timeout = Timeout(FiniteDuration(presetsConfig.getDuration("timeout").toMillis, TimeUnit.MILLISECONDS))
    new PresetsClient(
      apiUrl = url,
      mandatoryUserId = RunWorkflowJobContext.userId,
      mandatoryUserName = RunWorkflowJobContext.userName,
      credentials = Some(BasicHttpCredentials(
        presetsConfig.getString("user"),
        presetsConfig.getString("pass"))))(
      RunWorkflowJobContext.actorSystem, timeout)
  }

  def fetchPreset(id: Long): Future[Option[ClusterDetails]] = {
    logger.info(s"Fetching preset $id")
    presetsClient.fetchPreset(id)
  }
} 

package ai.deepsense.seahorse.scheduling.schedule

import java.net.URL
import java.util.concurrent.TimeUnit

import akka.util.Timeout

import scala.concurrent.Future
import scala.concurrent.duration.FiniteDuration

import ai.deepsense.commons.models.ClusterDetails
import ai.deepsense.commons.utils.LoggerForCallerClass
import ai.deepsense.graph.nodestate.name.NodeStatusName
import ai.deepsense.models.workflows.Workflow
import ai.deepsense.seahorse.scheduling.SchedulingManagerConfig
import ai.deepsense.sessionmanager.rest.client.SessionManagerClient
import ai.deepsense.sessionmanager.service.Status

private[schedule] object SessionsApi {
  import scala.concurrent.ExecutionContext.Implicits.global

  private[this] val logger = LoggerForCallerClass()

  private[this] val sessionManagerConfig = SchedulingManagerConfig.config.getConfig("session-manager")
  private[this] lazy val sessionManagerClient = {
    val url: URL = new URL(sessionManagerConfig.getString("url") + "/v1/sessions/")
    logger.info(s"Creating sessions client with url $url.")
    val timeout = Timeout(FiniteDuration(sessionManagerConfig.getDuration("timeout").toMillis, TimeUnit.MILLISECONDS))
    new SessionManagerClient(
      mandatoryUserId = RunWorkflowJobContext.userId,
      mandatoryUserName = RunWorkflowJobContext.userName,
      apiUrl = url,
      credentials = None)(
      RunWorkflowJobContext.actorSystem, timeout)
  }

  def startSession(id: Workflow.Id, clusterDetails: ClusterDetails): Future[Unit] = {
    logger.info(s"Creating session for $id with cluster: $clusterDetails.")
    def sessionStatus(): Future[Status.Value] = sessionManagerClient.fetchSession(id).map(_.status)
    def retryUntilFinished(): Future[Unit] = sessionStatus().flatMap { status =>
      if (status == Status.Running) {
        logger.info(s"Session for $id created successfully.")
        Future.successful(())
      } else if (status == Status.Error) {
        logger.error(s"There was an error while creating a session for $id.")
        Future.failed(new RuntimeException(s"Could not create a session for $id."))
      } else {
        Thread.sleep(5000)
        logger.info(s"Session for workflow $id isn't running yet; asking for status again.")
        retryUntilFinished()
      }
    }
    sessionManagerClient.createSession(id, clusterDetails).flatMap(_ => retryUntilFinished())
  }

  def deleteSession(id: Workflow.Id): Future[Unit] = {
    logger.info(s"Deleting session for $id.")
    sessionManagerClient.deleteSession(id).map(_ => ())
  }

  def runWorkflow(id: Workflow.Id): Future[Unit] = {
    val notFinishedStatuses = Set[NodeStatusName](NodeStatusName.Draft, NodeStatusName.Running, NodeStatusName.Queued)
    def isFinished(): Future[Boolean] = sessionManagerClient.queryNodeStatuses(id).map { response =>
      logger.info(s"Running status of workflow $id: $response")
      response.nodeStatuses.exists { statusesMap =>
        statusesMap
          .filterKeys(notFinishedStatuses.contains)
          .values
          .sum == 0
      }
    }

    def retryUntilFinished(): Future[Unit] = isFinished().flatMap { finished =>
      if (finished) {
        logger.info(s"Workflow $id finished running.")
        Future.successful(())
      } else {
        Thread.sleep(5000)
        logger.info(s"Workflow $id didn't finish running yet; asking for status again.")
        retryUntilFinished()
      }
    }

    logger.info(s"Running workflow $id.")
    sessionManagerClient.launchSession(id).flatMap(_ => retryUntilFinished())
  }
} 

package ai.deepsense.seahorse.scheduling.schedule

import java.net.URL
import java.util.concurrent.TimeUnit

import akka.util.Timeout
import spray.http.BasicHttpCredentials

import scala.concurrent.Future
import scala.concurrent.duration.FiniteDuration

import ai.deepsense.commons.utils.LoggerForCallerClass
import ai.deepsense.models.workflows.{Workflow, WorkflowInfo}
import ai.deepsense.seahorse.scheduling.SchedulingManagerConfig
import ai.deepsense.workflowmanager.client.WorkflowManagerClient
import ai.deepsense.workflowmanager.model.WorkflowDescription

private[schedule] object WorkflowsApi {
  private[this] val logger = LoggerForCallerClass()
  private[this] lazy val workflowManagerConfig = SchedulingManagerConfig.config.getConfig("workflow-manager")
  private[this] lazy val workflowClient = {
    val url = new URL(workflowManagerConfig.getString("url") + "/v1/workflows/")
    logger.info(s"Creating workflows client with url $url")
    val timeout = Timeout(FiniteDuration(workflowManagerConfig.getDuration("timeout").toMillis, TimeUnit.MILLISECONDS))
    new WorkflowManagerClient(
      apiUrl = url,
      mandatoryUserId = RunWorkflowJobContext.userId,
      mandatoryUserName = RunWorkflowJobContext.userName,
      credentials = Some(BasicHttpCredentials(
        workflowManagerConfig.getString("user"),
        workflowManagerConfig.getString("pass"))))(
      RunWorkflowJobContext.actorSystem, timeout)
  }

  def getWorkflowInfo(id: Workflow.Id): Future[WorkflowInfo] = {
    workflowClient.fetchWorkflowInfo(id)
  }

  def cloneWorkflow(id: Workflow.Id, workflowInfo: WorkflowInfo): Future[Workflow.Id] = {
    logger.info(s"Cloning workflow $id")
    workflowClient.cloneWorkflow(id, WorkflowDescription(
      s"""Scheduled execution of "${workflowInfo.name}"""",
      workflowInfo.description))
  }
} 

package ai.deepsense.sessionmanager.service.sessionspawner.sparklauncher

import java.io.File
import java.net.{URI, URL}

import com.google.inject.Inject
import com.google.inject.name.Named

class SparkLauncherConfig @Inject()(
  @Named("session-executor.parameters.class-name") val className: String,
  @Named("session-executor.parameters.application-jar-path") val weJarPath: String,
  @Named("session-executor.parameters.deps-zip-path") val weDepsPath: String,
  @Named("session-executor.parameters.spark-resources-jars-dir") val sparkResourcesJarsDir: String,
  @Named("session-executor.parameters.spark-home-path") val sparkHome: String,
  @Named("session-executor.parameters.queue.port") val queuePort: Int,
  @Named("session-executor.parameters.queue.host") val queueHost: String,
  @Named("session-executor.parameters.queue.user") val queueUser: String,
  @Named("session-executor.parameters.queue.pass") val queuePass: String,
  @Named("session-executor.parameters.workflow-manager.scheme") val wmScheme: String,
  @Named("session-executor.parameters.workflow-manager.address") val wmAddress: String,
  @Named("session-executor.parameters.workflow-manager.username") val wmUsername: String,
  @Named("session-executor.parameters.workflow-manager.password") val wmPassword: String,
  @Named("session-executor.parameters.mail-server.smtp.host") val mailServerHost: String,
  @Named("session-executor.parameters.mail-server.smtp.port") val mailServerPort: Int,
  @Named("session-executor.parameters.mail-server.user") val mailServerUser: String,
  @Named("session-executor.parameters.mail-server.password") val mailServerPassword: String,
  @Named("session-executor.parameters.mail-server.sender") val mailServerSender: String,
  @Named("session-executor.parameters.notebook-server.address") val notebookServerAddress: String,
  @Named("session-executor.parameters.datasource-server.address") val datasourceManagerServerAddress: String,
  @Named("session-executor.parameters.temp-dir") val tempDir: String,
  @Named("session-executor.parameters.python-driver-binary") val pythonDriverBinary: String,
  @Named("session-executor.parameters.python-executor-binary") val pythonExecutorBinary: String
) {

  def weDepsFileName: String = {
    new File(new URI(weDepsPath).getPath).getName
  }

} 

package ai.deepsense.sessionmanager.rest.client

import java.net.URL
import java.util.UUID

import scala.concurrent.{ExecutionContext, Future}
import akka.actor.ActorSystem
import akka.util.Timeout
import spray.client.pipelining._
import spray.http.{HttpCredentials, HttpResponse}

import ai.deepsense.commons.json.envelope.{Envelope, EnvelopeJsonFormat}
import ai.deepsense.commons.models.ClusterDetails
import ai.deepsense.commons.rest.client.RestClient
import ai.deepsense.models.workflows.Workflow
import ai.deepsense.sessionmanager.rest.SessionsJsonProtocol
import ai.deepsense.sessionmanager.rest.requests.CreateSession
import ai.deepsense.sessionmanager.rest.responses.{ListSessionsResponse, NodeStatusesResponse}
import ai.deepsense.sessionmanager.service.Session

class SessionManagerClient(
    override val apiUrl: URL,
    mandatoryUserId: UUID,
    mandatoryUserName: String,
    override val credentials: Option[HttpCredentials]) (
    implicit override val as: ActorSystem,
    override val timeout: Timeout) extends RestClient with SessionsJsonProtocol {

  override def userId: Option[UUID] = Some(mandatoryUserId)
  override def userName: Option[String] = Some(mandatoryUserName)

  implicit val envelopeFormat = new EnvelopeJsonFormat[Session]("session")

  def fetchSessions(): Future[Traversable[Session]] = {
    fetchResponse[ListSessionsResponse](Get(endpointPath(""))).map(_.sessions)
  }

  def fetchSession(workflowId: Workflow.Id): Future[Session] = {
    fetchResponse[Envelope[Session]](Get(endpointPath(workflowId.toString))).map(_.content)
  }

  def createSession(workflowId: Workflow.Id, cluster: ClusterDetails): Future[Session] = {
    fetchResponse[Envelope[Session]](Post(endpointPath(""), CreateSession(workflowId, cluster))).map(_.content)
  }

  def deleteSession(workflowId: Workflow.Id): Future[HttpResponse] = {
    fetchHttpResponse(Delete(endpointPath(workflowId.toString)))
  }

  def launchSession(workflowId: Workflow.Id): Future[HttpResponse] = {
    fetchHttpResponse(Post(endpointPath(workflowId.toString)))
  }

  def queryNodeStatuses(workflowId: Workflow.Id): Future[NodeStatusesResponse] = {
    fetchResponse[NodeStatusesResponse](Get(endpointPath(s"${workflowId.toString}/status")))
  }
} 

package org.platanios.tensorflow.data

import com.typesafe.scalalogging.Logger

import java.io.IOException
import java.net.URL
import java.nio.file.{Files, Path}

import scala.collection.compat.immutable.LazyList
import scala.io.Source
import scala.util.matching.Regex


trait Loader {
  protected val logger: Logger

  protected val googleDriveConfirmTokenRegex: Regex = {
    """<a id="uc-download-link".*href="/uc\?export=download&amp;(confirm=.*)&amp;id=.*">Download anyway</a>""".r
  }

  def maybeDownload(path: Path, url: String, bufferSize: Int = 8192): Boolean = {
    if (Files.exists(path)) {
      false
    } else {
      try {
        logger.info(s"Downloading file '$url'.")
        Files.createDirectories(path.getParent)
        download(path, url, bufferSize)

        // Small hack to deal with downloading large Google Drive files.
        if (Files.size(path) < 1024 * 1024 && url.contains("drive.google.com")) {
          val content = Source.fromFile(path.toFile).getLines().mkString("\n")
          googleDriveConfirmTokenRegex.findFirstMatchIn(content) match {
            case Some(confirmToken) => download(path, s"$url&${confirmToken.group(1)}", bufferSize)
            case None => ()
          }
        }

        logger.info(s"Downloaded file '$url'.")
        true
      } catch {
        case e: IOException =>
          logger.error(s"Could not download file '$url'", e)
          throw e
      }
    }
  }

  protected def download(path: Path, url: String, bufferSize: Int = 8192): Unit = {
    val connection = new URL(url).openConnection()
    val contentLength = connection.getContentLengthLong
    val inputStream = connection.getInputStream
    val outputStream = Files.newOutputStream(path)
    val buffer = new Array[Byte](bufferSize)
    var progress = 0L
    var progressLogTime = System.currentTimeMillis
    LazyList.continually(inputStream.read(buffer)).takeWhile(_ != -1).foreach(numBytes => {
      outputStream.write(buffer, 0, numBytes)
      progress += numBytes
      val time = System.currentTimeMillis
      if (time - progressLogTime >= 1e4) {
        if (contentLength > 0) {
          val numBars = Math.floorDiv(10 * progress, contentLength).toInt
          logger.info(s"[${"=" * numBars}${" " * (10 - numBars)}] $progress / $contentLength bytes downloaded.")
          progressLogTime = time
        } else {
          logger.info(s"$progress bytes downloaded.")
          progressLogTime = time
        }
      }
    })
    outputStream.close()
  }
} 

package org.scaladebugger.docs.layouts.partials.common

import java.net.URL
import java.util.Calendar

import org.scaladebugger.docs.styles.PageStyle

import scalatags.Text.all._


object Footer {
  def apply(authorName: String, authorUrl: URL, startYear: Int): Modifier = {
    tag("footer")(PageStyle.footerCls, PageStyle.sectionDark)(
      div(PageStyle.footerContent)(
        span(
          raw("Site contents "),
          i(`class` := "fa fa-copyright", attr("aria-hidden") := "true"),
          raw(" "),
          a(href := authorUrl.toString)(authorName),
          raw(s", $startYear-${Calendar.getInstance().get(Calendar.YEAR)}")
        )
      )
    )
  }
} 

package org.scaladebugger.docs.layouts

import java.net.URL

import org.scaladebugger.docs.layouts.partials.common._
import org.scaladebugger.docs.layouts.partials.common.vendor._
import org.scaladebugger.docs.styles.{PageStyle, TabsStyle, TopbarNavStyle}
import org.senkbeil.grus.layouts.{Context, Page}

import scalatags.Text.all._
import org.scaladebugger.docs.styles.Implicits._

import scalatags.Text


abstract class SitePage(
  val selectedMenuItems: Seq[String] = Nil,
  val syntaxHighlightTheme: String = "agate"
) extends Page {
  private val pixelId: String = "1622567071086905"

  override protected def preHeadContent(context: Context): Seq[Modifier] =
    super.preHeadContent(context) ++ Seq(
      FontAwesomeCSS(),
      HighlightCSS(theme = syntaxHighlightTheme),
      PageStyle.styleSheetText.toStyleTag,
      TopbarNavStyle.styleSheetText.toStyleTag,
      TabsStyle.styleSheetText.toStyleTag
    )

  override protected def preBodyContent(context: Context): Seq[Modifier] =
    super.preBodyContent(context) ++ Seq(
      Header(
        MainMenuBar(
          MainMenuLogo(),
          MainMenu(context.mainMenuItems: _*)
        )
      )
    )

  override protected def postHeadContent(context: Context): Seq[Text.all.Modifier] =
    super.postHeadContent(context) ++ Seq(
      script(
        raw(s"""
           |  !function(f,b,e,v,n,t,s)
           |  {if(f.fbq)return;n=f.fbq=function(){n.callMethod?
           |  n.callMethod.apply(n,arguments):n.queue.push(arguments)};
           |  if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
           |  n.queue=[];t=b.createElement(e);t.async=!0;
           |  t.src=v;s=b.getElementsByTagName(e)[0];
           |  s.parentNode.insertBefore(t,s)}(window, document,'script',
           |  'https://connect.facebook.net/en_US/fbevents.js');
           |  fbq('init', '$pixelId');
           |  fbq('track', 'PageView');
        """.stripMargin)
      ),
      tag("noscript")(img(
        height := "1",
        width := "1",
        style := "display: none;",
        src := s"https://www.facebook.com/tr?id=$pixelId&ev=PageView&noscript=1"
      )())
    )

  override protected def postBodyContent(context: Context): Seq[Modifier] =
    Seq(
      Footer(
        authorName = "Chip Senkbeil",
        authorUrl = new URL("https://chipsenkbeil.com/"),
        startYear = 2015
      ),
      ClipboardJS(),
      HighlightJS(),
      ClipboardJSInit(),
      HighlightJSInit()
    ) ++ super.postBodyContent(context)
} 

package io.amient.affinity.core.util

import java.net.URL

import org.scalacheck.Gen
import org.scalatest.prop.PropertyChecks
import org.scalatest.{Matchers, PropSpec}

class TimeCryptoProofSpec extends PropSpec with PropertyChecks with Matchers {

  val salts: Gen[Array[Byte]] = for {
    _ <- Gen.choose(Integer.MIN_VALUE + 1, Integer.MAX_VALUE)
  } yield TimeCryptoProof.generateSalt

  val hexSalts: Gen[String] = for {
    salt <- salts
  } yield TimeCryptoProof.toHex(salt)


  property("slat bytes to hex conversion is reversible") {
    forAll(salts) { salt =>
      val hexSalt = TimeCryptoProof.toHex(salt)
      val reversedSalt = TimeCryptoProof.fromHex(hexSalt)
      reversedSalt should equal(salt)
    }
  }

  property("CryptoProofSHA256 holds for all salts and args") {
    forAll(salts, Gen.alphaStr) { (salt, arg) =>

      val crypto = new TimeCryptoProofSHA256(salt)
      val signature: String = crypto.sign(arg)

      TimeCryptoProof.toHex(salt)
      val hexProof = new TimeCryptoProofSHA256(salt)
      val hexSignature = hexProof.sign(arg)
      signature should equal(hexSignature)
      try {
        assert(crypto.verify(signature, arg) == true)
        assert(hexProof.verify(signature, arg) == true)
        assert(crypto.verify(hexSignature, arg) == true)
        assert(hexProof.verify(hexSignature, arg) == true)
      } catch {
        case e: Throwable => e.printStackTrace(); throw e
      }
    }
  }

  property("example function is consistent with the implementation") {
    forAll(hexSalts, Gen.alphaStr) { (hexSalt, arg) =>
      val crypto = new TimeCryptoProofSHA256(hexSalt)
      try {
        assert(crypto.sign(arg) == crypto.timeBasedHash(arg, hexSalt, 0) == true)
        assert(crypto.verify(crypto.sign(arg), arg) == true)
        assert(crypto.verify(crypto.timeBasedHash(arg, hexSalt, 0), arg) == true)
      } catch {
        case e: Throwable => e.printStackTrace(); throw e
      }
    }
  }

  property("example function generates different signatures for different salts") {
    forAll(hexSalts, hexSalts) { case (salt1, salt2) =>
      whenever(salt1 != salt2) {
        val crypto1 = new TimeCryptoProofSHA256(salt1)
        val crypto2 = new TimeCryptoProofSHA256(salt2)
        val url = new URL("https://example.com/xyz?param=123456")
        val urlNoQuery = new URL("https://example.com/xyz")
        val sig1 = crypto1.sign(url.getPath)
        val sig1NoQuery = crypto1.sign(urlNoQuery.getPath)
        val sig2 = crypto2.sign(url.toString)
        val sig2NoQuery = crypto2.sign(urlNoQuery.toString)
        assert(sig1 != sig2)
        assert(sig1NoQuery != sig2NoQuery)
      }
    }
  }

} 

package pl.matisoft.swagger

import java.net.URL
import javax.inject.{Inject, Provider}

import com.wordnik.swagger.config.{FilterFactory, ScannerFactory, ConfigFactory}
import com.wordnik.swagger.core.SwaggerContext
import com.wordnik.swagger.core.filter.SwaggerSpecFilter
import com.wordnik.swagger.reader.ClassReaders
import play.api.inject.ApplicationLifecycle
import play.api.{Logger, Application}
import play.api.routing.Router
import play.modules.swagger.ApiListingCache
import scala.concurrent.ExecutionContext.Implicits.global

import scala.concurrent.Future

class SwaggerPluginProvider extends Provider[SwaggerPlugin] {

  val logger = Logger("swagger")

  @Inject
  private var router: Router = _

  @Inject
  private var app: Application = _

  @Inject
  private var lifecycle: ApplicationLifecycle = _

  override def get(): SwaggerPlugin = {
    lifecycle.addStopHook(() => Future {
      onStop()
    })

    onStart()
  }

  def onStart(): SwaggerPlugin = {
    val config = app.configuration
    logger.info("Swagger - starting initialisation...")

    val apiVersion = config.getString("api.version") match {
      case None => "beta"
      case Some(value) => value
    }

    val basePath = config.getString("swagger.api.basepath")
        .filter(path => !path.isEmpty)
        .map(getPathUrl(_))
        .getOrElse("http://localhost:9000")

    SwaggerContext.registerClassLoader(app.classloader)
    ConfigFactory.config.setApiVersion(apiVersion)
    ConfigFactory.config.setBasePath(basePath)
    ScannerFactory.setScanner(new PlayApiScanner(Option(router)))
    ClassReaders.reader = Some(new PlayApiReader(Option(router)))

    app.configuration.getString("swagger.filter")
      .filter(p => !p.isEmpty)
      .foreach(loadFilter(_))

    val docRoot = ""
    ApiListingCache.listing(docRoot)

    logger.info("Swagger - initialization done.")
    new SwaggerPlugin()
  }

  def onStop() {
    ApiListingCache.cache = None
    logger.info("Swagger - stopped.")
  }

  def loadFilter(filterClass: String): Unit = {
    try {
      FilterFactory.filter = SwaggerContext.loadClass(filterClass).newInstance.asInstanceOf[SwaggerSpecFilter]
      logger.info(s"Setting swagger.filter to $filterClass")
    } catch {
      case ex: Exception =>logger.error(s"Failed to load filter:$filterClass", ex)
    }
  }

  def getPathUrl(path: String): String = {
    try {
      val basePathUrl = new URL(path)
      logger.info(s"Basepath configured as:$path")
      path
    } catch {
      case ex: Exception =>
        logger.error(s"Misconfiguration - basepath not a valid URL:$path. Swagger abandoning initialisation!")
        throw ex
    }
  }

} 

package scalaDemo

import java.io.File
import java.net.URL


object GetResourcePathDemo extends  App{

  val file=getClass.getClassLoader.getResource("hdfs-site.xml").getFile
  println(file)
  //Thread.currentThread().getContextClassLoader,可以获取当前线程的引用,getContextClassLoader用来获取线程的上下文类加载器
  //getResource得到的是当前类class文件的URI目录,不包括自己
  val url = Thread.currentThread.getContextClassLoader.getResource("hdfs-site.xml")
  // final URL testLocalResource =ThreadLocal.class.getClassLoader().getResource("core-site.xml");
  System.out.println("==========" + url)
  val HADOOP_HOME = "/opt/cloudera/parcels/CDH/lib/hadoop/"
  val HADOOP_CONF_DIR = "/etc/hadoop/conf/"
  // //getResource得到的是当前类class文件的URI目录,不包括自己/根目录
  val keyStorePath = new File(this.getClass.getResource("/keystore").toURI).getAbsolutePath

  //String path=Thread.currentThread().getContextClassLoader().getResource("core-site.xml").toURI().getPath();
  val path = Thread.currentThread.getContextClassLoader.getResource("hdfs-site.xml").toURI.getPath
  //final String HADOOP_CONF_HDFS_SITE = ConfigurationManager.getHadoopConfDir() + "/hdfs-site.xml";
  //toURI此方法不会自动转义 URL 中的非法字符,
  //将抽象路径名转换为 URL：首先通过 toURI 方法将其转换为 URI，然后通过 URI.toURL 方法将 URI 装换为 URL
  System.out.println("===" + new File(path).getParent)
  System.out.println(path + "===" + new File(path).getAbsoluteFile.toURI.toURL + "===" + new File(path).getAbsoluteFile.toURI)
} 

package org.apache.spark.sql.hive

import java.net.URL

import org.apache.spark.SparkFunSuite


class ClasspathDependenciesSuite extends SparkFunSuite {
  //ClassLoader就是用来动态加载class文件到内存当中用
  private val classloader = this.getClass.getClassLoader

  private def assertLoads(classname: String): Unit = {
    val resourceURL: URL = Option(findResource(classname)).getOrElse {
      fail(s"Class $classname not found as ${resourceName(classname)}")
    }

    logInfo(s"Class $classname at $resourceURL")
    classloader.loadClass(classname)
  }

  private def assertLoads(classes: String*): Unit = {
    classes.foreach(assertLoads)
  }

  private def findResource(classname: String): URL = {
    val resource = resourceName(classname)
    classloader.getResource(resource)
  }

  private def resourceName(classname: String): String = {
    classname.replace(".", "/") + ".class"
  }

  private def assertClassNotFound(classname: String): Unit = {
    Option(findResource(classname)).foreach { resourceURL =>
      fail(s"Class $classname found at $resourceURL")
    }

    intercept[ClassNotFoundException] {
      classloader.loadClass(classname)
    }
  }

  private def assertClassNotFound(classes: String*): Unit = {
    classes.foreach(assertClassNotFound)
  }

  private val KRYO = "com.esotericsoftware.kryo.Kryo"

  private val SPARK_HIVE = "org.apache.hive."
  private val SPARK_SHADED = "org.spark-project.hive.shaded."

  test("shaded Protobuf") {
    assertLoads(SPARK_SHADED + "com.google.protobuf.ServiceException")
  }

  test("hive-common") {
    assertLoads("org.apache.hadoop.hive.conf.HiveConf")
  }

  test("hive-exec") {
    assertLoads("org.apache.hadoop.hive.ql.CommandNeedRetryException")
  }

  private val STD_INSTANTIATOR = "org.objenesis.strategy.StdInstantiatorStrategy"

  test("unshaded kryo") {
    assertLoads(KRYO, STD_INSTANTIATOR)
  }

  test("Forbidden Dependencies") {
    assertClassNotFound(
      SPARK_HIVE + KRYO,
      SPARK_SHADED + KRYO,
      "org.apache.hive." + KRYO,
      "com.esotericsoftware.shaded." + STD_INSTANTIATOR,
      SPARK_HIVE + "com.esotericsoftware.shaded." + STD_INSTANTIATOR,
      "org.apache.hive.com.esotericsoftware.shaded." + STD_INSTANTIATOR
    )
  }

  test("parquet-hadoop-bundle") {
    assertLoads(
      "parquet.hadoop.ParquetOutputFormat",
      "parquet.hadoop.ParquetInputFormat"
    )
  }
} 

package org.apache.spark.util

import java.net.{URLClassLoader, URL}
import java.util.Enumeration
import java.util.concurrent.ConcurrentHashMap

import scala.collection.JavaConversions._


  private val locks = new ConcurrentHashMap[String, Object]()

  override def loadClass(name: String, resolve: Boolean): Class[_] = {
    var lock = locks.get(name)
    if (lock == null) {
      val newLock = new Object()
      lock = locks.putIfAbsent(name, newLock)
      if (lock == null) {
        lock = newLock
      }
    }

    lock.synchronized {
      try {
        super.loadClass(name, resolve)
      } catch {
        case e: ClassNotFoundException =>
          parentClassLoader.loadClass(name, resolve)
      }
    }
  }

  override def getResource(name: String): URL = {
    val url = super.findResource(name)
    val res = if (url != null) url else parentClassLoader.getResource(name)
    res
  }

  override def getResources(name: String): Enumeration[URL] = {
    val urls = super.findResources(name)
    val res =
      if (urls != null && urls.hasMoreElements()) {
        urls
      } else {
        parentClassLoader.getResources(name)
      }
    res
  }

  override def addURL(url: URL) {
    super.addURL(url)
  }

}