java.net.URLClassLoader Scala Examples

package org.apache.spark.util

import java.net.{URL, URLClassLoader}
import java.util.Enumeration

import scala.collection.JavaConverters._


private[spark] class ChildFirstURLClassLoader(urls: Array[URL], parent: ClassLoader)
  extends MutableURLClassLoader(urls, null) {

  private val parentClassLoader = new ParentClassLoader(parent)

  override def loadClass(name: String, resolve: Boolean): Class[_] = {
    try {
      super.loadClass(name, resolve)
    } catch {
      case e: ClassNotFoundException =>
        parentClassLoader.loadClass(name, resolve)
    }
  }

  override def getResource(name: String): URL = {
    val url = super.findResource(name)
    val res = if (url != null) url else parentClassLoader.getResource(name)
    res
  }

  override def getResources(name: String): Enumeration[URL] = {
    val childUrls = super.findResources(name).asScala
    val parentUrls = parentClassLoader.getResources(name).asScala
    (childUrls ++ parentUrls).asJavaEnumeration
  }

  override def addURL(url: URL) {
    super.addURL(url)
  }

} 

package org.apache.spark.util

import java.net.{URLClassLoader, URL}
import java.util.Enumeration
import java.util.concurrent.ConcurrentHashMap

import scala.collection.JavaConversions._


  private val locks = new ConcurrentHashMap[String, Object]()

  override def loadClass(name: String, resolve: Boolean): Class[_] = {
    var lock = locks.get(name)
    if (lock == null) {
      val newLock = new Object()
      lock = locks.putIfAbsent(name, newLock)
      if (lock == null) {
        lock = newLock
      }
    }

    lock.synchronized {
      try {
        super.loadClass(name, resolve)
      } catch {
        case e: ClassNotFoundException =>
          parentClassLoader.loadClass(name, resolve)
      }
    }
  }

  override def getResource(name: String): URL = {
    val url = super.findResource(name)
    val res = if (url != null) url else parentClassLoader.getResource(name)
    res
  }

  override def getResources(name: String): Enumeration[URL] = {
    val urls = super.findResources(name)
    val res =
      if (urls != null && urls.hasMoreElements()) {
        urls
      } else {
        parentClassLoader.getResources(name)
      }
    res
  }

  override def addURL(url: URL) {
    super.addURL(url)
  }

} 

package org.apache.spark.util

import java.net.URLClassLoader

import org.apache.spark.{SparkContext, SparkException, SparkFunSuite, TestUtils}

class MutableURLClassLoaderSuite extends SparkFunSuite {

  val urls2 = List(TestUtils.createJarWithClasses(
      classNames = Seq("FakeClass1", "FakeClass2", "FakeClass3"),
      toStringValue = "2")).toArray
  val urls = List(TestUtils.createJarWithClasses(
      classNames = Seq("FakeClass1"),
      classNamesWithBase = Seq(("FakeClass2", "FakeClass3")), // FakeClass3 is in parent
      toStringValue = "1",
      classpathUrls = urls2)).toArray

  test("child first") {
    val parentLoader = new URLClassLoader(urls2, null)
    val classLoader = new ChildFirstURLClassLoader(urls, parentLoader)
    val fakeClass = classLoader.loadClass("FakeClass2").newInstance()
    val fakeClassVersion = fakeClass.toString
    assert(fakeClassVersion === "1")
    val fakeClass2 = classLoader.loadClass("FakeClass2").newInstance()
    assert(fakeClass.getClass === fakeClass2.getClass)
  }

  test("parent first") {
    val parentLoader = new URLClassLoader(urls2, null)
    val classLoader = new MutableURLClassLoader(urls, parentLoader)
    val fakeClass = classLoader.loadClass("FakeClass1").newInstance()
    val fakeClassVersion = fakeClass.toString
    assert(fakeClassVersion === "2")
    val fakeClass2 = classLoader.loadClass("FakeClass1").newInstance()
    assert(fakeClass.getClass === fakeClass2.getClass)
  }

  test("child first can fall back") {
    val parentLoader = new URLClassLoader(urls2, null)
    val classLoader = new ChildFirstURLClassLoader(urls, parentLoader)
    val fakeClass = classLoader.loadClass("FakeClass3").newInstance()
    val fakeClassVersion = fakeClass.toString
    assert(fakeClassVersion === "2")
  }

  test("child first can fail") {
    val parentLoader = new URLClassLoader(urls2, null)
    val classLoader = new ChildFirstURLClassLoader(urls, parentLoader)
    intercept[java.lang.ClassNotFoundException] {
      classLoader.loadClass("FakeClassDoesNotExist").newInstance()
    }
  }

  test("driver sets context class loader in local mode") {
    // Test the case where the driver program sets a context classloader and then runs a job
    // in local mode. This is what happens when ./spark-submit is called with "local" as the
    // master.
    val original = Thread.currentThread().getContextClassLoader

    val className = "ClassForDriverTest"
    val jar = TestUtils.createJarWithClasses(Seq(className))
    val contextLoader = new URLClassLoader(Array(jar), Utils.getContextOrSparkClassLoader)
    Thread.currentThread().setContextClassLoader(contextLoader)

    val sc = new SparkContext("local", "driverLoaderTest")

    try {
      sc.makeRDD(1 to 5, 2).mapPartitions { x =>
        val loader = Thread.currentThread().getContextClassLoader
        Class.forName(className, true, loader).newInstance()
        Seq().iterator
      }.count()
    }
    catch {
      case e: SparkException if e.getMessage.contains("ClassNotFoundException") =>
        fail("Local executor could not find class", e)
      case t: Throwable => fail("Unexpected exception ", t)
    }

    sc.stop()
    Thread.currentThread().setContextClassLoader(original)
  }
} 

package ai.deepsense.deeplang

import java.io.File
import java.net.{URL, URLClassLoader}

import ai.deepsense.commons.utils.LoggerForCallerClass
import ai.deepsense.deeplang.catalogs.DCatalog
import ai.deepsense.deeplang.catalogs.spi.CatalogRegistrant
import ai.deepsense.deeplang.catalogs.spi.CatalogRegistrar.DefaultCatalogRegistrar
import ai.deepsense.deeplang.refl.CatalogScanner
import org.apache.spark.SparkContext


class CatalogRecorder private (jars: Seq[URL]) {
  def withDir(jarsDir: File): CatalogRecorder = {
    val additionalJars =
      if (jarsDir.exists && jarsDir.isDirectory) {
        jarsDir.listFiles().toSeq.filter(f => f.isFile && f.getName.endsWith(".jar"))
      } else {
        Seq.empty
      }
    withJars(additionalJars)
  }

  def withJars(additionalJars: Seq[File]): CatalogRecorder = {
    new CatalogRecorder(jars ++ additionalJars.map(_.toURI.toURL))
  }

  def withSparkContext(sparkContext: SparkContext): CatalogRecorder = {
    new CatalogRecorder(jars ++ sparkContext.jars.map(new URL(_)))
  }

  lazy val catalogs: DCatalog = {
    val registrar = new DefaultCatalogRegistrar()
    val loader = URLClassLoader.newInstance(jars.toArray, getClass.getClassLoader)
    CatalogRegistrant.load(registrar, loader)
    new CatalogScanner(jars).register(registrar)
    registrar.catalog
  }
}

object CatalogRecorder {
  val logger = LoggerForCallerClass()

  def fromDir(dir: File): CatalogRecorder = {
    new CatalogRecorder(Seq.empty).withDir(dir)
  }
  def fromJars(jars: Seq[URL]): CatalogRecorder = {
    new CatalogRecorder(jars)
  }
  def fromSparkContext(sparkContext: SparkContext): CatalogRecorder = {
    new CatalogRecorder(Seq.empty).withSparkContext(sparkContext)
  }

  lazy val resourcesCatalogRecorder: CatalogRecorder = {
    fromDir(Config.jarsDir)
  }
} 

package ai.deepsense.deeplang.refl

import java.io.File
import java.net.{URL, URLClassLoader}

import ai.deepsense.commons.utils.Logging
import ai.deepsense.deeplang.catalogs.SortPriority
import ai.deepsense.deeplang.catalogs.spi.{CatalogRegistrant, CatalogRegistrar}
import ai.deepsense.deeplang.{DOperation, DOperationCategories, TypeUtils}
import org.reflections.Reflections
import org.reflections.util.ConfigurationBuilder

import scala.collection.JavaConversions._


  override def register(registrar: CatalogRegistrar): Unit = {
    logger.info(
      s"Scanning registrables. Following jars will be scanned: ${jarsUrls.mkString(";")}.")
    val scanned = scanForRegistrables().iterator
    val priorities = SortPriority.sdkInSequence
    for {
      (registrable, priority) <- scanned.zip(priorities)
    } {
      logger.debug(s"Trying to register class $registrable")
      registrable match {
        case DOperationMatcher(doperation) => registerDOperation(registrar, doperation, priority)
        case other => logger.warn(s"Only DOperation can be `@Register`ed. '$other' not supported.")
      }
    }
  }

  private def scanForRegistrables(): Set[Class[_]] = {

    val urls = thisJarURLOpt ++ jarsUrls

    if (urls.nonEmpty) {

      val configBuilder = ConfigurationBuilder
        .build(urls.toSeq: _*)
        .addClassLoader(getClass.getClassLoader)
        .setExpandSuperTypes(false)

      if (jarsUrls.nonEmpty) {
        configBuilder.addClassLoader(URLClassLoader.newInstance(jarsUrls.toArray, getClass.getClassLoader))
      }

      new Reflections(configBuilder).getTypesAnnotatedWith(classOf[Register]).toSet
    } else {
      Set()
    }

  }

  private lazy val thisJarURLOpt: Option[URL] = {
    val jarRegex = """jar:(file:.*\.jar)!.*""".r

    val url = getClass.getClassLoader.getResource(
      getClass.getCanonicalName.replaceAll("\\.", File.separator) + ".class")

    url.toString match {
      case jarRegex(jar) => Some(new URL(jar))
      case _ => None
    }
  }

  private def registerDOperation(
    registrar: CatalogRegistrar,
    operation: Class[DOperation],
    priority: SortPriority
  ): Unit = TypeUtils.constructorForClass(operation) match {
    case Some(constructor) =>
      registrar.registerOperation(
        DOperationCategories.UserDefined,
        () => TypeUtils.createInstance[DOperation](constructor),
        priority
      )
    case None => logger.error(
      s"Class $operation could not be registered." +
        "It needs to have parameterless constructor"
    )
  }

  class AssignableFromExtractor[T](targetClass: Class[T]) {
    def unapply(clazz: Class[_]): Option[Class[T]] = {
      if (targetClass.isAssignableFrom(clazz)) {
        Some(clazz.asInstanceOf[Class[T]])
      } else {
        None
      }
    }
  }

  object DOperationMatcher extends AssignableFromExtractor(classOf[DOperation])

} 

package org.scaladebugger.test.classes

import java.net.URLClassLoader

 {
    val testJarUrl = this.getClass.getResource("/TestJar.jar")
    val urlClassLoader = new URLClassLoader(Array(testJarUrl), null)

    val className = "org.scaladebugger.test.jar.CustomClass"
    val klass = urlClassLoader.loadClass(className)
    val instance = klass.newInstance() // Causes class to be loaded

    println(s"Loading classes from ${testJarUrl.getPath}")
    println(s"Class[CustomClass]: $klass")
    println(s"CustomClass: $instance")
  }

  // Suggest garbage collecting our out-of-scope classloader to unload the
  // associated class
  System.gc()

  while (true) { Thread.sleep(1000) }
} 

package org.apache.spark.util

import java.net.{URLClassLoader, URL}
import java.util.Enumeration
import java.util.concurrent.ConcurrentHashMap

import scala.collection.JavaConversions._


  private val locks = new ConcurrentHashMap[String, Object]()

  override def loadClass(name: String, resolve: Boolean): Class[_] = {
    var lock = locks.get(name)
    if (lock == null) {
      val newLock = new Object()
      lock = locks.putIfAbsent(name, newLock)
      if (lock == null) {
        lock = newLock
      }
    }

    lock.synchronized {
      try {
        super.loadClass(name, resolve)
      } catch {
        case e: ClassNotFoundException =>
          parentClassLoader.loadClass(name, resolve)
      }
    }
  }

  override def getResource(name: String): URL = {
    val url = super.findResource(name)
    val res = if (url != null) url else parentClassLoader.getResource(name)
    res
  }

  override def getResources(name: String): Enumeration[URL] = {
    val urls = super.findResources(name)
    val res =
      if (urls != null && urls.hasMoreElements()) {
        urls
      } else {
        parentClassLoader.getResources(name)
      }
    res
  }

  override def addURL(url: URL) {
    super.addURL(url)
  }

} 

package org.apache.spark.util

import java.net.URLClassLoader

import org.apache.spark.{SparkContext, SparkException, SparkFunSuite, TestUtils}

class MutableURLClassLoaderSuite extends SparkFunSuite {

  val urls2 = List(TestUtils.createJarWithClasses(
      classNames = Seq("FakeClass1", "FakeClass2", "FakeClass3"),
      toStringValue = "2")).toArray
  val urls = List(TestUtils.createJarWithClasses(
      classNames = Seq("FakeClass1"),
      classNamesWithBase = Seq(("FakeClass2", "FakeClass3")), // FakeClass3 is in parent
      toStringValue = "1",
      classpathUrls = urls2)).toArray

  test("child first") {//第一个子类
    val parentLoader = new URLClassLoader(urls2, null)
    val classLoader = new ChildFirstURLClassLoader(urls, parentLoader)
    val fakeClass = classLoader.loadClass("FakeClass2").newInstance()
    val fakeClassVersion = fakeClass.toString
    assert(fakeClassVersion === "1")
    val fakeClass2 = classLoader.loadClass("FakeClass2").newInstance()
    assert(fakeClass.getClass === fakeClass2.getClass)
  }

  test("parent first") {//第一个父类
    val parentLoader = new URLClassLoader(urls2, null)
    val classLoader = new MutableURLClassLoader(urls, parentLoader)
    val fakeClass = classLoader.loadClass("FakeClass1").newInstance()
    val fakeClassVersion = fakeClass.toString
    assert(fakeClassVersion === "2")
    val fakeClass2 = classLoader.loadClass("FakeClass1").newInstance()
    assert(fakeClass.getClass === fakeClass2.getClass)
  }

  test("child first can fall back") {//子第一次可以倒退
    val parentLoader = new URLClassLoader(urls2, null)
    val classLoader = new ChildFirstURLClassLoader(urls, parentLoader)
    val fakeClass = classLoader.loadClass("FakeClass3").newInstance()
    val fakeClassVersion = fakeClass.toString
    assert(fakeClassVersion === "2")
  }

  test("child first can fail") {//子第一次可以失败
    val parentLoader = new URLClassLoader(urls2, null)
    val classLoader = new ChildFirstURLClassLoader(urls, parentLoader)
    intercept[java.lang.ClassNotFoundException] {
      classLoader.loadClass("FakeClassDoesNotExist").newInstance()
    }
  }
  //驱动程序在本地模式下设置上下文类加载程序
  test("driver sets context class loader in local mode") {
    // Test the case where the driver program sets a context classloader and then runs a job
    // in local mode. This is what happens when ./spark-submit is called with "local" as the
    // master.
    //测试驱动程序设置上下文类加载器然后运行作业的情况
    //在本地模式,当./spark-submit以“local”作为调用时，会发生什么master
    val original = Thread.currentThread().getContextClassLoader

    val className = "ClassForDriverTest"
    val jar = TestUtils.createJarWithClasses(Seq(className))
    val contextLoader = new URLClassLoader(Array(jar), Utils.getContextOrSparkClassLoader)
    Thread.currentThread().setContextClassLoader(contextLoader)

    val sc = new SparkContext("local", "driverLoaderTest")

    try {
      sc.makeRDD(1 to 5, 2).mapPartitions { x =>
        val loader = Thread.currentThread().getContextClassLoader
        // scalastyle:off classforname
        Class.forName(className, true, loader).newInstance()
        // scalastyle:on classforname
        Seq().iterator
      }.count()
    }
    catch {
      case e: SparkException if e.getMessage.contains("ClassNotFoundException") =>
        fail("Local executor could not find class", e)
      case t: Throwable => fail("Unexpected exception ", t)
    }

    sc.stop()
    Thread.currentThread().setContextClassLoader(original)
  }
} 

package io.hydrosphere.mist.job

import java.io.File
import java.net.URLClassLoader

import io.hydrosphere.mist.core.CommonData.Action
import io.hydrosphere.mist.job
import io.hydrosphere.mist.utils.{Err, TryLoad}

class FunctionInstanceLoader(val classLoader: ClassLoader) {

  def loadFnInstance(className: String, action: Action): TryLoad[JvmFunctionInstance] = {
    loadClass(className).flatMap({
      case clz if FunctionInstance.isInstance(clz) =>
        TryLoad(job.FunctionInstance.loadObject(clz))
          .orElse(TryLoad(job.FunctionInstance.loadClass(clz)))
      case _ =>
        val e = new IllegalStateException(s"Can not instantiate job class: $className for action $action")
        Err(e)
    })
  }

  private def loadClass(name: String): TryLoad[Class[_]] =
    TryLoad(Class.forName(name, false, classLoader))
}

object FunctionInstanceLoader {

  val Common = new FunctionInstanceLoader(this.getClass.getClassLoader)

  def fromJar(file: File): FunctionInstanceLoader = {
    val url = file.toURI.toURL
    val loader = new URLClassLoader(Array(url), getClass.getClassLoader)
    new FunctionInstanceLoader(loader)
  }

} 

package org.apache.spark.util

import java.net.{URL, URLClassLoader}
import java.util.Enumeration

import scala.collection.JavaConverters._


private[spark] class ChildFirstURLClassLoader(urls: Array[URL], parent: ClassLoader)
  extends MutableURLClassLoader(urls, null) {

  private val parentClassLoader = new ParentClassLoader(parent)

  override def loadClass(name: String, resolve: Boolean): Class[_] = {
    try {
      super.loadClass(name, resolve)
    } catch {
      case e: ClassNotFoundException =>
        parentClassLoader.loadClass(name, resolve)
    }
  }

  override def getResource(name: String): URL = {
    val url = super.findResource(name)
    val res = if (url != null) url else parentClassLoader.getResource(name)
    res
  }

  override def getResources(name: String): Enumeration[URL] = {
    val childUrls = super.findResources(name).asScala
    val parentUrls = parentClassLoader.getResources(name).asScala
    (childUrls ++ parentUrls).asJavaEnumeration
  }

  override def addURL(url: URL) {
    super.addURL(url)
  }

} 

package io.github.lhotari.akka.http.health

import java.io.ByteArrayOutputStream
import java.lang.System.getProperty
import java.net.{URL, URLClassLoader}

import org.apache.commons.io.IOUtils

import scala.collection.JavaConverters._
import scala.reflect.runtime.universe._

case class ProcessResult(retval: Integer, output: String)

trait ProcessSpawner {
  lazy val classpath = resolveClassPath()
  val sep = getProperty("file.separator")
  val javaExecutablePath = getProperty("java.home") + sep + "bin" + sep + "java"

  private def resolveClassPath() = {
    getClass.getClassLoader match {
      case urlClassLoader: URLClassLoader =>
        urlClassLoader.getURLs.collect {
          case url: URL => url.getFile
        }.mkString(getProperty("path.separator"))
      case _ =>
        getProperty("java.class.path")
    }
  }

  def executeInSeparateProcess[T](mainClassType: T, maxMemoryMB: Integer = 100, extraJvmOpts: Seq[String] = Nil, args: Seq[String] = Nil)(implicit tag: WeakTypeTag[T]): ProcessResult = {
    val className = tag.tpe.termSymbol.fullName
    val processBuilder = new ProcessBuilder(javaExecutablePath).redirectErrorStream(true)
    val commands = processBuilder.command()
    commands.add(s"-Xmx${maxMemoryMB}m")
    commands.addAll(extraJvmOpts.asJava)
    commands.add("-cp")
    commands.add(classpath)
    commands.add(className)
    commands.addAll(args.asJava)
    println(String.join(" ", commands))
    val process = processBuilder.start()
    val output = new ByteArrayOutputStream()
    IOUtils.copy(process.getInputStream, output)
    ProcessResult(process.waitFor(), output.toString())
  }
} 

package org.apache.spark.util

import java.net.{URLClassLoader, URL}
import java.util.Enumeration
import java.util.concurrent.ConcurrentHashMap

import scala.collection.JavaConverters._


  private val locks = new ConcurrentHashMap[String, Object]()

  override def loadClass(name: String, resolve: Boolean): Class[_] = {
    var lock = locks.get(name)
    if (lock == null) {
      val newLock = new Object()
      lock = locks.putIfAbsent(name, newLock)
      if (lock == null) {
        lock = newLock
      }
    }

    lock.synchronized {
      try {
        super.loadClass(name, resolve)
      } catch {
        case e: ClassNotFoundException =>
          parentClassLoader.loadClass(name, resolve)
      }
    }
  }

  override def getResource(name: String): URL = {
    val url = super.findResource(name)
    val res = if (url != null) url else parentClassLoader.getResource(name)
    res
  }

  override def getResources(name: String): Enumeration[URL] = {
    val childUrls = super.findResources(name).asScala
    val parentUrls = parentClassLoader.getResources(name).asScala
    (childUrls ++ parentUrls).asJavaEnumeration
  }

  override def addURL(url: URL) {
    super.addURL(url)
  }

} 

package scalan.util

import java.io.File
import java.net.URLClassLoader

object ClassLoaderUtil {
  def classPath(loader: ClassLoader): Seq[File] = loader match {
    case loader: URLClassLoader =>
      loader.getURLs.map(FileUtil.urlToFile) ++ classPath(loader.getParent)
    case _ =>
      Nil
  }

  def URLClassLoader(files: TraversableOnce[File], parent: ClassLoader): URLClassLoader =
    new URLClassLoader(files.map(_.toURI.toURL).toArray, parent)
} 

package com.github.jllk.analyser

import java.io.File
import java.net.{URLClassLoader, URL}

import scala.collection.mutable
import scala.collection.mutable.ListBuffer


object Analyser {
  def notCareClass(fullClassName: String): Boolean =
    fullClassName.startsWith("java") ||
    fullClassName.startsWith("scala") ||
    fullClassName.startsWith("\"[") ||
    (fullClassName.startsWith("android") && !fullClassName.startsWith("android/support"))
}

class Analyser(private val dependenceJarPath: List[File]) {

  import Analyser._

  def analysis(fullClassName: String): mutable.Set[String] = {
    val dependentClasses = mutable.Set[String]()
    val importDependence = analysisImportDependence(fullClassName)
    importDependence
      .foreach(c => {
        dependentClasses += c
        dependentClasses ++= analysisInheritDependence(c)
      })
    dependentClasses
  }

  private def analysisImportDependence(fullClassName: String): List[String] = {
    val dependentClasses = new ListBuffer[String]()
    val classpath = dependenceJarPath.map(f => s"-classpath ${f.toPath}") mkString " "
    val classReport = ProcessUtils.exec(s"javap -verbose $classpath ${fullClassName.replace('.', '/')}")
    val lines = classReport.split('\n')
    lines
      .filter(l => l.contains("= Class") && !l.contains("\"[Ljava/lang/Object;\""))
      .foreach(l => dependentClasses += l.substring(l.indexOf("//") + 2).replaceAll(" ", "").replaceAll("/", "\\.").trim())
    dependentClasses
      .filter(notCareClass)
      .toList
  }

  private def analysisInheritDependence(fullClassName: String): List[String] = {
    val urls = ListBuffer[URL]()
    dependenceJarPath.foreach(f => urls += f.toURI.toURL)
    val classLoader = new URLClassLoader(urls.toArray)
    doClassInheritSearch(fullClassName, classLoader)
  }

  private def doClassInheritSearch(fullClassName: String, classLoader: URLClassLoader): List[String] = {
    if (notCareClass(fullClassName)) {
      List.empty[String]
    } else {

      val dependentClasses = mutable.Set[String]()
      dependentClasses += fullClassName
      dependentClasses ++= analysisImportDependence(fullClassName)
      dependentClasses.foreach(fullClassName => {
        val targetClass: Either[Class[_], Exception] =
          try
            Left(classLoader.loadClass(fullClassName))
          catch {
            case e: ClassNotFoundException => Right(e)
            case e: Exception => Right(e)
          }

        targetClass match {
          case Left(c) =>
            val superclass = c.getSuperclass
            if (superclass != null) {
              dependentClasses ++= doClassInheritSearch(superclass.getName, classLoader)
            }
            c.getInterfaces.foreach(i => dependentClasses ++= doClassInheritSearch(i.getName, classLoader))
          case Right(e) =>
            println(s"[doClassInheritSearch] exception happened: ${e.getMessage}, please check your dependenceJarPath.")
        }

      })
      dependentClasses.toList
    }
  }
} 

package at.chaosfield.packupdate.server

import java.io.File
import java.net.{URL, URLClassLoader}
import java.util.jar.{Attributes, JarFile, Manifest}

import scala.collection.JavaConverters._

object Launcher {
  def launchServer(file: File, args: Array[String]): Unit = {
    val manifest = new JarFile(file).getManifest
    val cp = manifest.getMainAttributes.getValue("Class-Path").split(' ').map(path => new File("libraries", path).toURI.toURL)
    val loader = new URLClassLoader(cp ++ Array(file.toURI.toURL))
    val klass = Class.forName(manifest.getMainAttributes.getValue("Main-Class"), true, loader)
    val m = klass.getDeclaredMethod("main", classOf[Array[String]])
    m.invoke(null, args)
  }
} 

package com.spotify.scio.repl

import java.io.{File, FileOutputStream}
import java.net.{URL, URLClassLoader}
import java.nio.file.Files
import java.util.jar.{JarEntry, JarOutputStream}
import java.lang.invoke.{MethodHandles, MethodType}

import com.spotify.scio.repl.compat.ILoopClassLoader
import org.slf4j.LoggerFactory

import scala.tools.nsc.io._

object ScioReplClassLoader {
  private val Logger = LoggerFactory.getLogger(this.getClass)

  private[this] val JDK9OrHigher: Boolean = util.Properties.isJavaAtLeast("9")

  private[this] val BootClassLoader: ClassLoader = {
    if (!JDK9OrHigher) null
    else {
      try {
        MethodHandles
          .lookup()
          .findStatic(
            classOf[ClassLoader],
            "getPlatformClassLoader",
            MethodType.methodType(classOf[ClassLoader])
          )
          .invoke()
      } catch { case _: Throwable => null }
    }
  }

  def classLoaderURLs(cl: ClassLoader): Array[java.net.URL] = cl match {
    case null                       => Array.empty
    case u: java.net.URLClassLoader => u.getURLs ++ classLoaderURLs(cl.getParent)
    case _                          => classLoaderURLs(cl.getParent)
  }

  @inline final def apply(urls: Array[URL]) =
    new ScioReplClassLoader(urls, BootClassLoader)
}


  private def addVirtualDirectoryToJar(
    dir: Iterable[AbstractFile],
    entryPath: String,
    jarStream: JarOutputStream
  ): Unit = dir.foreach { file =>
    if (file.isDirectory) {
      // Recursively descend into subdirectories, adjusting the package name as we do.
      val dirPath = entryPath + file.name + "/"
      jarStream.putNextEntry(new JarEntry(dirPath))
      jarStream.closeEntry()
      addVirtualDirectoryToJar(file, dirPath, jarStream)
    } else if (file.hasExtension("class")) {
      // Add class files as an entry in the jar file and write the class to the jar.
      jarStream.putNextEntry(new JarEntry(entryPath + file.name))
      jarStream.write(file.toByteArray)
      jarStream.closeEntry()
    }
  }
} 

package com.sope.etl.register

import java.io.File
import java.net.URLClassLoader

import com.sope.etl.getObjectInstance
import com.sope.etl.transform.exception.YamlDataTransformException
import com.sope.etl.utils.JarUtils
import com.sope.utils.Logging
import org.apache.commons.io.FileUtils
import org.apache.spark.sql.expressions.UserDefinedFunction

import scala.tools.nsc.Settings
import scala.tools.nsc.interpreter.IMain

object  UDFBuilder extends Logging {

  val DefaultClassLocation = "/tmp/sope/dynamic/"
  val DefaultJarLocation = "/tmp/sope/sope-dynamic-udf.jar"


  
  def buildDynamicUDFs(udfCodeMap: Map[String, String]): Map[String, UserDefinedFunction] = {
    val file = new java.io.File(UDFBuilder.DefaultClassLocation)
    FileUtils.deleteDirectory(file)
    file.mkdirs()
    val udfMap = evalUDF(udfCodeMap)
    JarUtils.buildJar(DefaultClassLocation, DefaultJarLocation)
    udfMap
  }

} 

package io.deepsense.deeplang.refl

import java.io.File
import java.net.{URL, URLClassLoader}

import scala.collection.JavaConversions._

import org.reflections.Reflections
import org.reflections.util.ConfigurationBuilder

import io.deepsense.commons.utils.Logging
import io.deepsense.deeplang.catalogs.doperable.DOperableCatalog
import io.deepsense.deeplang.catalogs.doperations.DOperationsCatalog
import io.deepsense.deeplang.{DOperation, DOperationCategories, TypeUtils}


  def scanAndRegister(
      dOperableCatalog: DOperableCatalog,
      dOperationsCatalog: DOperationsCatalog
  ): Unit = {
    logger.info(
      s"Scanning registrables. Following jars will be scanned: ${jarsUrls.mkString(";")}.")
    for (registrable <- scanForRegistrables()) {
      logger.debug(s"Trying to register class $registrable")
      registrable match {
        case DOperationMatcher(doperation) => registerDOperation(dOperationsCatalog, doperation)
        case other => logger.warn(s"Only DOperation can be `@Register`ed")
      }
    }
  }

  private def scanForRegistrables(): Set[Class[_]] = {

    val urls = thisJarURLOpt ++ jarsUrls

    if (urls.nonEmpty) {

      val configBuilder = ConfigurationBuilder.build(urls.toSeq: _*)

      if (jarsUrls.nonEmpty) {
        configBuilder.addClassLoader(URLClassLoader.newInstance(jarsUrls.toArray))
      }

      new Reflections(configBuilder).getTypesAnnotatedWith(classOf[Register]).toSet
    } else {
      Set()
    }

  }

  private lazy val thisJarURLOpt: Option[URL] = {
    val jarRegex = """jar:(file:.*\.jar)!.*""".r

    val url = getClass.getClassLoader.getResource(
      getClass.getCanonicalName.replaceAll("\\.", File.separator) + ".class")

    url.toString match {
      case jarRegex(jar) => Some(new URL(jar))
      case _ => None
    }
  }


  private def registerDOperation(
      catalog: DOperationsCatalog,
      operation: Class[DOperation]
  ): Unit = TypeUtils.constructorForClass(operation) match {
    case Some(constructor) =>
      catalog.registerDOperation(
        DOperationCategories.UserDefined,
        () => TypeUtils.createInstance[DOperation](constructor)
      )
    case None => logger.error(
      s"Class $operation could not be registered." +
        "It needs to have parameterless constructor"
    )
  }

  class AssignableFromExtractor[T](targetClass: Class[T]) {
    def unapply(clazz: Class[_]): Option[Class[T]] = {
      if (targetClass.isAssignableFrom(clazz)) {
        Some(clazz.asInstanceOf[Class[T]])
      } else {
        None
      }
    }
  }

  object DOperationMatcher extends AssignableFromExtractor(classOf[DOperation])

} 

package org.apache.spark.util

import java.net.URLClassLoader

import org.scalatest.FunSuite

import org.apache.spark.{LocalSparkContext, SparkContext, SparkException, TestUtils}
import org.apache.spark.util.Utils

class MutableURLClassLoaderSuite extends FunSuite {

  val urls2 = List(TestUtils.createJarWithClasses(
      classNames = Seq("FakeClass1", "FakeClass2", "FakeClass3"),
      toStringValue = "2")).toArray
  val urls = List(TestUtils.createJarWithClasses(
      classNames = Seq("FakeClass1"),
      classNamesWithBase = Seq(("FakeClass2", "FakeClass3")), // FakeClass3 is in parent
      toStringValue = "1",
      classpathUrls = urls2)).toArray

  test("child first") {
    val parentLoader = new URLClassLoader(urls2, null)
    val classLoader = new ChildFirstURLClassLoader(urls, parentLoader)
    val fakeClass = classLoader.loadClass("FakeClass2").newInstance()
    val fakeClassVersion = fakeClass.toString
    assert(fakeClassVersion === "1")
    val fakeClass2 = classLoader.loadClass("FakeClass2").newInstance()
    assert(fakeClass.getClass === fakeClass2.getClass)
  }

  test("parent first") {
    val parentLoader = new URLClassLoader(urls2, null)
    val classLoader = new MutableURLClassLoader(urls, parentLoader)
    val fakeClass = classLoader.loadClass("FakeClass1").newInstance()
    val fakeClassVersion = fakeClass.toString
    assert(fakeClassVersion === "2")
    val fakeClass2 = classLoader.loadClass("FakeClass1").newInstance()
    assert(fakeClass.getClass === fakeClass2.getClass)
  }

  test("child first can fall back") {
    val parentLoader = new URLClassLoader(urls2, null)
    val classLoader = new ChildFirstURLClassLoader(urls, parentLoader)
    val fakeClass = classLoader.loadClass("FakeClass3").newInstance()
    val fakeClassVersion = fakeClass.toString
    assert(fakeClassVersion === "2")
  }

  test("child first can fail") {
    val parentLoader = new URLClassLoader(urls2, null)
    val classLoader = new ChildFirstURLClassLoader(urls, parentLoader)
    intercept[java.lang.ClassNotFoundException] {
      classLoader.loadClass("FakeClassDoesNotExist").newInstance()
    }
  }

  test("driver sets context class loader in local mode") {
    // Test the case where the driver program sets a context classloader and then runs a job
    // in local mode. This is what happens when ./spark-submit is called with "local" as the
    // master.
    val original = Thread.currentThread().getContextClassLoader

    val className = "ClassForDriverTest"
    val jar = TestUtils.createJarWithClasses(Seq(className))
    val contextLoader = new URLClassLoader(Array(jar), Utils.getContextOrSparkClassLoader)
    Thread.currentThread().setContextClassLoader(contextLoader)

    val sc = new SparkContext("local", "driverLoaderTest")

    try {
      sc.makeRDD(1 to 5, 2).mapPartitions { x =>
        val loader = Thread.currentThread().getContextClassLoader
        Class.forName(className, true, loader).newInstance()
        Seq().iterator
      }.count()
    }
    catch {
      case e: SparkException if e.getMessage.contains("ClassNotFoundException") =>
        fail("Local executor could not find class", e)
      case t: Throwable => fail("Unexpected exception ", t)
    }

    sc.stop()
    Thread.currentThread().setContextClassLoader(original)
  }
} 

package org.apache.spark.util

import java.net.{URL, URLClassLoader}
import java.util.Enumeration

import scala.collection.JavaConverters._


private[spark] class ChildFirstURLClassLoader(urls: Array[URL], parent: ClassLoader)
  extends MutableURLClassLoader(urls, null) {

  private val parentClassLoader = new ParentClassLoader(parent)

  override def loadClass(name: String, resolve: Boolean): Class[_] = {
    try {
      super.loadClass(name, resolve)
    } catch {
      case e: ClassNotFoundException =>
        parentClassLoader.loadClass(name, resolve)
    }
  }

  override def getResource(name: String): URL = {
    val url = super.findResource(name)
    val res = if (url != null) url else parentClassLoader.getResource(name)
    res
  }

  override def getResources(name: String): Enumeration[URL] = {
    val childUrls = super.findResources(name).asScala
    val parentUrls = parentClassLoader.getResources(name).asScala
    (childUrls ++ parentUrls).asJavaEnumeration
  }

  override def addURL(url: URL) {
    super.addURL(url)
  }

} 

package wvlet.airframe.http.codegen
import java.io.File
import java.net.{URL, URLClassLoader}
import java.util.jar.JarFile

import wvlet.log.LogSupport


object ClassScanner extends LogSupport {

  def scanClasses(cl: ClassLoader, targetPackageNames: Seq[String]): Seq[String] = {
    def loop(c: ClassLoader): Seq[URL] = {
      c match {
        case null =>
          Seq.empty
        case u: URLClassLoader =>
          u.getURLs.toSeq ++ loop(u.getParent)
        case _ =>
          loop(c.getParent)
      }
    }

    val urls = loop(cl)
    val b    = Seq.newBuilder[String]

    def findClasses(url: URL): Seq[String] = {
      url match {
        case dir if dir.getProtocol == "file" && { val d = new File(dir.getPath); d.exists() && d.isDirectory } =>
          scanClassesInDirectory(dir.getPath, targetPackageNames)
        case jarFile if jarFile.getProtocol == "file" && jarFile.getPath.endsWith(".jar") =>
          scanClassesInJar(jarFile.getPath, targetPackageNames)
        case _ =>
          Seq.empty
      }
    }

    urls.foreach(x => b ++= findClasses(x))

    b.result().filterNot { x => x.contains("$anon$") }.distinct
  }

  private def toFilePath(packageName: String): String = {
    packageName.replaceAll("\\.", "/") + "/"
  }

  private def scanClassesInDirectory(dir: String, targetPackageNames: Seq[String]): Seq[String] = {
    val classes = Seq.newBuilder[String]

    def loop(baseDir: File, f: File): Unit = {
      f match {
        case d: File if f.isDirectory =>
          val files = d.listFiles()
          if (files != null) {
            files.foreach(loop(baseDir, _))
          }
        case f: File if f.getPath.endsWith(".class") =>
          val className = f.getPath
            .stripSuffix(".class").replaceAllLiterally(baseDir.getPath, "").replaceFirst("\\/", "")
            .replaceAll("\\/", ".")

          classes += className
        case _ =>
      }
    }

    val baseDir = new File(dir)
    if (baseDir.exists() && baseDir.isDirectory) {
      val dirs = targetPackageNames.map(toFilePath).map { path => new File(baseDir, path) }
      dirs.foreach(x => loop(baseDir, x))
    }

    classes.result()
  }

  private def scanClassesInJar(jarFile: String, targetPackageNames: Seq[String]): Seq[String] = {
    val jf: JarFile = new JarFile(jarFile)
    val entryEnum   = jf.entries

    val targetPaths = targetPackageNames.map(toFilePath)

    val classes = Seq.newBuilder[String]

    while (entryEnum.hasMoreElements) {
      val jarEntry  = entryEnum.nextElement
      val entryName = jarEntry.getName
      if (entryName.endsWith(".class") && targetPaths.exists(p => entryName.startsWith(p))) {
        val clsName = entryName
          .stripSuffix(".class")
          .replaceAll("\\/", ".")
        classes += clsName
      }
    }

    classes.result()
  }
} 

package wvlet.airframe.http.codegen
import java.net.URLClassLoader

import example.api._
import wvlet.airframe.http._
import wvlet.airspec.AirSpec


class HttpClientGeneratorTest extends AirSpec {
  val router =
    RouteScanner.buildRouter(Seq(classOf[ResourceApi], classOf[QueryApi], classOf[BookApi]))

  test("build router") {
    debug(router)

    val r = router.routes.find(x => x.method == HttpMethod.GET && x.path == "/v1/resources/:id")
    r shouldBe defined

    val q = router.routes.find(x => x.method == HttpMethod.GET && x.path == "/v1/query")
    q shouldBe defined
  }

  test("generate async client") {
    val code = HttpClientGenerator.generate(
      router,
      HttpClientGeneratorConfig("example.api:async:example.api.client")
    )
    code.contains("package example.api.client") shouldBe true
    code.contains("import example.api.Query") shouldBe true
    code.contains("class ServiceClient[F[_], Req, Resp]") shouldBe true
    code.contains("import scala.collection.Seq") shouldBe false
    code.contains("import scala.language.higherKinds") shouldBe true

    code.contains("HttpRequest[Request]") shouldBe false

    test("Map client parameters to GET query strings") {
      code.contains("def getBooks(limit: Int") shouldBe true
    }
  }

  test("generate sync client") {
    val code = HttpClientGenerator.generate(
      router,
      HttpClientGeneratorConfig("example.api:sync")
    )
    code.contains("package example.api") shouldBe true
    code.contains("class ServiceSyncClient[Req, Resp]")
  }

  test("generate Scala.js client") {
    val code = HttpClientGenerator.generate(
      router,
      HttpClientGeneratorConfig("example.api:scalajs:example.api.client.js")
    )
    code.contains("package example.api.client.js") shouldBe true
    code.contains("class ServiceJSClient") shouldBe true

    test("Map client parameters to GET query strings") {
      code.contains("def getBooks(limit: Int") shouldBe true
      code.contains("""Map("limit" -> limit, "sort" -> sort)""") shouldBe true
    }
    code.contains("import java.lang.Object") shouldBe false
  }

  test("scan classes") {
    skip("This is a test used only for debugging")
    val urls = Array[java.net.URL](
      new java.net.URL("file:./sbt-airframe/target/scala-2.12/sbt-1.0/test-classes/"),
      new java.net.URL(
        "file:/Users/leo/.coursier/cache/v1/https/repo1.maven.org/maven2/org/wvlet/airframe/airframe-json_2.12/20.2.1/airframe-json_2.12-20.2.1.jar"
      )
    )
    val cl      = new URLClassLoader(urls)
    val classes = ClassScanner.scanClasses(cl, Seq("example", "wvlet.airframe.json"))
    debug(classes)
  }

} 

package com.amazon.milan.compiler.flink.runtime

import java.net.URLClassLoader

import com.amazon.milan.cmd.{ArgumentsBase, NamedArgument}
import com.typesafe.scalalogging.Logger
import org.apache.flink.contrib.streaming.state.RocksDBStateBackend
import org.apache.flink.runtime.state.StateBackend
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
import org.apache.flink.streaming.api.{CheckpointingMode, TimeCharacteristic}
import org.slf4j.LoggerFactory


class MilanApplicationCmdArgs extends ArgumentsBase {
  @NamedArgument(Name = "max-parallelism", ShortName = "mp", Required = false, DefaultValue = "0")
  var maxParallelism: Int = 0

  @NamedArgument(Name = "list-classpath", ShortName = "listcp", Required = false, DefaultValue = "false")
  var listClassPath: Boolean = false

  @NamedArgument(Name = "state-backend", ShortName = "sbe", Required = false, DefaultValue = "default")
  var stateBackend: String = _

  @NamedArgument(Name = "checkpoint-directory", ShortName = "cd", Required = false, DefaultValue = "file:///tmp/checkpoints")
  var checkpointDirectory: String = _

  @NamedArgument(Name = "checkpoint-interval", ShortName = "ci", Required = false, DefaultValue = "30")
  var checkpointIntervalSeconds: Int = 30
}


abstract class MilanApplicationBase {
  private val logger = Logger(LoggerFactory.getLogger(getClass))

  def buildFlinkApplication(env: StreamExecutionEnvironment): Unit

  def hasCycles: Boolean

  def execute(args: Array[String]): Unit = {
    val cmdArgs = new MilanApplicationCmdArgs
    cmdArgs.parse(args, allowUnknownArguments = true)

    if (cmdArgs.listClassPath) {
      this.listClassPathUrls()
    }

    val env = StreamExecutionEnvironment.getExecutionEnvironment
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)

    if (cmdArgs.maxParallelism > 0) {
      env.setMaxParallelism(cmdArgs.maxParallelism)
    }

    cmdArgs.stateBackend match {
      case "rocksdb" =>
        this.logger.info("Using RocksDB state back-end.")
        env.setStateBackend(new RocksDBStateBackend(cmdArgs.checkpointDirectory, true).asInstanceOf[StateBackend])

      case "default" => ()

      case _ =>
        throw new IllegalArgumentException("state-backend must be one of: rocksdb, default")
    }

    if (!this.hasCycles) {
      env.enableCheckpointing(cmdArgs.checkpointIntervalSeconds * 1000, CheckpointingMode.AT_LEAST_ONCE)
    }

    this.buildFlinkApplication(env)
    env.execute()
  }

  private def listClassPathUrls(): Unit = {
    ClassLoader.getSystemClassLoader match {
      case urlClassLoader: URLClassLoader =>
        urlClassLoader.getURLs.foreach(url => logger.info(s"ClassPath: $url"))

      case _ =>
        this.logger.error(s"Can't list ClassPath URLs for ClassLoader of type '${ClassLoader.getSystemClassLoader.getClass.getName}'.")
    }
  }
} 

package com.stratio.sparta.serving.core.utils

import java.io.Serializable
import java.net.URLClassLoader

import akka.event.slf4j.SLF4JLogging
import com.stratio.sparta.sdk.pipeline.aggregation.cube.DimensionType
import com.stratio.sparta.sdk.pipeline.aggregation.operator.Operator
import com.stratio.sparta.sdk.pipeline.input.Input
import com.stratio.sparta.sdk.pipeline.output.Output
import com.stratio.sparta.sdk.pipeline.transformation.Parser
import org.reflections.Reflections
import com.stratio.sparta.serving.core.exception.ServingCoreException

import scala.collection.JavaConversions._

class ReflectionUtils extends SLF4JLogging {

  def tryToInstantiate[C](classAndPackage: String, block: Class[_] => C): C = {
    val clazMap: Map[String, String] = getClasspathMap
    val finalClazzToInstance = clazMap.getOrElse(classAndPackage, classAndPackage)
    try {
      val clazz = Class.forName(finalClazzToInstance)
      block(clazz)
    } catch {
      case cnfe: ClassNotFoundException =>
        throw ServingCoreException.create(
          "Class with name " + classAndPackage + " Cannot be found in the classpath.", cnfe)
      case ie: InstantiationException =>
        throw ServingCoreException.create("Class with name " + classAndPackage + " cannot be instantiated", ie)
      case e: Exception =>
        throw ServingCoreException.create("Generic error trying to instantiate " + classAndPackage, e)
    }
  }

  def instantiateParameterizable[C](clazz: Class[_], properties: Map[String, Serializable]): C =
    clazz.getDeclaredConstructor(classOf[Map[String, Serializable]]).newInstance(properties).asInstanceOf[C]

  def printClassPath(cl: ClassLoader): Unit = {
    val urls = cl.asInstanceOf[URLClassLoader].getURLs()
    urls.foreach(url => log.debug(url.getFile))
  }

  lazy val getClasspathMap: Map[String, String] = {
    val reflections = new Reflections("com.stratio.sparta")

    try {
      log.debug("#######")
      log.debug("####### SPARK MUTABLE_URL_CLASS_LOADER:")
      log.debug(getClass.getClassLoader.toString)
      printClassPath(getClass.getClassLoader)
      log.debug("#######")
      log.debug("####### APP_CLASS_LOADER / SYSTEM CLASSLOADER:")
      log.debug(ClassLoader.getSystemClassLoader().toString)
      printClassPath(ClassLoader.getSystemClassLoader())
      log.debug("#######")
      log.debug("####### EXTRA_CLASS_LOADER:")
      log.debug(getClass.getClassLoader.getParent.getParent.toString)
      printClassPath(getClass.getClassLoader.getParent.getParent)
    } catch {
      case e: Exception => //nothing
    }

    val inputs = reflections.getSubTypesOf(classOf[Input]).toList
    val dimensionTypes = reflections.getSubTypesOf(classOf[DimensionType]).toList
    val operators = reflections.getSubTypesOf(classOf[Operator]).toList
    val outputs = reflections.getSubTypesOf(classOf[Output]).toList
    val parsers = reflections.getSubTypesOf(classOf[Parser]).toList
    val plugins = inputs ++ dimensionTypes ++ operators ++ outputs ++ parsers
    val result = plugins map (t => t.getSimpleName -> t.getCanonicalName) toMap

    log.debug("#######")
    log.debug("####### Plugins to be loaded:")
    result.foreach {
      case (simpleName: String, canonicalName: String) => log.debug(s"$canonicalName")
    }

    result
  }
} 

package com.lightbend.lagom.maven

import java.net.URLClassLoader
import javax.inject.Inject
import javax.inject.Singleton

import org.eclipse.aether.artifact.Artifact


  def extractScalaClassLoader(artifacts: Seq[Artifact]): ClassLoader = synchronized {
    val scalaArtifacts = artifacts.filter { artifact =>
      ScalaLibs.contains(artifact.getGroupId -> stripScalaVersion(artifact.getArtifactId))
    }

    val cacheKey = createCacheKey(scalaArtifacts)
    cache.get(cacheKey) match {
      case Some(classLoader) =>
        logger.debug(s"ScalaClassLoader cache hit - $cacheKey")
        classLoader
      case None =>
        logger.debug(s"ScalaClassLoader cache miss - $cacheKey")
        // Use System classloader parent as documented here:
        // https://svn.apache.org/repos/infra/websites/production/maven/content/reference/maven-classloading.html#Maven_API_classloader
        // Keep in mind this does not contain any application or javaagent classes, which will
        // be added in the classLoader below.
        //
        // This behaves a little different depending on the Java version used:
        // - For Java 8: the parent is the boostrap class loader (or null), which in the end
        //   means the boostrap class loader is used.
        // - For Java9+: the parent is the platform class loader is a parent or an ancestor
        //   of the system class loader that all platform classes are visible to it.
        val parent      = ClassLoader.getSystemClassLoader().getParent()
        val classLoader = new URLClassLoader(scalaArtifacts.map(_.getFile.toURI.toURL).toArray, parent)
        cache += (cacheKey -> classLoader)
        classLoader
    }
  }
} 

package org.apache.spark.util

import java.net.{URL, URLClassLoader}
import java.util.Enumeration

import scala.collection.JavaConverters._


private[spark] class ChildFirstURLClassLoader(urls: Array[URL], parent: ClassLoader)
  extends MutableURLClassLoader(urls, null) {

  private val parentClassLoader = new ParentClassLoader(parent)

  override def loadClass(name: String, resolve: Boolean): Class[_] = {
    try {
      super.loadClass(name, resolve)
    } catch {
      case e: ClassNotFoundException =>
        parentClassLoader.loadClass(name, resolve)
    }
  }

  override def getResource(name: String): URL = {
    val url = super.findResource(name)
    val res = if (url != null) url else parentClassLoader.getResource(name)
    res
  }

  override def getResources(name: String): Enumeration[URL] = {
    val childUrls = super.findResources(name).asScala
    val parentUrls = parentClassLoader.getResources(name).asScala
    (childUrls ++ parentUrls).asJavaEnumeration
  }

  override def addURL(url: URL) {
    super.addURL(url)
  }

} 

package org.apache.spark.util

import java.net.{URLClassLoader, URL}
import java.util.Enumeration
import java.util.concurrent.ConcurrentHashMap

import scala.collection.JavaConversions._

import org.apache.spark.util.ParentClassLoader


  private val locks = new ConcurrentHashMap[String, Object]()

  override def loadClass(name: String, resolve: Boolean): Class[_] = {
    var lock = locks.get(name)
    if (lock == null) {
      val newLock = new Object()
      lock = locks.putIfAbsent(name, newLock)
      if (lock == null) {
        lock = newLock
      }
    }

    lock.synchronized {
      try {
        super.loadClass(name, resolve)
      } catch {
        case e: ClassNotFoundException =>
          parentClassLoader.loadClass(name, resolve)
      }
    }
  }

  override def getResource(name: String): URL = {
    val url = super.findResource(name)
    val res = if (url != null) url else parentClassLoader.getResource(name)
    res
  }

  override def getResources(name: String): Enumeration[URL] = {
    val urls = super.findResources(name)
    val res =
      if (urls != null && urls.hasMoreElements()) {
        urls
      } else {
        parentClassLoader.getResources(name)
      }
    res
  }

  override def addURL(url: URL) {
    super.addURL(url)
  }

} 

package com.kakao.cuesheet.deps

import java.net.{URL, URLClassLoader}

import com.kakao.mango.concurrent.KeyedSingletons
import com.kakao.mango.logging.Logging
import com.kakao.mango.reflect.Accessible

import scala.reflect.io.AbstractFile

class DependencyAnalyzer(loader: ClassLoader = getClass.getClassLoader) extends Logging {
  val chain: List[ClassLoader] = DependencyAnalyzer.classLoaderChain(loader)

  lazy val graph = {
    // collect all nodes
    val nodes = for (
      loader <- chain;
      url <- DependencyAnalyzer.components(loader)
    ) yield {
      DependencyNode.resolve(url)
    }
    new DependencyGraph(nodes)
  }
}

object DependencyAnalyzer extends KeyedSingletons[ClassLoader, DependencyAnalyzer] with Logging {

  
  def components(loader: ClassLoader): Seq[URL] = {
    loader match {
      case _ if loader.getClass.getName.startsWith("sun.misc.Launcher$ExtClassLoader") =>
        Nil // ignore extension class loader
      case loader: URLClassLoader =>
        loader.getURLs
      case _ if loader.getClass.getName == "scala.tools.nsc.interpreter.AbstractFileClassLoader" =>
        val root = Accessible.field(loader.getClass, "root")
        Seq(root.get(loader).asInstanceOf[AbstractFile].toURL)
      case _ if loader.getClass.getName == "scala.reflect.internal.util.AbstractFileClassLoader" =>
        val root = Accessible.field(loader.getClass, "root")
        Seq(root.get(loader).asInstanceOf[AbstractFile].toURL)
      case _ if Seq(loader.getClass.getName).exists(c => c.startsWith("xsbt.") || c.startsWith("sbt.")) =>
        Nil // ignore SBT's internal loader
      case _ =>
        throw new RuntimeException("Unknown ClassLoader Type: " + loader.getClass.getName)
    }
  }

  override def newInstance(loader: ClassLoader): DependencyAnalyzer = {
    new DependencyAnalyzer(loader)
  }

  def apply(): DependencyAnalyzer = apply(getClass.getClassLoader)

} 

package org.codeoverflow.chatoverflow.framework.helper

import java.net.{URL, URLClassLoader}

import org.codeoverflow.chatoverflow.WithLogger


private object PluginClassLoader extends WithLogger {
  val appClassloader: ClassLoader = this.getClass.getClassLoader
  val platformClassloader: ClassLoader = {
    var current = appClassloader
    while (current != null && !current.getClass.getName.contains("ExtClassLoader") && // ExtClassLoader is java < 9
      !current.getClass.getName.contains("PlatformClassLoader")) { // PlatformClassLoader is java >= 9
      current = current.getParent
    }

    if (current != null) {
      current
    } else {
      logger error "Platform classloader couldn't be found. Falling back to normal app classloader."
      appClassloader
    }
  }
} 

package scalaclean.model.impl

import scalaclean.model._
import java.io.File
import java.net.{URL, URLClassLoader}
import java.nio.file.{Files, Path, Paths}
import java.util.Properties
import java.util.concurrent.ConcurrentHashMap

import scalafix.v1.SymbolInformation

import scala.meta.internal.symtab.{GlobalSymbolTable, SymbolTable}
import scala.meta.io.{AbsolutePath, Classpath}

object Project {

  import org.scalaclean.analysis.PropertyNames._

  def apply(propsPath: Path, projects: ProjectSet): Project = {
    val props = new Properties()
    println("PropsPath = " + propsPath)
    props.load(Files.newBufferedReader(propsPath))
    val classpathValue = props.getProperty(prop_classpath)
    val outputPath = props.getProperty(prop_outputDir)
    val elementsFilePath = props.getProperty(prop_elementsFile)
    val relationshipsFilePath = props.getProperty(prop_relationshipsFile)
    val extensionsFilePath = props.getProperty(prop_extensionsFile)
    val src = props.getProperty(prop_src)
    val srcBuildBase = props.getProperty(prop_srcBuildBase)
    val srcFiles = props.getProperty(prop_srcFiles, "").split(File.pathSeparatorChar).toSet
    val srcRoots = props.getProperty(prop_srcRoots).split(File.pathSeparatorChar).toList.sortWith((s1, s2) => s1.length > s1.length || s1 < s2).map(AbsolutePath(_))
    println("srcRoots = " + srcRoots)
    assert(classpathValue ne null, props.keys)
    assert(outputPath ne null, props.keys)

    val classPath = Classpath.apply(classpathValue)

    new Project(projects, classPath, outputPath, src, srcRoots, srcBuildBase, elementsFilePath, relationshipsFilePath, extensionsFilePath, srcFiles)
  }
}

class Project private(
                       val projects: ProjectSet, val classPath: Classpath, val outputPath: String, val src: String, val srcRoots: List[AbsolutePath], val srcBuildBase: String,
                       elementsFilePath: String, relationshipsFilePath: String, extensionsFilePath: String,
                       val srcFiles: Set[String]) {
  def symbolTable: SymbolTable = GlobalSymbolTable(classPath, includeJdk = true)

  lazy val classloader: ClassLoader = new URLClassLoader(Array(new URL("file:" + outputPath + "/")), null)

  private val infos = new ConcurrentHashMap[LegacyElementId, SymbolInformation]()

  def symbolInfo(viewedFrom: ElementModelImpl, symbol: LegacyElementId): SymbolInformation = {
    infos.computeIfAbsent(symbol,
      s => //any doc in the project would do though
        viewedFrom.source.doc.info(s.symbol).orNull)
  }

  def read: (Vector[ElementModelImpl], BasicRelationshipInfo) = ModelReader.read(this, elementsFilePath, relationshipsFilePath, extensionsFilePath)

  private val sourcesMap = new ConcurrentHashMap[String, SourceData]()

  def source(name: String): SourceData = {
    sourcesMap.computeIfAbsent(name, p => SourceData(this, Paths.get(p)))
  }

} 

package coursier.launcher.native

import java.io.File
import java.net.URLClassLoader

import coursier.launcher.Parameters.ScalaNative

trait NativeBuilder {
  def build(
    mainClass: String,
    files: Seq[File],
    output: File,
    options: ScalaNative.ScalaNativeOptions,
    log: String => Unit = s => Console.err.println(s),
    verbosity: Int = 0
  ): Unit
}

object NativeBuilder {

  def load(
    fetch: Seq[String] => Seq[File],
    version: String,
    parentLoader: ClassLoader = Thread.currentThread().getContextClassLoader
  ): NativeBuilder = {

    val sbv = scala.util.Properties.versionNumberString.split('.').take(2).mkString(".")
    val coursierVersion = coursier.launcher.internal.Properties.version
    val dep = s"io.get-coursier:coursier-launcher-native_${version}_$sbv:$coursierVersion"

    val files = fetch(Seq(dep))

    val loader = new URLClassLoader(files.map(_.toURI.toURL).toArray, parentLoader)

    val cls = loader.loadClass("coursier.launcher.NativeBuilderImpl")
    cls.newInstance().asInstanceOf[NativeBuilder]
  }

} 

package better.files

import java.net.{URL, URLClassLoader}

import better.files.test_pkg.ResourceSpecHelper

final class ResourceSpec extends CommonSpec {
  implicit val charset = java.nio.charset.StandardCharsets.US_ASCII
  val testFileText     = "This is the test-file.txt file."
  val altTestFileText  = "This is the another-test-file.txt file."
  val testFile         = "better/files/test-file.txt"
  val testFileRel      = "test-file.txt"
  val testFileAltRel   = "another-test-file.txt"
  val testFileFromCL   = "files/test-file.txt"

  "Resource" can "look up from the context class loader" in {
    assert(Resource.asStream(testFile).get.asString() startsWith testFileText)
  }

  it can "look up from a specified class loader" in {
    val clURL = new URL(Resource.my.getUrl("ResourceSpec.class"), "../")
    assert(clURL.toExternalForm endsWith "/")
    assert(Resource.from(new URLClassLoader(Array(clURL))).getAsString(testFileFromCL) startsWith testFileText)
  }

  it can "look up from the call site" in {
    assert(Resource.my.asStream(testFileRel).get.asString() startsWith testFileText)
    // This tests that Resource.my uses the correct call site when called from outside the better.files package.
    assert((new ResourceSpecHelper).openTestStream().asString() startsWith altTestFileText)
  }

  it can "look up from a statically-known type" in {
    assert(Resource.at[ResourceSpec].getAsString(testFileRel) startsWith testFileText)
    assert(Resource.at[Resource.type].getAsString(testFileRel) startsWith testFileText)
  }

  it can "look up from a java.lang.Class" in {
    assert(Resource.at(Class.forName("better.files.File")).getAsString(testFileRel) startsWith testFileText)
  }

  it can "look up a file in another package" in {
    assert(Resource.at[ResourceSpecHelper].getAsString(testFileAltRel) startsWith altTestFileText)
  }

  it should "require a concrete type" in {
    """def foo[T] = better.files.Resource.at[T].asStream("foo")""" shouldNot typeCheck
  }

  it should "fetch root url" in {
    assert(Option(Resource.getUrl()).isDefined)
  }

  it should "work with using util" in {
    File.usingTemporaryFile() { file =>
      file.appendText("hello world")
      val lines = using(file.newInputStream) { is =>
        is.lines.toList
      }
      assert(lines === "hello world" :: Nil)
    }
  }

  it should "close multiple resources" in {
    def emit(dir: File, partitions: Int, lines: Int) = {
      for {
        writers <- Vector.tabulate(partitions)(i => (dir / s"partition-$i.csv").newPrintWriter()).autoClosed
        line    <- 1 to lines
      } writers(line % partitions).println(line)
    }

    File.usingTemporaryDirectory() { dir =>
      val lines = 1000
      emit(dir = dir, partitions = 5, lines = lines)
      val expected = dir.list(filter = _.extension.contains(".csv")).flatMap(_.lines).map(_.toInt).toSet
      assert((1 to lines).forall(expected))
    }
  }
} 

package org.argus.jawa.core.util

import java.io.{BufferedReader, InputStreamReader}
import java.net.URLClassLoader
import java.text.NumberFormat

 
object JVMUtil {
	def startSecondJVM[C](clazz: Class[C], jvmArgs: List[String], args: List[String], redirectStream: Boolean): Int = {
    val separator = System.getProperty("file.separator")
    val classpath = Thread.currentThread().getContextClassLoader.asInstanceOf[URLClassLoader].getURLs.map(_.getPath()).reduce((c1, c2) => c1 + java.io.File.pathSeparator + c2)
    val path = System.getProperty("java.home") + separator + "bin" + separator + "java"
    val commands: IList[String] = List(path) ::: jvmArgs ::: List("-cp", classpath, clazz.getCanonicalName.stripSuffix("$")) ::: args
    import scala.collection.JavaConverters._
    val processBuilder = new ProcessBuilder(commands.asJava)
    processBuilder.redirectErrorStream(redirectStream)
    val process = processBuilder.start()
    val is = process.getInputStream
    val isr = new InputStreamReader(is)
    val br = new BufferedReader(isr)
    var line = br.readLine()
    while (line != null) {
      println(line)
      line = br.readLine()
    }
    process.waitFor()
  }
  
  def showMemoryUsage(): Unit = {
    val runtime = Runtime.getRuntime
    val format = NumberFormat.getInstance()
    
    val sb = new StringBuilder()
    val maxMemory = runtime.maxMemory()
    val allocatedMemory = runtime.totalMemory()
    val freeMemory = runtime.freeMemory()
    
    sb.append("free memory: " + format.format(freeMemory / 1024 / 1024) + " ")
    sb.append("allocated memory: " + format.format(allocatedMemory / 1024 / 1024) + " ")
    sb.append("max memory: " + format.format(maxMemory / 1024 / 1024) + " ")
    sb.append("total free memory: " + format.format((freeMemory + (maxMemory - allocatedMemory)) / 1024 / 1024) + " ")
    println(sb.toString())
  }
} 

package cmwell.util.loading

import java.io.{File, InputStream}
import java.net.{URL, URLClassLoader}

// scalastyle:off
import sun.misc.CompoundEnumeration
// scalastyle:on
import scala.util.{Failure, Success, Try}


class ChildFirstURLClassLoader(urls: Array[URL], parent: ClassLoader, except: Seq[String] = Seq())
    extends URLClassLoader(urls, parent) {

  protected override def loadClass(name: String, resolve: Boolean): Class[_] = {
    def tryFind(findAction: => Class[_]): Option[Class[_]] = Try(findAction) match {
      case Failure(e: ClassNotFoundException) => None
      case Failure(e)                         => throw e
      case Success(c)                         => Some(c)
    }

    def loadLocally = if (except.exists(name.startsWith)) None else tryFind(findClass(name))
    def loadFromParent = if (getParent == null) None else tryFind(getParent.loadClass(name))

    val alreadyLoaded = findLoadedClass(name)
    if (alreadyLoaded != null) {
      alreadyLoaded
    } else {

      val `class` = loadLocally.getOrElse(loadFromParent.orNull)

      if (resolve)
        resolveClass(`class`)
      `class`
    }
  }

  override def getResource(name: String): URL = findResource(name) match {
    case null => super.getResource(name)
    case u    => u
  }

  override def getResources(name: String): java.util.Enumeration[URL] = {
    val parent = getParent
    val localUrls = findResources(name)
    val parentUrls: java.util.Enumeration[URL] =
      if (parent != null) parent.getResources(name) else java.util.Collections.emptyEnumeration()
    new CompoundEnumeration(Array(localUrls, parentUrls))
  }

  override def getResourceAsStream(name: String): InputStream = {
    getResource(name) match {
      case null => null
      case url =>
        Try(url.openStream) match {
          case Success(x) => x
          case Failure(_) => null
        }
    }
  }
}

object ChildFirstURLClassLoader {
  def loadClassFromJar[T](className: String, jarPath: String, commonPackageNames:String, excludes: Seq[String] = Seq()): T =
    Loader(jarPath, excludes :+ commonPackageNames).load(className)

  case class Loader(jarPath: String, excludes: Seq[String] = Seq()) {
    val urls = if(new java.io.File(jarPath).isFile) Array(new File(jarPath).toURI.toURL) else Array[URL](new URL(jarPath))
    private val cl =
      new ChildFirstURLClassLoader(urls, this.getClass.getClassLoader, excludes)
    def load[T](className: String) = cl.loadClass(className).newInstance.asInstanceOf[T]
  }

} 

package io.univalence

import java.net.URLClassLoader
import java.sql.Date

import io.univalence.centrifuge.Sparknarrow
import org.apache.spark.SparkConf
import org.apache.spark.sql.types._
import org.apache.spark.sql.Encoders
import org.apache.spark.sql.SparkSession
import org.scalatest.FunSuite

case class Person(name: String, age: Int, date: Date)

class SparknarrowTest extends FunSuite {

  val conf: SparkConf = new SparkConf()
  conf.setAppName("yo")
  conf.set("spark.sql.caseSensitive", "true")
  conf.setMaster("local[2]")

  implicit val ss: SparkSession = SparkSession.builder.config(conf).getOrCreate
  import ss.implicits._

  test("testBasicCC") {

    val classDef = Sparknarrow.basicCC(Encoders.product[Person].schema).classDef
    checkDefinition(classDef)

  }

  def checkDefinition(scalaCode: String): Unit = {
    //TODO do a version for 2.11 and 2.12
    
  }

  test("play with scala eval") {

    val code =
      """
        case class Tata(str: String)
        case class Toto(age: Int, tata: Tata)
      """

    checkDefinition(code)
    checkDefinition(code)

  }

  ignore("printSchema StructType") {
    val yo = StructType(
      Seq(
        StructField("name", StringType),
        StructField("tel", ArrayType(StringType))
      )
    )

    yo.printTreeString()
  }

}