better.files.File Scala Examples

package io.shiftleft.console.workspacehandling

import better.files.Dsl._
import better.files.File
import io.shiftleft.semanticcpg.testing.MockCpg
import org.scalatest.{Matchers, WordSpec}

import scala.collection.mutable.ListBuffer

class WorkspaceTests extends WordSpec with Matchers {

  "toString" should {

    "return an \"empty\" when no projects are present" in {
      val workspace = new Workspace(ListBuffer())
      workspace.toString shouldBe "empty"
    }

    "return a valid row for a project" in {
      File.usingTemporaryDirectory("project") { project =>
        mkdir(project / "overlays")
        val inputPath = "/input/path"
        val projectFile = ProjectFile(inputPath, project.name)
        val cpg = MockCpg().withMetaData("C", List("foo", "bar"), List()).cpg
        val projects = ListBuffer(
          Project(projectFile, project.path, Some(cpg))
        )
        val workspace = new Workspace(projects)
        val output = workspace.toString
        val lines = output.split("\n")
        lines.length shouldBe 5
        lines(4).contains(project.name) shouldBe true
        lines(4).contains(inputPath)
        lines(4).contains("foo,bar")
      }

    }

  }

}

object WorkspaceTests {

  def createFakeProject(workspaceFile: File, projectName: String): File = {
    mkdir(workspaceFile / projectName)
    mkdir(workspaceFile / projectName / "overlays")
    (workspaceFile / projectName / "project.json")
      .write(s"""{"inputPath":"foo","name":"$projectName"}""")
    touch(workspaceFile / projectName / "cpg.bin")
  }

} 

package com.ebiznext.comet.database.extractor

import better.files.File
import scopt.{OParser, RenderingMode}

case class ExtractScriptGenConfig(
  domain: String = "",
  scriptTemplateFile: File = File("."),
  scriptOutputDir: File = File(".")
)

object ExtractScriptGenConfig {

  val builder = OParser.builder[ExtractScriptGenConfig]

  def exists(name: String)(path: String): Either[String, Unit] =
    if (File(path).exists) Right(())
    else Left(s"$name at path $path does not exist")

  val parser: OParser[Unit, ExtractScriptGenConfig] = {
    import builder._
    OParser.sequence(
      programName("comet"),
      head("comet", "1.x"),
      note(
        """
          |The schemas should at least, specify :
          |     - a table name (schemas.name)
          |     - a file pattern (schemas.pattern) which is used as the export file base name
          |     - a write mode (schemas.metadata.write): APPEND or OVERWRITE
          |     - a delta column (schemas.merge.timestamp) if in APPEND mode : the column which is used to determine new rows for each exports
          |     - the columns to extract (schemas.attributes.name*)
          |
          |You also have to provide a Mustache (http://mustache.github.io/mustache.5.html) template file.
          |
          |In there you'll write your extraction export process (sqlplus for Oracle, pgsql for PostgreSQL as an example).
          |In that template you can use the following parameters:
          | - table_name  -> the table to export
          | - delimiter   -> the resulting dsv file delimiter
          | - columns     -> the columns to export
          |   columns is a Mustache map, it gives you access, for each column, to:
          |    - name               -> the column name
          |    - trailing_col_char  -> the separator to append to the column (, if there are more columns to come, "" otherwise)
          |                            Here is an example how to use it in a template:
          |                              SELECT
          |                              {{#columns}}
          |                              TO_CHAR({{name}}){{trailing_col_char}}
          |                              {{/columns}}
          |                              FROM
          |                              {{table_name}};
          | export_file -> the export file name
          | full_export -> if the export is a full or delta export (the logic is to be implemented in your script)
          |""".stripMargin
      ),
      cmd("script-gen"),
      opt[String]("domain")
        .action((x, c) => c.copy(domain = x))
        .required()
        .text("The domain for which to generate extract scripts"),
      opt[String]("templateFile")
        .validate(exists("Script template file"))
        .action((x, c) => c.copy(scriptTemplateFile = File(x)))
        .required()
        .text("Script template file"),
      opt[String]("scriptsOutputDir")
        .validate(exists("Script output folder"))
        .action((x, c) => c.copy(scriptOutputDir = File(x)))
        .required()
        .text("Scripts output folder")
    )
  }
  val usage: String = OParser.usage(parser, RenderingMode.TwoColumns)

  
  def parse(args: Seq[String]): Option[ExtractScriptGenConfig] =
    OParser.parse(parser, args, ExtractScriptGenConfig.apply())
} 

package com.ebiznext.comet.utils

import java.io.{BufferedInputStream, InputStream}
import java.nio.file.{Files, Paths}

import better.files.File
import org.apache.commons.compress.archivers.{
  ArchiveEntry,
  ArchiveInputStream,
  ArchiveStreamFactory
}
import org.apache.commons.compress.compressors.{CompressorInputStream, CompressorStreamFactory}
import org.apache.commons.compress.utils.IOUtils
import org.apache.commons.io.input.CloseShieldInputStream

import scala.util.Try

object Unpacker {

  def unpack(archiveFile: File, directory: File): Try[Unit] = {
    for {
      inputStream <- Try(Files.newInputStream(Paths.get(archiveFile.pathAsString)))
      it          <- open(inputStream)
    } yield {
      while (it.hasNext) {
        val (entry, is) = it.next()
        if (entry.isDirectory) {
          throw new Exception("Compressed archive cannot directories")
        }
        val targetFile = File(directory, entry.getName)
        val o = Files.newOutputStream(targetFile.path)
        try {
          IOUtils.copy(is, o)
        } finally {
          if (o != null) o.close()
        }
      }
    }
  }

  // https://alexwlchan.net/2019/09/unpacking-compressed-archives-in-scala/
  
  def open(inputStream: InputStream): Try[Iterator[(ArchiveEntry, InputStream)]] =
    for {
      uncompressedInputStream <- createUncompressedStream(inputStream)
      archiveInputStream      <- createArchiveStream(uncompressedInputStream)
      iterator = createIterator(archiveInputStream)
    } yield iterator

  private def createUncompressedStream(inputStream: InputStream): Try[CompressorInputStream] =
    Try {
      new CompressorStreamFactory().createCompressorInputStream(
        getMarkableStream(inputStream)
      )
    }

  private def createArchiveStream(
    uncompressedInputStream: CompressorInputStream
  ): Try[ArchiveInputStream] =
    Try {
      new ArchiveStreamFactory()
        .createArchiveInputStream(
          getMarkableStream(uncompressedInputStream)
        )
    }

  private def createIterator(
    archiveInputStream: ArchiveInputStream
  ): Iterator[(ArchiveEntry, InputStream)] =
    new Iterator[(ArchiveEntry, InputStream)] {
      var latestEntry: ArchiveEntry = _

      override def hasNext: Boolean = {
        latestEntry = archiveInputStream.getNextEntry
        latestEntry != null
      }

      override def next(): (ArchiveEntry, InputStream) =
        (latestEntry, new CloseShieldInputStream(archiveInputStream))
    }

  private def getMarkableStream(inputStream: InputStream): InputStream =
    if (inputStream.markSupported())
      inputStream
    else
      new BufferedInputStream(inputStream)

} 

package app.swagger

import app.SwaggerCodegen.property2Scala
import better.files.File
import io.swagger.models.Swagger
import utils.CaseClassMetaHelper
import utils.ScalaFmtHelper
import scala.meta._
import scala.meta.Source
import scala.meta.Stat
import better.files._
import scala.collection.JavaConversions._
import app._

object ModelsGen {
  def gen(swagger: Swagger, apiVersion: String, f: File): Unit = {
    println(s"- Starting Models Generator for ${f.pathAsString}")
    val modelsFolder = file"shared/src/main/scala/shared/models/swagger/${f.nameWithoutExtension}/$apiVersion"

    swagger.getDefinitions.toVector.foreach {
      case (name, model) =>
        val modelName = name.toUpperCamelCase

        val propertiesAsScala: Vector[String] = model.getProperties.toVector.map { e =>
          s"${e._1.toCamelCase}: ${property2Scala(e._2)}"
        }

        val modelAsCaseClass = s"case class $modelName(${propertiesAsScala.mkString(", ")})"

        val targetFile = modelsFolder./(s"$modelName.scala")
        if (targetFile.notExists) {
          // Create Template
          val template =
            s"""
               |package shared.models.swagger.${f.nameWithoutExtension}.$apiVersion
               |
            |import java.time._
               |
            |$modelAsCaseClass
          """.trim.stripMargin

          targetFile.createIfNotExists(createParents = true).overwrite(template)
        } else {
          // Update existing Source
          val source = targetFile.toJava.parse[Source].get
          val caseClassStat =
            modelAsCaseClass.parse[Stat].get
          val tree = CaseClassMetaHelper.updateOrInsert(source, caseClassStat)
          targetFile.write(ScalaFmtHelper.formatCode(tree.syntax))
        }
    }
  }

} 

package ansible

import scala.io.Source
import scala.sys.process.{Process, ProcessIO}

import ansible.IniEncode._
import ansible.IniEncoders._
import better.files.File
import com.typesafe.scalalogging.LazyLogging

object Runner extends LazyLogging {
  def runPlaybook(inv: Inventory)(pb: Playbook, opts: Option[String] = None): Unit = {
    val invFile = File.newTemporaryFile("ansible-inventory")
    val pbFile  = File.newTemporaryFile("ansible-playbook", ".yml")

    val pio = new ProcessIO(
      _ => (),
      out  => Source.fromInputStream(out).getLines.foreach(println),
      err => Source.fromInputStream(err).getLines.foreach(System.err.println)
    )

    val cmd = s"ansible-playbook ${opts.getOrElse("")} -i ${invFile.path} ${pbFile.path}"
    val env = Seq("ANSIBLE_FORCE_COLOR" -> "true")
    val process = Process(cmd, cwd = None, env: _*).run(pio)

    invFile.write(inv.iniEncode)
    pbFile.write(YAML.fromPlaybook(pb))
    logger.info(cmd)

    val exitCode = process.exitValue()
    logger.info(s"run completed with exit code: $exitCode")
    process.destroy()
  }
} 

package cool.graph.localfaas

import java.io.FileInputStream

import better.files.File
import org.apache.commons.compress.archivers.{ArchiveEntry, ArchiveStreamFactory}
import org.apache.commons.compress.utils.IOUtils

import scala.util.{Failure, Try}

object Utils {
  def unzip(source: File, target: File): Unit = {
    val inputStream   = new FileInputStream(source.path.toFile)
    val archiveStream = new ArchiveStreamFactory().createArchiveInputStream(ArchiveStreamFactory.ZIP, inputStream)

    def stream: Stream[ArchiveEntry] = archiveStream.getNextEntry match {
      case null  => Stream.empty
      case entry => entry #:: stream
    }

    def closeStreams = {
      archiveStream.close()
      inputStream.close()
    }

    Try {
      for (entry <- stream if !entry.isDirectory) {
        val outFile = (target / entry.getName).createIfNotExists(asDirectory = false, createParents = true).clear()
        val os      = outFile.newOutputStream

        Try { IOUtils.copy(archiveStream, os) } match {
          case Failure(e) => os.close(); throw e
          case _          => os.close()
        }
      }
    } match {
      case Failure(e) => closeStreams; throw e
      case _          => closeStreams
    }
  }
} 

package cool.graph.localfaas.actors

import akka.actor.Actor
import better.files.File
import cool.graph.localfaas.actors.MappingActor.{GetHandler, HandlerMap, SaveMapping}
import play.api.libs.json._

import scala.collection.mutable

object MappingActor {
  case class SaveMapping(projectId: String, functionName: String, handlerPath: String)
  case class GetHandler(projectId: String, functionName: String)

  type HandlerMap = mutable.HashMap[String, mutable.HashMap[String, String]]
}

case class MappingActor(handlerFile: File) extends Actor {
  import Conversions._

  // projectId -> functionName -> handlerPath
  val handlers = loadHandlers

  // load handlers on creation
  def loadHandlers: HandlerMap = {
    val content = handlerFile.contentAsString

    if (handlerFile.contentAsString.isEmpty) {
      new HandlerMap
    } else {
      Json.parse(content).validate[HandlerMap] match {
        case JsSuccess(result, _) => println("Using mapping from file."); result
        case JsError(_)           => println("Unable to parse handler map from file, using empty map."); new HandlerMap
      }
    }
  }

  def flush(): Unit = {
    val compactJson: String = Json.stringify(Json.toJson(handlers))
    handlerFile.overwrite(compactJson)
  }

  override def receive: Receive = {
    case GetHandler(pid, fnName) =>
      val projectHandlerMap = handlers.getOrElse(pid, new mutable.HashMap[String, String]())
      sender ! projectHandlerMap.getOrElse(fnName, "")

    case SaveMapping(pid, fnName, handlerPath) =>
      val projectHandlerMap = handlers.getOrElseUpdate(pid, new mutable.HashMap[String, String]())
      projectHandlerMap += fnName -> handlerPath
      flush()
  }
} 

package org.dbpedia.databus

import java.util

import better.files.File
import com.typesafe.scalalogging.LazyLogging
import org.apache.maven.plugin.AbstractMojo
import org.apache.maven.plugins.annotations.Parameter

object skipmodules {
  var skipmodules = false
}

abstract class Operations extends AbstractMojo with LazyLogging with Properties {

  @Parameter(
    property = "modules",
    defaultValue = "${project.modules}"
  )
  val modules: util.ArrayList[String] = new util.ArrayList[String]

  def listFiles(): List[File] = {
    val dir = File(s"$version")
    if (dir.isDirectory) {
      dir.list.toList
    } else {
      List[File]()
    }
  }

  def listFiles(artifact: String): List[File] = {
    val dir = File(s"$artifact/$version")
    if (dir.isDirectory) {
      dir.list.toList
    } else {
      List[File]()
    }

  }

} 

package org.dbpedia.databus

import java.util

import better.files.File
import com.typesafe.scalalogging.LazyLogging
import org.apache.maven.plugin.{AbstractMojo, MojoExecutionException}
import org.apache.maven.plugins.annotations.{LifecyclePhase, Mojo, Parameter}



@Mojo(name = "rm", requiresOnline = true, threadSafe = true)
class RemoveVersion extends Operations {


  @throws[MojoExecutionException]
  override def execute(): Unit = {

    getLog.info("the following version folders will be deleted:")

    if (isParent()) {
      skipmodules.skipmodules = true
      modules.forEach(m => {
        getLog.info(
          s"""${listFiles(m).size} files in ${File(s"$m/$version").toJava.getAbsoluteFile}""".stripMargin)
      })

      getLog.info("proceed? [y/N]")
      val c: String = scala.io.StdIn.readLine()
      if (c.trim.equalsIgnoreCase("y")) {
        getLog.info("deleting")
        modules.forEach(m => {
          //DELETE
          val vdir = File(s"$m/$version")
          vdir.delete(true)
          getLog.info(s"${!vdir.isDirectory} $vdir")
        })


      }else {
        println(s"aborted, read '$c'")
      }


    } else {
      if (!skipmodules.skipmodules) {
        getLog.info(
          s"""##########
             |databus:rm works only on group to delete current version of all artifacts, use:
             |rm -r $artifactId/$version"
           """.stripMargin)
      }
      return
    }
  }

} 

package org.dbpedia.databus

import better.files.File
import org.apache.maven.plugin.MojoExecutionException
import org.apache.maven.plugins.annotations.Mojo



@Mojo(name = "ls", requiresOnline = true, threadSafe = true)
class ListVersionFiles extends Operations {

  val separator = ", "

  @throws[MojoExecutionException]
  override def execute(): Unit = {


    if (isParent()) {
      skipmodules.skipmodules = true
      modules.forEach(m => {
        //todo .map(_.relativize(File("")))
        getLog.info(
          s"""$m/$version (${listFiles(m).size} files)
             |${listFiles(m).mkString(separator)}
           """.stripMargin)

      })
    } else {
      if (skipmodules.skipmodules) {
        return
      }
      getLog.info(
        s"""$artifactId/$version  (${listFiles().size} files)
           |${listFiles().mkString(separator)}
           """.stripMargin)
    }

  }

} 

package io.shiftleft.console.workspacehandling

import better.files.Dsl.mkdir
import better.files.File
import org.scalatest.{Matchers, WordSpec}

import scala.reflect.io.Directory

class WorkspaceLoaderTests extends WordSpec with Matchers {

  private val tmpDirPrefix = "workspace-tests"

  "WorkspaceLoader" should {

    "create workspace and workspace directory if nonexistent" in {
      val dir = File.newTemporaryDirectory(tmpDirPrefix)
      new Directory(dir.toJava).deleteRecursively()
      TestLoader().load(dir.path.toString)
      try {
        dir.exists shouldBe true
      } finally {
        new Directory(dir.toJava).deleteRecursively()
      }
    }

    "handle broken project.json gracefully by skipping project" in {
      File.usingTemporaryDirectory(tmpDirPrefix) { tmpDir =>
        mkdir(tmpDir / "1")
        (tmpDir / "1" / "project.json").write("{foo")
        TestLoader().load(tmpDir.path.toString).numberOfProjects shouldBe 0
      }
    }

    "load project correctly" in {
      File.usingTemporaryDirectory(tmpDirPrefix) { tmpDir =>
        val projectName = "foo"
        WorkspaceTests.createFakeProject(tmpDir, projectName)
        val project = TestLoader().loadProject((tmpDir / projectName).path)
        project match {
          case Some(p) =>
            p.name shouldBe "foo"
            p.inputPath shouldBe "foo"
            p.cpg shouldBe None
          case None => fail
        }
      }
    }

    "initialize workspace's project list correctly" in {
      File.usingTemporaryDirectory(tmpDirPrefix) { tmpDir =>
        val projectName = "foo"
        WorkspaceTests.createFakeProject(tmpDir, projectName)
        val workspace = TestLoader().load(tmpDir.toString)
        workspace.numberOfProjects shouldBe 1
      }
    }
  }

  "ProjectFile" should {

    import org.json4s.DefaultFormats
    import org.json4s.native.Serialization.{read => jsonRead, write => jsonWrite}
    implicit val formats: DefaultFormats.type = DefaultFormats

    "be serializable to json" in {
      jsonWrite(ProjectFile("foo", "aname")) shouldBe """{"inputPath":"foo","name":"aname"}"""
    }

    "be deserializable from json" in {
      val projectFile = jsonRead[ProjectFile]("""{"inputPath":"foo","name":"aname"}""")
      projectFile.inputPath shouldBe "foo"
      projectFile.name shouldBe "aname"
    }

  }

} 

package com.ebiznext.comet.database.extractor

import java.util.regex.Pattern

import better.files.File
import com.ebiznext.comet.schema.model._
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class TemplateParamsSpec extends AnyFlatSpec with Matchers {
  val scriptOutputFolder: File = File("/tmp")

  "fromSchema" should "generate the correct TemplateParams for a given Schema" in {
    val schema: Schema = Schema(
      name = "table1",
      pattern = Pattern.compile("output_file.*.csv"),
      List(Attribute(name = "col1"), Attribute(name = "col2")),
      metadata = Option(Metadata(write = Some(WriteMode.APPEND))),
      merge = Some(MergeOptions(List("col1", "col2"), None, timestamp = Some("updateCol"))),
      comment = None,
      presql = None,
      postsql = None
    )

    val expectedTemplateParams = TemplateParams(
      tableToExport = "table1",
      columnsToExport = List("col1", "col2"),
      fullExport = false,
      dsvDelimiter = ",",
      deltaColumn = Some("updateCol"),
      exportOutputFileBase = "output_file",
      scriptOutputFile = scriptOutputFolder / "EXTRACT_table1.sql"
    )
    TemplateParams.fromSchema(schema, scriptOutputFolder) shouldBe expectedTemplateParams
  }

  it should "generate the correct TemplateParams for an other Schema" in {
    val schema: Schema = Schema(
      name = "table1",
      pattern = Pattern.compile("output_file.*.csv"),
      List(Attribute(name = "col1"), Attribute(name = "col2")),
      metadata = Option(Metadata(write = Some(WriteMode.OVERWRITE), separator = Some("|"))),
      merge = Some(MergeOptions(List("col1", "col2"), None, timestamp = Some("updateCol"))),
      comment = None,
      presql = None,
      postsql = None
    )

    val expectedTemplateParams = TemplateParams(
      tableToExport = "table1",
      columnsToExport = List("col1", "col2"),
      fullExport = true,
      dsvDelimiter = "|",
      deltaColumn = None,
      exportOutputFileBase = "output_file",
      scriptOutputFile = scriptOutputFolder / "EXTRACT_table1.sql"
    )
    TemplateParams.fromSchema(schema, scriptOutputFolder) shouldBe expectedTemplateParams
  }
} 

package io.shiftleft.console

import better.files.File
import org.scalatest.{Matchers, WordSpec}

class ConsoleConfigTest extends WordSpec with Matchers {
  "An InstallConfig" should {
    "set the rootPath to the current working directory by default" in {
      val config = new InstallConfig(environment = Map.empty)
      config.rootPath shouldBe File(".")
    }

    "set the rootPath to SHIFTLEFT_OCULAR_INSTALL_DIR if it is defined" in {
      val config = new InstallConfig(environment = Map("SHIFTLEFT_OCULAR_INSTALL_DIR" -> "/tmp"))
      config.rootPath shouldBe File("/tmp")
    }
  }
} 

package io.shiftleft.console.scripting

import better.files.File
import cats.effect.IO
import org.scalatest.{Inside, Matchers, WordSpec}

import io.shiftleft.codepropertygraph.Cpg
import io.shiftleft.console.scripting.ScriptManager.{ScriptCollections, ScriptDescription, ScriptDescriptions}

import java.nio.file.{FileSystemNotFoundException, NoSuchFileException, Path}

import scala.io.Source
import scala.util.Try

class ScriptManagerTest extends WordSpec with Matchers with Inside {

  private object TestScriptExecutor extends AmmoniteExecutor {
    override protected def predef: String = ""
    override def runScript(scriptPath: Path, parameters: Map[String, String], cpg: Cpg): IO[Any] = IO.fromTry(
      Try {
        val source = Source.fromFile(scriptPath.toFile)
        val result = source.getLines.mkString(System.lineSeparator())
        source.close()
        result
      }
    )
  }

  private object TestScriptManager extends ScriptManager(TestScriptExecutor)

  protected val DEFAULT_CPG_NAME: String = {
    if (File(".").name == "console") {
      (File("..") / "resources" / "testcode" / "cpgs" / "method" / "cpg.bin.zip").pathAsString
    } else {
      (File("resources") / "testcode" / "cpgs" / "method" / "cpg.bin.zip").pathAsString
    }
  }

  def withScriptManager(f: ScriptManager => Unit): Unit = {
    f(TestScriptManager)
  }

  "listing scripts" should {
    "be correct" in withScriptManager { scriptManager =>
      val scripts = scriptManager.scripts()
      val expected = List(
        ScriptCollections("general",
                          ScriptDescriptions(
                            "A collection of general purpose scripts.",
                            List(ScriptDescription("list-funcs.sc", "Lists all functions."))
                          )),
        ScriptCollections("java",
                          ScriptDescriptions(
                            "A collection of java-specific scripts.",
                            List(ScriptDescription("list-sl-ns.sc", "Lists all shiftleft namespaces."))
                          )),
        ScriptCollections("general/general_plus",
                          ScriptDescriptions(
                            "Even more general purpose scripts.",
                            List.empty
                          ))
      )

      scripts should contain theSameElementsAs expected
    }
  }

  "running scripts" should {
    "be correct when explicitly specifying a CPG" in withScriptManager { scriptManager =>
      val expected =
        """|@main def main() = {
           |  cpg.method.name.l
           |}""".stripMargin

      scriptManager.runScript("general/list-funcs.sc", Map.empty, Cpg.emptyCpg) shouldBe expected
    }

    "be correct when specifying a CPG filename" in withScriptManager { scriptManager =>
      val expected =
        """|@main def main() = {
           |  cpg.method.name.l
           |}""".stripMargin

      scriptManager.runScript("general/list-funcs.sc", Map.empty, DEFAULT_CPG_NAME) shouldBe expected
    }

    "throw an exception if the specified CPG can not be found" in withScriptManager { scriptManager =>
      intercept[FileSystemNotFoundException] {
        scriptManager.runScript("general/list-funcs.sc", Map.empty, "cake.bin.zip")
      }
    }

    "throw an exception if the specified script can not be found" in withScriptManager { scriptManager =>
      intercept[NoSuchFileException] {
        scriptManager.runScript("list-funcs.sc", Map.empty, Cpg.emptyCpg)
      }
    }
  }

} 

package io.shiftleft.console.workspacehandling

import java.nio.file.Path

import better.files.File
import better.files.Dsl._
import io.shiftleft.codepropertygraph.Cpg
import io.shiftleft.semanticcpg.Overlays

object Project {
  val workCpgFileName = "cpg.bin.tmp"
  val persistentCpgFileName = "cpg.bin"
}

case class ProjectFile(inputPath: String, name: String)


  def close: Project = {
    cpg.foreach { c =>
      c.close()
      System.err.println("Turning working copy into new persistent CPG")
      val workingCopy = path.resolve(workCpgFileName)
      val persistent = path.resolve(persistentCpgFileName)
      cp(workingCopy, persistent)
    }
    cpg = None
    this
  }

} 

package io.shiftleft.console.workspacehandling

import java.nio.file.Path

import better.files.Dsl.mkdirs
import better.files.File
import org.json4s.DefaultFormats
import org.json4s.native.Serialization.{read => jsonRead}

import scala.collection.mutable.ListBuffer
import scala.util.{Failure, Success, Try}


  def load(path: String): Workspace[ProjectType] = {
    val dirFile = File(path)
    val dirPath = dirFile.path.toAbsolutePath

    if (!dirFile.exists) {
      println(s"creating workspace directory: ${dirFile.path.toString}")
      mkdirs(dirFile)
    }
    new Workspace(ListBuffer.from(loadProjectsFromFs(dirPath)))
  }

  private def loadProjectsFromFs(cpgsPath: Path): LazyList[ProjectType] = {
    cpgsPath.toFile.listFiles
      .filter(_.isDirectory)
      .to(LazyList)
      .flatMap(f => loadProject(f.toPath))
  }

  def loadProject(path: Path): Option[ProjectType] = {
    Try {
      val projectFile = readProjectFile(path)
      createProject(projectFile, path)
    } match {
      case Success(v) => Some(v)
      case Failure(e) =>
        System.err.println(s"Error loading project at $path - skipping: ")
        System.err.println(e)
        None
    }
  }

  def createProject(projectFile: ProjectFile, path: Path): ProjectType

  private val PROJECTFILE_NAME = "project.json"
  implicit val formats: DefaultFormats.type = DefaultFormats

  private def readProjectFile(projectDirName: Path): ProjectFile = {
    // TODO see `writeProjectFile`
    val content = File(projectDirName.resolve(PROJECTFILE_NAME)).contentAsString
    val map = jsonRead[Map[String, String]](content)
    ProjectFile(map("inputPath"), map("name"))
  }

} 

package io.shiftleft.console.testing

import java.nio.file.Path
import java.util.concurrent.LinkedBlockingQueue

import better.files.Dsl.mkdir
import better.files.File
import io.shiftleft.console.cpgcreation.{CpgGenerator, LanguageFrontend}
import io.shiftleft.console.{Console, ConsoleConfig, DefaultAmmoniteExecutor, InstallConfig}
import io.shiftleft.console.workspacehandling.{Project, ProjectFile, WorkspaceLoader}
import io.shiftleft.fuzzyc2cpg.FuzzyC2Cpg
import io.shiftleft.proto.cpg.Cpg.CpgStruct

object ConsoleFixture {
  def apply[T <: Console[Project]](constructor: String => T = { x =>
    new TestConsole(x)
  })(fun: (T, File) => Unit): Unit = {
    File.usingTemporaryDirectory("console") { workspaceDir =>
      File.usingTemporaryDirectory("console") { codeDir =>
        mkdir(codeDir / "dir1")
        mkdir(codeDir / "dir2")
        (codeDir / "dir1" / "foo.c")
          .write("int main(int argc, char **argv) { char *ptr = 0x1 + argv; return argc; }")
        (codeDir / "dir2" / "bar.c").write("int bar(int x) { return x; }")
        val console = constructor(workspaceDir.toString)
        fun(console, codeDir)
      }
    }
  }

}

object TestWorkspaceLoader extends WorkspaceLoader[Project] {
  override def createProject(projectFile: ProjectFile, path: Path): Project = Project(projectFile, path)
}

class TestConsole(workspaceDir: String) extends Console[Project](DefaultAmmoniteExecutor, TestWorkspaceLoader) {
  override def config = new ConsoleConfig(
    install = new InstallConfig(Map("SHIFTLEFT_OCULAR_INSTALL_DIR" -> workspaceDir))
  )

  override val cpgGenerator = new TestCpgGenerator(config)

}

class TestCpgGenerator(config: ConsoleConfig) extends CpgGenerator(config) {
  override def createFrontendByPath(
      inputPath: String,
  ): Option[LanguageFrontend] = {
    Some(new FuzzyCTestingFrontend)
  }

  override def createFrontendByLanguage(language: String): Option[LanguageFrontend] = {
    Some(new FuzzyCTestingFrontend)
  }

  private class FuzzyCTestingFrontend extends LanguageFrontend {

    override def generate(inputPath: String, outputPath: String, namespaces: List[String]): Option[String] = {
      val queue = new LinkedBlockingQueue[CpgStruct.Builder]()
      val factory =
        new io.shiftleft.fuzzyc2cpg.output.overflowdb.OutputModuleFactory(outputPath, queue)
      val fuzzyc = new FuzzyC2Cpg(factory)
      File(inputPath).list.foreach(println(_))
      fuzzyc.runAndOutput(Set(inputPath), Set(".c"))
      Some(outputPath)
    }

    def isAvailable: Boolean = true

  }

} 

package io.shiftleft.console.cpgcreation

import better.files.File

import scala.sys.process._


  def generate(inputPath: String, outputPath: String = "cpg.bin.zip", namespaces: List[String] = List()): Option[String]

  protected def runShellCommand(program: String, arguments: Seq[String]): Option[String] = {
    if (!File(program).exists) {
      System.err.println("Support for this language is only available in ShiftLeft Ocular with an appropriate license")
      return None
    }
    val cmd = Seq[String](program) ++ arguments
    val exitValue = cmd.run.exitValue()
    if (exitValue == 0) {
      Some(cmd.toString)
    } else {
      System.err.println(s"Error running shell command: $cmd")
      None
    }
  }

} 

package io.shiftleft.console.cpgcreation

import java.nio.file.Path

import better.files.Dsl._
import better.files.File
import io.shiftleft.codepropertygraph.cpgloading.{CpgLoader, CpgLoaderConfig}
import io.shiftleft.console.ConsoleConfig
import overflowdb.OdbConfig
import io.shiftleft.console.LanguageHelper.{cpgGeneratorForLanguage, languageIsKnown}

import scala.util.Try

class CpgGenerator(config: ConsoleConfig) {

  
  def createFrontendByLanguage(language: String): Option[LanguageFrontend] = {
    Some(language)
      .filter(languageIsKnown)
      .flatMap(
        lang =>
          cpgGeneratorForLanguage(
            lang,
            config.frontend,
            config.install.rootPath.path
        ))
  }

  def runLanguageFrontend(frontend: LanguageFrontend,
                          inputPath: String,
                          outputPath: String,
                          namespaces: List[String] = List()): Option[Path] = {
    val outputFileOpt: Option[File] =
      frontend.generate(inputPath, outputPath, namespaces).map(File(_))
    outputFileOpt.map { outFile =>
      val parentPath = outFile.parent.path.toAbsolutePath
      if (isZipFile(outFile)) {
        report("Creating database from bin.zip")
        val srcFilename = outFile.path.toAbsolutePath.toString
        val dstFilename = parentPath.resolve("cpg.bin").toAbsolutePath.toString
        // MemoryHelper.hintForInsufficientMemory(srcFilename).map(report)
        convertProtoCpgToOverflowDb(srcFilename, dstFilename)
      } else {
        report("moving cpg.bin.zip to cpg.bin because it is already a database file")
        val srcPath = parentPath.resolve("cpg.bin.zip")
        if (srcPath.toFile.exists()) {
          mv(srcPath, parentPath.resolve("cpg.bin"))
        }
      }
      parentPath
    }
  }

  def convertProtoCpgToOverflowDb(srcFilename: String, dstFilename: String): Unit = {
    val odbConfig = OdbConfig.withDefaults.withStorageLocation(dstFilename)
    val config = CpgLoaderConfig.withDefaults.doNotCreateIndexesOnLoad.withOverflowConfig(odbConfig)
    CpgLoader.load(srcFilename, config).close
    File(srcFilename).delete()
  }

  def isZipFile(file: File): Boolean = {
    val bytes = file.bytes
    Try {
      bytes.next() == 'P' && bytes.next() == 'K'
    }.getOrElse(false)
  }

  private def report(str: String): Unit = System.err.println(str)

} 

package io.shiftleft.semanticcpg.codedumper

import better.files.File
import io.shiftleft.codepropertygraph.generated.{Languages, nodes}
import io.shiftleft.semanticcpg.language._
import io.shiftleft.semanticcpg.language.NodeSteps
import io.shiftleft.codepropertygraph.Cpg
import overflowdb.OdbGraph
import io.shiftleft.utils.{Source, SourceHighlighter}
import org.apache.logging.log4j.LogManager

import scala.util.Try

object CodeDumper {

  private val logger = LogManager.getLogger(CodeDumper)
  val arrow: CharSequence = "
  def code(filename: String, startLine: Integer, endLine: Integer, lineToHighlight: Option[Integer] = None): String = {
    val lines = Try(File(filename).lines.toList).getOrElse {
      logger.warn("error reading from: " + filename);
      List()
    }
    lines
      .slice(startLine - 1, endLine)
      .zipWithIndex
      .map {
        case (line, lineNo) =>
          if (lineToHighlight.isDefined && lineNo == lineToHighlight.get - startLine) {
            line + " " + arrow
          } else {
            line
          }
      }
      .mkString("\n")
  }

} 

package io.shiftleft.semanticcpg.language.dotextension

import better.files.File

import scala.sys.process.Process
import scala.util.{Failure, Success, Try}

trait ImageViewer {
  def view(pathStr: String): Try[String]
}

object Shared {

  def plotAndDisplay(dotStrings: List[String], viewer: ImageViewer): Unit = {
    dotStrings.foreach { dotString =>
      File.usingTemporaryFile("semanticcpg") { dotFile =>
        File.usingTemporaryFile("semanticcpg") { svgFile =>
          dotFile.write(dotString)
          createSvgFile(dotFile, svgFile).toOption.foreach(_ => viewer.view(svgFile.path.toAbsolutePath.toString))
        }
      }
    }
  }

  private def createSvgFile(in: File, out: File): Try[String] = {
    Try {
      Process(Seq("dot", "-Tsvg", in.path.toAbsolutePath.toString, "-o", out.path.toAbsolutePath.toString)).!!
    } match {
      case Success(v) => Success(v)
      case Failure(exc) =>
        System.err.println("Executing `dot` failed: is `graphviz` installed?")
        System.err.println(exc)
        Failure(exc)
    }
  }

} 

package io.shiftleft.semanticcpg.layers

import better.files.File
import io.shiftleft.SerializedCpg
import io.shiftleft.codepropertygraph.Cpg
import org.apache.logging.log4j.LogManager
import io.shiftleft.semanticcpg.Overlays

abstract class LayerCreator {

  private val logger = LogManager.getLogger(this.getClass)

  val overlayName: String
  val description: String
  val dependsOn: List[String] = List()

  def run(context: LayerCreatorContext, serializeInverse: Boolean = false): Unit = {
    val appliedOverlays = Overlays.appliedOverlays(context.cpg).toSet
    if (!dependsOn.toSet.subsetOf(appliedOverlays)) {
      logger.warn(
        s"${this.getClass.getName} depends on $dependsOn but CPG only has $appliedOverlays - skipping creation")
    } else if (appliedOverlays.contains(overlayName)) {
      logger.warn(s"The overlay $overlayName already exists - skipping creation")
    } else {
      create(context, serializeInverse)
      Overlays.appendOverlayName(context.cpg, overlayName)
    }
  }

  protected def initSerializedCpg(outputDir: Option[String], passName: String, index: Int): SerializedCpg = {
    outputDir match {
      case Some(dir) => new SerializedCpg((File(dir) / s"${index}_$passName").path.toAbsolutePath.toString)
      case None      => new SerializedCpg()
    }
  }

  def create(context: LayerCreatorContext, serializeInverse: Boolean = false): Unit

  
  def probe(cpg: Cpg): Boolean

}

class LayerCreatorContext(val cpg: Cpg, val outputDir: Option[String] = None) {}
class LayerCreatorOptions() 

package stryker4s.mutants

import better.files.File
import grizzled.slf4j.Logging
import stryker4s.model.{MutatedFile, MutationsInSource, SourceTransformations}
import stryker4s.mutants.applymutants.{MatchBuilder, StatementTransformer}
import stryker4s.mutants.findmutants.MutantFinder

import scala.meta.Tree

class Mutator(mutantFinder: MutantFinder, transformer: StatementTransformer, matchBuilder: MatchBuilder)
    extends Logging {
  def mutate(files: Iterable[File]): Iterable[MutatedFile] = {
    val mutatedFiles = files
      .map { file =>
        val mutationsInSource = findMutants(file)
        val transformed = transformStatements(mutationsInSource)
        val builtTree = buildMatches(transformed)

        MutatedFile(file, builtTree, mutationsInSource.mutants, mutationsInSource.excluded)
      }
      .filterNot(mutatedFile => mutatedFile.mutants.isEmpty && mutatedFile.excludedMutants == 0)

    logMutationResult(mutatedFiles, files.size)

    mutatedFiles
  }

  
  private def buildMatches(transformedMutantsInSource: SourceTransformations): Tree =
    matchBuilder.buildNewSource(transformedMutantsInSource)

  private def logMutationResult(mutatedFiles: Iterable[MutatedFile], totalAmountOfFiles: Int): Unit = {
    val includedMutants = mutatedFiles.flatMap(_.mutants).size
    val excludedMutants = mutatedFiles.map(_.excludedMutants).sum
    val totalMutants = includedMutants + excludedMutants

    info(s"Found ${mutatedFiles.size} of $totalAmountOfFiles file(s) to be mutated.")
    info(s"$totalMutants Mutant(s) generated.${if (excludedMutants > 0)
      s" Of which $excludedMutants Mutant(s) are excluded."
    else ""}")

    if (totalAmountOfFiles == 0) {
      warn(s"No files marked to be mutated. ${dryRunText("mutate")}")
    } else if (includedMutants == 0 && excludedMutants > 0) {
      warn(s"All found mutations are excluded. ${dryRunText("excluded-mutations")}")
    } else if (totalMutants == 0) {
      info("Files to be mutated are found, but no mutations were found in those files.")
      info("If this is not intended, please check your configuration and try again.")
    }

    def dryRunText(configProperty: String): String =
      s"""Stryker4s will perform a dry-run without actually mutating anything.
         |You can configure the `$configProperty` property in your configuration""".stripMargin
  }
} 

package api

import better.files.File
import com.typesafe.config.ConfigFactory


trait Config {
  val configKey = "full-scala-stack"
  val config: com.typesafe.config.Config = {
    val confFileName =
      System.getProperty("application.conf", "./src/main/resources/application.conf")
    val confFile = File(confFileName)
    val config = ConfigFactory
      .parseFile(confFile.toJava)
      .withFallback(ConfigFactory.load())
    config
  }
} 

package com.nulabinc.backlog.r2b.exporter.actor

import java.util.concurrent.CountDownLatch

import akka.actor.Actor
import better.files.File
import com.nulabinc.backlog.migration.common.convert.Convert
import com.nulabinc.backlog.migration.common.domain.{BacklogComment, BacklogIssue, BacklogTextFormattingRule}
import com.nulabinc.backlog.migration.common.utils.{DateUtil, IOUtil, Logging}
import com.nulabinc.backlog.r2b.exporter.convert.{IssueWrites, JournalWrites}
import com.nulabinc.backlog.r2b.exporter.core.ExportContext
import com.nulabinc.backlog.r2b.exporter.service.{ChangeLogReducer, CommentReducer, IssueInitializer}
import com.taskadapter.redmineapi.Include
import com.taskadapter.redmineapi.bean.{Attachment, _}
import spray.json._

import scala.jdk.CollectionConverters._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._


private[exporter] class IssueActor(exportContext: ExportContext, backlogTextFormattingRule: BacklogTextFormattingRule) extends Actor with Logging {

  import com.nulabinc.backlog.migration.common.formatters.BacklogJsonProtocol._
  import IssueActor.ConsoleF

  private implicit val issueWrites: IssueWrites = exportContext.issueWrites
  private implicit val journalWrites: JournalWrites = exportContext.journalWrites


  override def preRestart(reason: Throwable, message: Option[Any]): Unit = {
    logger.debug(s"preRestart: reason: $reason, message: $message")
    for { value <- message } yield {
      context.system.scheduler.scheduleOnce(10.seconds, self, value)
    }
  }

  def receive: Receive = {
    case IssueActor.Do(issueId: Int, completion: CountDownLatch, allCount: Int, console: ConsoleF)=>
      logger.debug(s"[START ISSUE]$issueId thread numbers:${java.lang.Thread.activeCount()}")

      val issue                        = exportContext.issueService.issueOfId(issueId, Include.attachments, Include.journals)
      val journals                     = issue.getJournals.asScala.toSeq.sortWith((c1, c2) => c1.getCreatedOn.before(c2.getCreatedOn))
      val attachments: Seq[Attachment] = issue.getAttachments.asScala.toSeq

      exportIssue(issue, journals, attachments)
      exportComments(issue, journals, attachments)

      completion.countDown()
      console((allCount - completion.getCount).toInt, allCount)
  }

  private[this] def exportIssue(issue: Issue, journals: Seq[Journal], attachments: Seq[Attachment]): File = {
    val issueCreated     = DateUtil.tryIsoParse(Option(issue.getCreatedOn).map(DateUtil.isoFormat))
    val issueDirPath     = exportContext.backlogPaths.issueDirectoryPath("issue", issue.getId.intValue(), issueCreated, 0)
    val issueInitializer = new IssueInitializer(exportContext, issueDirPath, journals, attachments, backlogTextFormattingRule)
    val backlogIssue     = issueInitializer.initialize(issue)

    IOUtil.output(exportContext.backlogPaths.issueJson(issueDirPath), backlogIssue.toJson.prettyPrint)
  }

  private[this] def exportComments(issue: Issue, journals: Seq[Journal], attachments: Seq[Attachment]): Unit = {
    val backlogIssue    = Convert.toBacklog(issue)
    val backlogComments = journals.map(Convert.toBacklog(_))
    backlogComments.zipWithIndex.foreach {
      case (comment, index) =>
        exportComment(comment, backlogIssue, backlogComments, attachments, index)
    }
  }

  private[this] def exportComment(comment: BacklogComment,
                                  issue: BacklogIssue,
                                  comments: Seq[BacklogComment],
                                  attachments: Seq[Attachment],
                                  index: Int) : File = {
    val commentCreated   = DateUtil.tryIsoParse(comment.optCreated)
    val issueDirPath     = exportContext.backlogPaths.issueDirectoryPath("comment", issue.id, commentCreated, index)
    val changeLogReducer = new ChangeLogReducer(exportContext, issueDirPath, issue, comments, attachments)
    val commentReducer   = new CommentReducer(issue.id, changeLogReducer)
    val reduced          = commentReducer.reduce(comment)

    IOUtil.output(exportContext.backlogPaths.issueJson(issueDirPath), reduced.toJson.prettyPrint)
  }

}

private[exporter] object IssueActor {

  type ConsoleF = (Int, Int) => Unit

  case class Do(issueId: Int, completion: CountDownLatch, allCount: Int, console: ConsoleF)

} 

package com.nulabinc.backlog.r2b.mapping.file

import better.files.File
import com.nulabinc.backlog.migration.common.conf.{BacklogApiConfiguration, BacklogConfiguration}
import com.nulabinc.backlog.migration.common.domain.BacklogUser
import com.nulabinc.backlog.migration.common.modules.{ServiceInjector => BacklogInjector}
import com.nulabinc.backlog.migration.common.service.{UserService => BacklogUserService}
import com.nulabinc.backlog.migration.common.utils.{IOUtil, StringUtil}
import com.nulabinc.backlog.r2b.mapping.core.MappingDirectory
import com.nulabinc.backlog.r2b.mapping.domain.MappingJsonProtocol._
import com.nulabinc.backlog.r2b.mapping.domain.{Mapping, MappingsWrapper}
import com.nulabinc.backlog.r2b.redmine.conf.RedmineApiConfiguration
import com.nulabinc.backlog.r2b.redmine.modules.{ServiceInjector => RedmineInjector}
import com.nulabinc.backlog.r2b.redmine.service.{UserService => RedmineUserService}
import com.osinka.i18n.Messages
import com.taskadapter.redmineapi.bean.{User => RedmineUser}
import spray.json.JsonParser


class UserMappingFile(redmineApiConfig: RedmineApiConfiguration, backlogApiConfig: BacklogApiConfiguration, users: Seq[RedmineUser])
    extends MappingFile
    with BacklogConfiguration {

  private[this] val redmineItems = getRedmineItems()
  private[this] val backlogItems = getBacklogItems()

  private[this] def getRedmineItems(): Seq[MappingItem] = {
    val injector    = RedmineInjector.createInjector(redmineApiConfig)
    val userService = injector.getInstance(classOf[RedmineUserService])

    def resolve(user: RedmineUser): Option[RedmineUser] = {
      (Option(user.getLogin), Option(user.getFullName)) match {
        case (Some(_), Some(_)) => Some(user)
        case _                  => userService.optUserOfId(user.getId)
      }
    }

    def condition(user: RedmineUser): Boolean = {
      StringUtil.notEmpty(user.getLogin).nonEmpty
    }

    def createItem(user: RedmineUser): MappingItem = {
      MappingItem(user.getLogin, user.getFullName)
    }

    val redmineUsers = users.flatMap(resolve).filter(condition)
    redmineUsers.map(createItem)
  }

  private[this] def getBacklogItems(): Seq[MappingItem] = {
    def createItem(user: BacklogUser): MappingItem = {
      if (backlogApiConfig.url.contains(NaiSpaceDomain)) {
        MappingItem(user.optMailAddress.getOrElse(""), user.name)
      } else {
        MappingItem(user.optUserId.getOrElse(""), user.name)
      }
    }
    val backlogUsers = allUsers()
    backlogUsers.map(createItem)
  }

  private[this] def allUsers(): Seq[BacklogUser] = {
    val injector    = BacklogInjector.createInjector(backlogApiConfig)
    val userService = injector.getInstance(classOf[BacklogUserService])
    userService.allUsers()
  }

  private[this] def convertForNAI(backlogUsers: Seq[BacklogUser])(mapping: Mapping) = {
    if (backlogApiConfig.url.contains(NaiSpaceDomain)) {
      val targetBacklogUser = backlogUsers
        .find(backlogUser => backlogUser.optMailAddress.getOrElse("") == mapping.backlog)
        .getOrElse(throw new NoSuchElementException(s"User ${mapping.backlog} not found"))
      mapping.copy(backlog = targetBacklogUser.optUserId.getOrElse(s"UserId ${mapping.backlog} not found"))
    } else mapping
  }

  override def tryUnmarshal(): Seq[Mapping] = {
    val path    = File(filePath).path.toAbsolutePath
    val json    = IOUtil.input(path).getOrElse("")
    val convert = convertForNAI(allUsers()) _
    JsonParser(json).convertTo[MappingsWrapper].mappings.map(convert)
  }

  override def matchItem(redmine: MappingItem): String =
    backlogs.map(_.name).find(_ == redmine.name).getOrElse("")

  override def redmines: Seq[MappingItem] = redmineItems

  override def backlogs: Seq[MappingItem] = backlogItems

  override def filePath: String = MappingDirectory.USER_MAPPING_FILE

  override def itemName: String = Messages("common.users")

  override def description: String = {
    Messages("cli.mapping.configurable", itemName, backlogs.map(_.name).mkString(","))
  }

  override def isDisplayDetail: Boolean = true

} 

package io.brunk

import better.files.File

import scala.util.Random.shuffle


object DatasetSplitter {
  val datasetSplitNames = Seq("train", "validation", "test")

  def main(args: Array[String]): Unit = {
    val inputDir = File(args(0))
    val outputDir = File(args(1))
    val splitSizes = args.drop(2).map(_.toFloat).toSeq

    val imgClassDirs = inputDir.list.filter(_.isDirectory).toVector.sortBy(_.name)

    // in case of different numbers of samples per class, use the smallest one
    val numSamplesPerClass = imgClassDirs.map(_.glob("*.jpg").size).min
    println(s"Number of samples per class (balanced): $numSamplesPerClass")

    val samplesPerClass =  {
      imgClassDirs.flatMap { imgClassDir =>
        shuffle(imgClassDir.children.toVector)
          .take(numSamplesPerClass)       // balance samples to have the same number for each class
          .map((imgClassDir.name, _))
      }.groupBy(_._1).mapValues(_.map(_._2))
    }

    val numSamples = samplesPerClass.map(_._2.size).sum
    println(s"Number of samples (balanced): $numSamples")
    val splitSizesAbsolute = splitSizes.map(_ / 100.0).map(_ * numSamplesPerClass).map(_.toInt)
    println(s"Number of samples per split: ${datasetSplitNames.zip(splitSizesAbsolute).mkString(" ")}")
    val splitIndices = splitSizesAbsolute
      .map(_ -1)
      .scanLeft(-1 to -1)((prev, current) => prev.last + 1 to (prev.last + current + 1)).tail // TODO cleaner solution
    println(splitIndices)

    val datasetNamesWithIndices = datasetSplitNames.zip(splitIndices)

    val datasetIndices = (for {
      (name, indices) <- datasetNamesWithIndices
      index <- indices
    } yield (index, name))
      .sortBy(_._1)
      .map(_._2)

    // create directories
    for {
      dataset <- datasetSplitNames
      imgClassDir <- imgClassDirs
      imgClass = imgClassDir.name
    } {
      (outputDir/dataset/imgClass).createDirectories()
    }
    // write into train, validation and test folders
    for {
      (imgClass, samples) <- samplesPerClass
      (filename, dataset) <- samples.zip(datasetIndices)
    } {
      filename.copyTo(outputDir/dataset/imgClass/filename.name)
    }

  }
} 

package stryker4s.testutil.stubs

import better.files.File
import stryker4s.run.process.{Command, ProcessRunner}

import scala.util.{Success, Try}

object TestProcessRunner {
  def apply(testRunExitCode: Try[Int]*): TestProcessRunner = new TestProcessRunner(true, testRunExitCode: _*)
  def failInitialTestRun(): TestProcessRunner = new TestProcessRunner(false)
}

class TestProcessRunner(initialTestRunSuccess: Boolean, testRunExitCode: Try[Int]*) extends ProcessRunner {
  val timesCalled: Iterator[Int] = Iterator.from(0)

  
  override def apply(command: Command, workingDir: File, envVar: (String, String)): Try[Int] = {
    if (envVar._2.equals("None")) {
      Success(if (initialTestRunSuccess) 0 else 1)
    } else {
      timesCalled.next()
      testRunExitCode(envVar._2.toInt)
    }
  }
} 

package stryker4s.config

import better.files.File
import stryker4s.testutil.Stryker4sSuite

class ConfigTest extends Stryker4sSuite {
  describe("toHoconString") {
    it("should print toString with default values") {
      val sut = Config.default

      val result = Config.toHoconString(sut)

      val expected =
        s"""base-dir="${File.currentWorkingDirectory.pathAsString.replace("\\", "\\\\")}"
           |dashboard {
           |    base-url="https://dashboard.stryker-mutator.io"
           |    report-type=full
           |}
           |excluded-mutations=[]
           |mutate=[
           |    "**/main/scalaBar.scala"
           |]
           |reporters=[
           |    html
           |]
           |test-filter=[]
           |thresholds {
           |    break=0
           |    high=80
           |    low=60
           |}
           |""".stripMargin
      result.toString should equal(expected.toString)
    }
  }
} 

package stryker4s.report.mapper
import java.nio.file.Path

import better.files.File
import mutationtesting._
import org.scalatest.Inside
import stryker4s.config.{Config, Thresholds => ConfigThresholds}
import stryker4s.extension.FileExtensions._
import stryker4s.extension.ImplicitMutationConversion._
import stryker4s.extension.mutationtype._
import stryker4s.model.{Killed, Mutant, Survived}
import stryker4s.scalatest.FileUtil
import stryker4s.testutil.Stryker4sSuite

import scala.meta.{Lit, Term}

class MutantRunResultMapperTest extends Stryker4sSuite with Inside {
  describe("mapper") {
    it("should map 4 files to valid MutationTestReport") {
      val sut = new MutantRunResultMapper {}
      implicit val config: Config = Config(thresholds = ConfigThresholds(high = 60, low = 40))

      val path = FileUtil.getResource("scalaFiles/ExampleClass.scala").relativePath
      val mutantRunResult = Killed(
        toMutant(0, EqualTo, NotEqualTo, path),
        path
      )
      val mutantRunResult2 = Survived(
        toMutant(1, Lit.String("Hugo"), EmptyString, path),
        path
      )
      val path3 = FileUtil.getResource("scalaFiles/simpleFile.scala").relativePath
      val mutantRunResult3 = Killed(
        toMutant(0, GreaterThan, LesserThan, path3),
        path3
      )

      val mutationRunResults = List(mutantRunResult, mutantRunResult2, mutantRunResult3)

      val result = sut.toReport(mutationRunResults)
      inside(result) {
        case MutationTestReport(_, _, thresholds, files) =>
          thresholds should equal(Thresholds(high = 60, low = 40))
          files should have size 2
          val firstResult = files.find(_._1.endsWith("scalaFiles/ExampleClass.scala")).value
          files.find(_._1.endsWith("scalaFiles/simpleFile.scala")).value
          inside(firstResult._2) {
            case MutationTestResult(source, mutants, language) =>
              language should equal("scala")
              mutants should (
                contain.only(
                  MutantResult(
                    "0",
                    "EqualityOperator",
                    "!=",
                    Location(Position(4, 27), Position(4, 29)),
                    MutantStatus.Killed
                  ),
                  MutantResult(
                    "1",
                    "StringLiteral",
                    "\"\"",
                    Location(Position(6, 31), Position(6, 37)),
                    MutantStatus.Survived
                  )
                )
              )
              source should equal(FileUtil.getResource("scalaFiles/ExampleClass.scala").contentAsString)
          }
      }
    }
  }

  
  private def toMutant(id: Int, original: Term, category: SubstitutionMutation[_ <: Term], file: Path) = {
    import stryker4s.extension.TreeExtensions.FindExtension

    import scala.meta._
    val parsed = File(file).contentAsString.parse[Source]
    val foundOrig = parsed.get.find(original).value
    Mutant(id, foundOrig, category.tree, category)
  }
} 

package stryker4s.run.process

import better.files.File
import grizzled.slf4j.Logging

import scala.concurrent.duration.{Duration, MINUTES}
import scala.sys.process.{Process, ProcessLogger}
import scala.util.Try
import cats.effect.IO

trait ProcessRunner extends Logging {
  def apply(command: Command, workingDir: File): Try[Seq[String]] = {
    Try {
      Process(s"${command.command} ${command.args}", workingDir.toJava)
        .!!<(ProcessLogger(debug(_)))
        .linesIterator
        .toSeq
    }
  }

  def apply(command: Command, workingDir: File, envVar: (String, String)): Try[Int] = {
    val mutantProcess = Process(s"${command.command} ${command.args}", workingDir.toJava, envVar)
      .run(ProcessLogger(debug(_)))

    val exitCodeFuture = IO(mutantProcess.exitValue())
    // TODO: Maybe don't use unsafeRunTimed
    // TODO: Use timeout decided by initial test-run duration
    Try(exitCodeFuture.unsafeRunTimed(Duration(2, MINUTES)).get)
  }
}

object ProcessRunner {
  private def isWindows: Boolean = sys.props("os.name").toLowerCase.contains("windows")

  def apply(): ProcessRunner = {
    if (isWindows) new WindowsProcessRunner
    else new UnixProcessRunner
  }
} 

package stryker4s.run

import java.nio.file.Path

import better.files.File
import grizzled.slf4j.Logging
import mutationtesting.{Metrics, MetricsResult}
import stryker4s.config.Config
import stryker4s.extension.FileExtensions._
import stryker4s.model._
import stryker4s.mutants.findmutants.SourceCollector
import stryker4s.report.Reporter
import stryker4s.report.mapper.MutantRunResultMapper
import stryker4s.report.FinishedRunReport
import scala.concurrent.duration._
import stryker4s.extension.exception.InitialTestRunFailedException

abstract class MutantRunner(sourceCollector: SourceCollector, reporter: Reporter)(implicit
    config: Config
) extends MutantRunResultMapper
    with Logging {
  type Context <: TestRunnerContext

  def apply(mutatedFiles: Iterable[MutatedFile]): MetricsResult = {
    val context = prepareEnv(mutatedFiles)

    initialTestRun(context)

    val runResults = runMutants(mutatedFiles, context)

    val report = toReport(runResults)
    val metrics = Metrics.calculateMetrics(report)

    // Timeout of 15 seconds is a little longer to make sure http requests etc finish
    // TODO: Don't use unsafeRun
    reporter.reportRunFinished(FinishedRunReport(report, metrics)).unsafeRunTimed(15.seconds)
    metrics
  }

  private def prepareEnv(mutatedFiles: Iterable[MutatedFile]): Context = {
    val files = sourceCollector.filesToCopy
    val tmpDir: File = {
      val targetFolder = config.baseDir / "target"
      targetFolder.createDirectoryIfNotExists()

      File.newTemporaryDirectory("stryker4s-", Some(targetFolder))
    }

    debug("Using temp directory: " + tmpDir)

    files.foreach(copyFile(_, tmpDir))

    // Overwrite files to mutated files
    mutatedFiles.foreach(writeMutatedFile(_, tmpDir))
    initializeTestContext(tmpDir)
  }

  private def copyFile(file: File, tmpDir: File): Unit = {
    val filePath = tmpDir / file.relativePath.toString

    filePath.createIfNotExists(file.isDirectory, createParents = true)

    val _ = file.copyTo(filePath, overwrite = true)
  }

  private def writeMutatedFile(mutatedFile: MutatedFile, tmpDir: File): File = {
    val filePath = mutatedFile.fileOrigin.inSubDir(tmpDir)
    filePath.overwrite(mutatedFile.tree.syntax)
  }

  private def runMutants(mutatedFiles: Iterable[MutatedFile], context: Context): Iterable[MutantRunResult] =
    for {
      mutatedFile <- mutatedFiles
      subPath = mutatedFile.fileOrigin.relativePath
      mutant <- mutatedFile.mutants
    } yield {
      val totalMutants = mutatedFiles.flatMap(_.mutants).size
      // TODO: Don't use unsafeRun
      reporter.reportMutationStart(mutant).unsafeRunTimed(5.seconds)
      val result = runMutant(mutant, context)(subPath)
      reporter.reportMutationComplete(result, totalMutants).unsafeRunTimed(5.seconds)
      result
    }

  def runMutant(mutant: Mutant, context: Context): Path => MutantRunResult

  def initialTestRun(context: Context): Unit = {
    info("Starting initial test run...")
    if (!runInitialTest(context)) {
      throw InitialTestRunFailedException(
        "Initial test run failed. Please make sure your tests pass before running Stryker4s."
      )
    }
    info("Initial test run succeeded! Testing mutants...")
  }

  def runInitialTest(context: Context): Boolean

  def initializeTestContext(tmpDir: File): Context
} 

package stryker4s.config.implicits

import java.nio.file.Path

import better.files.File
import com.typesafe.config.ConfigRenderOptions
import pureconfig.ConfigWriter
import stryker4s.config.{DashboardReportType, ExcludedMutations, ReporterType}
import pureconfig.generic.semiauto._

object ConfigWriterImplicits {
  implicit private[config] val fileWriter: ConfigWriter[File] =
    ConfigWriter[Path] contramap (_.path)

  implicit private[config] val exclusionsWriter: ConfigWriter[ExcludedMutations] =
    ConfigWriter[List[String]] contramap (_.exclusions.toList)

  implicit private[config] val reportersWriter: ConfigWriter[ReporterType] =
    deriveEnumerationWriter[ReporterType]

  implicit private[config] val dashboardReportTypeWriter: ConfigWriter[DashboardReportType] =
    deriveEnumerationWriter[DashboardReportType]

  private[config] val options: ConfigRenderOptions = ConfigRenderOptions
    .defaults()
    .setOriginComments(false)
    .setJson(false)
} 

package io.shiftleft.utils

import better.files.File
import io.shiftleft.codepropertygraph.generated.Languages
import org.apache.logging.log4j.LogManager
import scala.sys.process.Process


case class Source(code: String, language: String)

object SourceHighlighter {
  private val logger = LogManager.getLogger(this)

  def highlight(source: Source): Option[String] = {
    val langFlag = source.language match {
      case Languages.C => "-sC"
      case other       => throw new RuntimeException(s"Attempting to call highlighter on unsupported language: $other")
    }

    val tmpSrcFile = File.newTemporaryFile("dump")
    tmpSrcFile.writeText(source.code)
    try {
      val highlightedCode = Process(Seq("source-highlight-esc.sh", tmpSrcFile.path.toString, langFlag)).!!
      Some(highlightedCode)
    } catch {
      case exception: Exception =>
        logger.info("syntax highlighting not working. Is `source-highlight` installed?")
        logger.info(exception)
        Some(source.code)
    } finally {
      tmpSrcFile.delete()
    }
  }

} 

package stryker4s.mutants.findmutants

import better.files.File
import grizzled.slf4j.Logging
import stryker4s.config.Config
import stryker4s.extension.FileExtensions._
import stryker4s.model.{Mutant, MutationsInSource}

import scala.meta.Source
import scala.meta.parsers.{Parsed, XtensionParseInputLike}

class MutantFinder(matcher: MutantMatcher)(implicit config: Config) extends Logging {
  def mutantsInFile(filePath: File): MutationsInSource = {
    val parsedSource = parseFile(filePath)
    val (included, excluded) = findMutants(parsedSource)
    MutationsInSource(parsedSource, included, excluded)
  }

  def findMutants(source: Source): (Seq[Mutant], Int) = {
    val (included, excluded) = source.collect(matcher.allMatchers).flatten.partition(_.isDefined)
    (included.flatten, excluded.size)
  }

  def parseFile(file: File): Source =
    file.toJava.parse[Source] match {
      case Parsed.Success(source) =>
        source
      case Parsed.Error(_, msg, ex) =>
        error(s"Error while parsing file '${file.relativePath}', $msg")
        throw ex
    }
} 

package stryker4s.command.runner

import java.nio.file.Path

import better.files.File
import stryker4s.config.Config
import stryker4s.model._
import stryker4s.mutants.findmutants.SourceCollector
import stryker4s.report.Reporter
import stryker4s.run.MutantRunner
import stryker4s.run.process.{Command, ProcessRunner}

import scala.concurrent.TimeoutException
import scala.util.{Failure, Success}

class ProcessMutantRunner(
    command: Command,
    processRunner: ProcessRunner,
    sourceCollector: SourceCollector,
    reporter: Reporter
)(implicit config: Config)
    extends MutantRunner(sourceCollector, reporter) {
  type Context = CommandRunnerContext

  override def runMutant(mutant: Mutant, context: Context): Path => MutantRunResult = {
    val id = mutant.id
    processRunner(command, context.tmpDir, ("ACTIVE_MUTATION", id.toString)) match {
      case Success(0)                         => Survived(mutant, _)
      case Success(exitCode) if exitCode != 0 => Killed(mutant, _)
      case Failure(_: TimeoutException)       => TimedOut(mutant, _)
      case _                                  => Error(mutant, _)
    }
  }

  override def runInitialTest(context: Context): Boolean = {
    processRunner(command, context.tmpDir, ("ACTIVE_MUTATION", "None")) match {
      case Success(0)                         => true
      case Success(exitCode) if exitCode != 0 => false
      case Failure(_: TimeoutException)       => false
    }
  }

  override def initializeTestContext(tmpDir: File): Context = CommandRunnerContext(tmpDir)

} 

package stryker4s.sbt.runner

import java.io.{PrintStream, File => JFile}
import java.nio.file.Path

import better.files.{File, _}
import sbt.Keys._
import sbt._
import sbt.internal.LogManager
import stryker4s.config.{Config, TestFilter}
import stryker4s.extension.FileExtensions._
import stryker4s.extension.exception.InitialTestRunFailedException
import stryker4s.model._
import stryker4s.mutants.findmutants.SourceCollector
import stryker4s.report.Reporter
import stryker4s.run.MutantRunner
import stryker4s.sbt.Stryker4sMain.autoImport.stryker
import sbt.Tests.Output

class SbtMutantRunner(state: State, sourceCollector: SourceCollector, reporter: Reporter)(implicit config: Config)
    extends MutantRunner(sourceCollector, reporter) {
  type Context = SbtRunnerContext

  
  private def runTests[T](state: State, onError: => T, onSuccess: => T, onFailed: => T): T =
    Project.runTask(executeTests in Test, state) match {
      case Some((_, Value(Output(TestResult.Passed, _, _)))) => onSuccess
      case Some((_, Value(Output(TestResult.Failed, _, _)))) => onFailed
      case _                                                 => onError
    }

  private def mutationSetting(mutation: Int): Def.Setting[_] =
    javaOptions in Test += s"-DACTIVE_MUTATION=${String.valueOf(mutation)}"

  private def tmpDirFor(conf: Configuration, tmpDir: File): Def.Initialize[JFile] =
    (scalaSource in conf)(_.toScala)(source => (source inSubDir tmpDir).toJava)
} 

package be.dataminded.lighthouse.testing

import better.files.File
import org.apache.spark.sql.SparkSession

trait SharedSparkSession {

  private object AutoImport {
    lazy val tmp: File = File.newTemporaryDirectory().deleteOnExit()

    lazy val localMetastorePath: String  = (tmp / "metastore").pathAsString
    lazy val localWarehousePath: String  = (tmp / "warehouse").pathAsString
    lazy val checkPointDirectory: String = (tmp / "checkpoint").pathAsString
  }

  import AutoImport._

  lazy val spark: SparkSession = SparkSession
    .builder()
    .master("local[*]")
    .appName("LocalSparkTesting")
    .config("javax.jdo.option.ConnectionURL", s"jdbc:derby:;databaseName=$localMetastorePath;create=true")
    .config("datanucleus.rdbms.datastoreAdapterClassName", "org.datanucleus.store.rdbms.adapter.DerbyAdapter")
    .config("spark.sql.streaming.checkpointLocation", checkPointDirectory)
    .config("spark.sql.warehouse.dir", localWarehousePath)
    .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
    .config("spark.sql.avro.compression.codec", "snappy")
    .config("spark.sql.parquet.compression.codec", "snappy")
    .config("spark.ui.enabled", "false")
    .config("spark.sql.sources.partitionOverwriteMode", "dynamic")
    .enableHiveSupport()
    .getOrCreate()
} 

package sangria.gateway.schema

import akka.NotUsed
import akka.stream.scaladsl.Source
import better.files.File
import io.circe.Json
import sangria.execution.{Executor, Middleware}
import sangria.execution.deferred.DeferredResolver
import sangria.schema.Schema

import scala.concurrent.{Await, Future}

trait SchemaProvider[Ctx, Val] {
  def schemaInfo: Future[Option[SchemaInfo[Ctx, Val]]]
  def schemaChanges: Option[Source[Boolean, NotUsed]]
}

case class SchemaInfo[Ctx, Val](
  schema: Schema[Ctx, Val],
  ctx: Ctx,
  value: Val,
  middleware: List[Middleware[Ctx]],
  deferredResolver: DeferredResolver[Ctx] = DeferredResolver.empty,
  schemaRendered: String,
  schemaIntrospection: Json,
  files: Vector[File]) 

package sangria.gateway.schema

import java.util.concurrent.atomic.AtomicReference

import akka.actor.ActorSystem
import akka.stream.{Materializer, OverflowStrategy}
import akka.stream.scaladsl.{BroadcastHub, Keep, RunnableGraph, Source}
import better.files.File
import sangria.gateway.AppConfig
import sangria.gateway.file.FileMonitorActor
import sangria.gateway.http.client.HttpClient
import sangria.gateway.schema.materializer.{GatewayContext, GatewayMaterializer}
import sangria.gateway.util.Logging

import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success}

// TODO: on a timer reload all external schemas and check for changes
class ReloadableSchemaProvider(config: AppConfig, client: HttpClient, mat: GatewayMaterializer)(implicit system: ActorSystem, ec: ExecutionContext, amat: Materializer) extends SchemaProvider[GatewayContext, Any] with Logging {
  val loader = new SchemaLoader(config, client, mat)
  val schemaRef = new AtomicReference[Option[SchemaInfo[GatewayContext, Any]]](None)

  system.actorOf(FileMonitorActor.props(config.watch.allFiles, config.watch.threshold, config.watch.allGlobs, reloadSchema))

  private val producer = Source.actorRef[Boolean](100, OverflowStrategy.dropTail)
  private val runnableGraph = producer.toMat(BroadcastHub.sink(bufferSize = 256))(Keep.both)
  private val (changesPublisher, changesSource) = runnableGraph.run()

  val schemaChanges = Some(changesSource)

  def schemaInfo =
    schemaRef.get() match {
      case v @ Some(_) ⇒ Future.successful(v)
      case None ⇒ reloadSchema
    }

  def reloadSchema(files: Vector[File]): Unit = {
    logger.info(s"Schema files are changed: ${files mkString ", "}. Reloading schema")

    reloadSchema
  }

  def reloadSchema: Future[Option[SchemaInfo[GatewayContext, Any]]] =
    loader.loadSchema.andThen {
      case Success(Some(newSchema)) ⇒
        schemaRef.get() match {
          case Some(currentSchema) ⇒
            val changes = newSchema.schema.compare(currentSchema.schema)
            val renderedChanges =
              if (changes.nonEmpty)
                " with following changes:\n" + changes.map(c ⇒ "  * " + c.description + (if (c.breakingChange) " (breaking)" else "")).mkString("\n")
              else
                " without any changes."

            changesPublisher ! true
            logger.info(s"Schema successfully reloaded$renderedChanges")
          case None ⇒
            logger.info(s"Schema successfully loaded from files:\n${newSchema.files.map(f ⇒ "  * " + f).mkString("\n")}")
        }

        schemaRef.set(Some(newSchema))
      case Failure(error) ⇒
        logger.error("Failed to load the schema", error)
    }
} 

package sangria.gateway.schema

import better.files.File
import sangria.ast.Document
import sangria.execution.Executor
import sangria.execution.deferred.DeferredResolver
import sangria.gateway.AppConfig
import sangria.gateway.file.FileUtil
import sangria.gateway.http.client.HttpClient
import sangria.gateway.schema.materializer.{GatewayContext, GatewayMaterializer}
import sangria.gateway.util.Logging
import sangria.parser.QueryParser
import sangria.schema.Schema
import sangria.marshalling.circe._

import scala.concurrent.{ExecutionContext, Future}
import scala.util.control.NonFatal
import scala.util.{Failure, Success}

class SchemaLoader(config: AppConfig, client: HttpClient, mat: GatewayMaterializer)(implicit ec: ExecutionContext) extends Logging {
  def loadSchema: Future[Option[SchemaInfo[GatewayContext, Any]]] = {
    val files = FileUtil.loadFiles(config.watch.allFiles, config.watch.allGlobs)

    if (files.nonEmpty) {
      val parsed =
        files.map {
          case (path, content) ⇒ path → QueryParser.parse(content)
        }

      val failed = parsed.collect {case (path, Failure(e)) ⇒ path → e}

      if (failed.nonEmpty) {
        failed.foreach { case (path, error) ⇒
          logger.error(s"Can't parse file '$path':\n${error.getMessage}")
        }

        Future.successful(None)
      } else {
        val successful = parsed.collect {case (path, Success(doc)) ⇒ path → doc}
        val document = Document.merge(successful.map(_._2))

        try {
          val info =
            for {
              ctx ← GatewayContext.loadContext(config, client, document)
              schema = Schema.buildFromAst(document, mat.schemaBuilder(ctx).validateSchemaWithException(document))
              intro ← executeIntrospection(schema, ctx)
            } yield Some(SchemaInfo(
              schema,
              ctx,
              (),
              Nil,
              DeferredResolver.empty,
              schema.renderPretty,
              intro,
              files.map(_._1)))

          info.recover(handleError(files))
        } catch {
          case e if handleError(files).isDefinedAt(e) ⇒
            Future.successful(handleError(files)(e))
        }
      }
    } else {
      logger.error("No schema files found!")
      Future.successful(None)
    }
  }

  private def handleError(files: Vector[(File, String)]): PartialFunction[Throwable, Option[SchemaInfo[GatewayContext, Any]]] = {
    case NonFatal(e) ⇒
      logger.error(s"Can't create the schema from files: ${files.map(_._1).mkString(", ")}. " + e.getMessage)
      None
  }

  private def executeIntrospection(schema: Schema[GatewayContext, Any], ctx: GatewayContext) =
    Executor.execute(schema, sangria.introspection.introspectionQuery(schemaDescription = false, directiveRepeatableFlag = false), ctx)
} 

package sangria.gateway

import better.files.File
import com.typesafe.config.Config
import net.ceedubs.ficus.Ficus._
import net.ceedubs.ficus.readers.ArbitraryTypeReader._

import scala.concurrent.duration.FiniteDuration

case class WatchConfig(
  enabled: Boolean,
  paths: Seq[String],
  pathsStr: Option[String],
  threshold: FiniteDuration,
  glob: Seq[String],
  globStr: Option[String]
) {
  lazy val allPaths = pathsStr.map(_.split("\\s*,\\s*").toSeq) getOrElse paths
  lazy val allFiles = allPaths.map(File(_))
  lazy val allGlobs = globStr.map(_.split("\\s*,\\s*").toSeq) getOrElse glob
}

case class LimitConfig(
  complexity: Double,
  maxDepth: Int,
  allowIntrospection: Boolean)

case class SlowLogConfig(
  enabled: Boolean,
  threshold: FiniteDuration,
  extension: Boolean,
  apolloTracing: Boolean)

case class AppConfig(
  port: Int,
  bindHost: String,
  graphiql: Boolean,
  playground: Boolean,
  slowLog: SlowLogConfig,
  watch: WatchConfig,
  limit: LimitConfig,
  includeDirectives: Option[Seq[String]],
  includeDirectivesStr: Option[String],
  excludeDirectives: Option[Seq[String]],
  excludeDirectivesStr: Option[String]
) {
  lazy val allIncludeDirectives = includeDirectivesStr.map(_.split("\\s*,\\s*").toSeq) orElse includeDirectives
  lazy val allExcludeDirectives = excludeDirectivesStr.map(_.split("\\s*,\\s*").toSeq) orElse excludeDirectives

  def isEnabled(directivesName: String) =
    !allExcludeDirectives.exists(_.contains(directivesName)) && (
      allIncludeDirectives.isEmpty ||
      allIncludeDirectives.exists(_.contains(directivesName)))
}

object AppConfig {
  def load(config: Config): AppConfig = config.as[AppConfig]
} 

package sangria.gateway.file

import akka.actor._
import better.files.{Disposable, File, newMultiMap, repeat}
import better.files._

import scala.collection.mutable


class FileWatcher(file: File, maxDepth: Int) extends Actor {
  import FileWatcher._

  def this(file: File, recursive: Boolean = true) = this(file, if (recursive) Int.MaxValue else 0)

  protected[this] val callbacks = newMultiMap[Event, Callback]

  protected[this] val monitor: File.Monitor = new FileMonitor(file, maxDepth) {
    override def onEvent(event: Event, file: File, count: Int) = self ! Message.NewEvent(event, file, count)
    override def onException(exception: Throwable) = self ! Status.Failure(exception)
  }

  override def preStart() = monitor.start()(executionContext = context.dispatcher)

  override def receive = {
    case Message.NewEvent(event, target, count) if callbacks.contains(event) => callbacks(event).foreach(f => repeat(count)(f(event -> target)))
    case Message.RegisterCallback(events, callback) => events.foreach(event => callbacks.addBinding(event, callback))
    case Message.RemoveCallback(event, callback) => callbacks.removeBinding(event, callback)
  }

  override def postStop() = monitor.stop()
}

object FileWatcher {
  import java.nio.file.{Path, WatchEvent}

  type Event = WatchEvent.Kind[Path]
  type Callback = PartialFunction[(Event, File), Unit]

  sealed trait Message
  object Message {
    case class NewEvent(event: Event, file: File, count: Int) extends Message
    case class RegisterCallback(events: Traversable[Event], callback: Callback) extends Message
    case class RemoveCallback(event: Event, callback: Callback) extends Message
  }

  implicit val disposeActorSystem: Disposable[ActorSystem] =
    Disposable(_.terminate())

  implicit class FileWatcherOps(file: File) {
    def watcherProps(recursive: Boolean): Props =
      Props(new FileWatcher(file, recursive))

    def newWatcher(recursive: Boolean = true)(implicit ctx: ActorRefFactory): ActorRef =
      ctx.actorOf(watcherProps(recursive))
  }

  def when(events: Event*)(callback: Callback): Message =
    Message.RegisterCallback(events, callback)

  def on(event: Event)(callback: File => Unit): Message =
    when(event) { case (`event`, file) => callback(file) }

  def stop(event: Event, callback: Callback): Message =
    Message.RemoveCallback(event, callback)

  private def newMultiMap[A, B]: mutable.MultiMap[A, B] = new mutable.HashMap[A, mutable.Set[B]] with mutable.MultiMap[A, B]
  @inline private def repeat[U](n: Int)(f: => U): Unit = (1 to n).foreach(_ ⇒ f)
} 

package com.ebiznext.comet.database.extractor

import better.files.File
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class ExtractScriptGenSpec extends AnyFlatSpec with Matchers {

  val scriptOutputFolder: File = File("/tmp")

  "templatize" should "generate an export script from a TemplateSettings" in {
    val templateParams: TemplateParams = TemplateParams(
      tableToExport = "table1",
      columnsToExport = List("col1", "col2"),
      fullExport = false,
      dsvDelimiter = ",",
      deltaColumn = Some("updateCol"),
      exportOutputFileBase = "output_file",
      scriptOutputFile = scriptOutputFolder / "EXTRACT_table1.sql"
    )

    val templatePayload: String = ScriptGen.templatize(
      File(
        getClass.getResource("/sample/database/EXTRACT_TABLE.sql.mustache").getPath
      ),
      templateParams
    )

    templatePayload shouldBe File(
      getClass.getResource("/sample/database/expected_script_payload.txt").getPath
    ).lines.mkString("\n")
  }

} 

package com.codacy.analysis.cli.analysis

import java.nio.file.Path

import better.files.File
import com.codacy.analysis.core
import com.codacy.analysis.core.model.FileMetrics

import scala.util.{Failure, Success}

object MetricsToolExecutor {

  import com.codacy.analysis.cli.analysis.AnalyseExecutor._

  def reduceMetricsToolResultsByFile(metricsResults: Seq[MetricsToolExecutorResult]): Seq[MetricsToolExecutorResult] = {
    val (successfulMetricsResults, failedMetricsResults) =
      metricsResults.partition(_.analysisResults.isSuccess)

    successfulMetricsResults
      .groupBy(_.language)
      .values
      .flatMap {
        _.foldLeft(Option.empty[MetricsToolExecutorResult]) {
          case (
                Some(metricsExecutorResAcc @ MetricsToolExecutorResult(_, _, Success(fileMetricsAcc))),
                metricsExecutorRes @ MetricsToolExecutorResult(_, _, Success(fileMetrics))) =>
            val allFiles = metricsExecutorResAcc.files ++ metricsExecutorRes.files
            val reducedFileMetrics = reduceFileMetricsByFile(fileMetrics ++ fileMetricsAcc)
            Some(metricsExecutorResAcc.copy(files = allFiles, analysisResults = Success(reducedFileMetrics)))
          case (_, o) => Some(o)
        }
      }(collection.breakOut) ++ failedMetricsResults
  }

  private def reduceFileMetricsByFile(fileMetrics: Set[FileMetrics]): Set[FileMetrics] = {
    fileMetrics
      .groupBy(_.filename)
      .flatMap {
        case (filePath, fMetrics) =>
          fMetrics.reduceOption { (fMetricsAccumulator, fMetricsElement) =>
            FileMetrics(
              filePath,
              fMetricsAccumulator.complexity.orElse(fMetricsElement.complexity),
              fMetricsAccumulator.loc.orElse(fMetricsElement.loc),
              fMetricsAccumulator.cloc.orElse(fMetricsElement.cloc),
              fMetricsAccumulator.nrMethods.orElse(fMetricsElement.nrMethods),
              fMetricsAccumulator.nrClasses.orElse(fMetricsElement.nrClasses),
              if (fMetricsAccumulator.lineComplexities.nonEmpty) {
                fMetricsAccumulator.lineComplexities
              } else {
                fMetricsElement.lineComplexities
              })
          }
      }(collection.breakOut)
  }

  def calculateMissingFileMetrics(
    directory: File,
    metricsResults: Seq[AnalyseExecutor.MetricsToolExecutorResult]): Seq[MetricsToolExecutorResult] = {

    val fileMetricsByFilePath: Map[Path, FileMetrics] = metricsResults.flatMap { result =>
      result.analysisResults.map(_.map(fileMetrics => (fileMetrics.filename, fileMetrics))).getOrElse(Set.empty)
    }(collection.breakOut)

    metricsResults.foldLeft(Seq.empty[MetricsToolExecutorResult]) {
      case (metricsAccumulator, res @ AnalyseExecutor.MetricsToolExecutorResult(_, _, Success(_))) =>
        metricsAccumulator :+ countMissingLoc(directory, fileMetricsByFilePath, res)
      case (metricsAccumulator, res @ AnalyseExecutor.MetricsToolExecutorResult(lang, files, Failure(_)))
          if !metricsResults.exists(r => r.language == lang && r.files == files && r.analysisResults.isSuccess) =>
        metricsAccumulator :+ res :+ countMissingLoc(directory, fileMetricsByFilePath, res)
      case (metricsAccumulator, res) =>
        metricsAccumulator :+ res
    }
  }

  private def countMissingLoc(directory: File,
                              fileMetricsByFilePath: Map[Path, FileMetrics],
                              metricsRes: AnalyseExecutor.MetricsToolExecutorResult): MetricsToolExecutorResult = {
    val fileMetrics = metricsRes.files.map { file =>
      fileMetricsByFilePath.get(file) match {
        case None =>
          FileMetrics(
            filename = file,
            nrClasses = None,
            nrMethods = None,
            loc = countLoc(directory, file),
            cloc = None,
            complexity = None,
            lineComplexities = Set.empty)
        case Some(metrics) if metrics.loc.isEmpty => metrics.copy(loc = countLoc(directory, file))
        case Some(metrics)                        => metrics
      }
    }
    metricsRes.copy(analysisResults = Success(fileMetrics))
  }

  private def countLoc(directory: File, file: Path): Option[Int] = {
    val fileAbsolutePath = (directory / file.toString).path.toAbsolutePath.toString
    core.utils.FileHelper.countLoc(fileAbsolutePath)
  }
} 

package com.codacy.analysis.core.files

import better.files.File
import cats.MonadError

class FallbackFileCollector[T[_]](fileCollectorCompanions: List[FileCollectorCompanion[T]])(implicit
  monadError: MonadError[T, Throwable])
    extends FileCollector[T] {

  private val fileCollectors: List[FileCollector[T]] = fileCollectorCompanions.map(_.apply())

  override def list(directory: File): T[FilesTarget] = {
    list(fileCollectors, directory)
  }

  private def list(fileCollectorList: List[FileCollector[T]], directory: File): T[FilesTarget] = {
    fileCollectorList match {
      case fileCollector :: tail =>
        monadError.recoverWith(fileCollector.list(directory)) {
          case _ =>
            logger.info(s"Failed to list files with ${fileCollector.getClass.getName}")
            list(tail, directory)
        }
      case Nil =>
        val errorMessage =
          s"All FileCollectors failed to list files: ${fileCollectorCompanions.map(_.name).mkString(",")}"
        logger.error(errorMessage)

        monadError.raiseError(new Exception(errorMessage))
    }
  }
}

class FallbackFileCollectorCompanion[T[_]](fileCollectorCompanions: List[FileCollectorCompanion[T]])(implicit
  monadError: MonadError[T, Throwable])
    extends FileCollectorCompanion[T] {

  val name: String = s"fallback:${fileCollectorCompanions.map(_.name).mkString(",")}"

  override def apply(): FileCollector[T] = new FallbackFileCollector(fileCollectorCompanions)

} 

package com.codacy.analysis.core.tools

import java.nio.file.Path

import better.files.File
import com.codacy.analysis.core.model.DuplicationClone
import com.codacy.plugins.api.duplication.DuplicationTool.CodacyConfiguration
import com.codacy.plugins.api.languages.Language
import com.codacy.plugins.api
import com.codacy.plugins.duplication.traits
import com.codacy.plugins.runners.{BinaryDockerRunner, DockerRunner}
import com.codacy.plugins.utils.PluginHelper
import org.log4s.getLogger

import scala.concurrent.duration._
import scala.util.Try

class DuplicationTool(private val duplicationTool: traits.DuplicationTool, val languageToRun: Language) extends ITool {

  override def name: String = "duplication"
  override def supportedLanguages: Set[Language] = duplicationTool.languages.to[Set]

  def run(directory: File,
          files: Set[Path],
          timeout: Option[Duration] = Option.empty[Duration]): Try[Set[DuplicationClone]] = {
    val dockerRunner = new BinaryDockerRunner[api.duplication.DuplicationClone](duplicationTool)
    val runner = new traits.DuplicationRunner(duplicationTool, dockerRunner)

    for {
      duplicationClones <- runner.run(
        directory.toJava,
        CodacyConfiguration(Option(languageToRun), Option.empty),
        timeout.getOrElse(DockerRunner.defaultRunTimeout),
        None)
      clones = filterDuplicationClones(duplicationClones, files)
    } yield {
      clones.map(clone => DuplicationClone(clone.cloneLines, clone.nrTokens, clone.nrLines, clone.files.to[Set]))(
        collection.breakOut): Set[DuplicationClone]
    }
  }

  private def filterDuplicationClones(duplicationClones: List[api.duplication.DuplicationClone],
                                      files: Set[Path],
                                      minCloneLines: Int = 5): List[api.duplication.DuplicationClone] = {
    // The duplication files should be more than 1. If it is one, then it means
    // that the other clone was in an ignored file. This is based on the assumption
    // that the duplication results will contain more than one entry for files
    // with duplicated clones with themselves.
    duplicationClones.collect {
      case clone if clone.nrLines >= minCloneLines =>
        val commitFileNames = files.map(_.toString)
        val filteredFiles = filterUnignoredFiles(clone.files, commitFileNames)
        (clone.copy(files = filteredFiles), filteredFiles.length)
    }.collect { case (clone, nrCloneFiles) if nrCloneFiles > 1 => clone }
  }

  private def filterUnignoredFiles(duplicated: Seq[api.duplication.DuplicationCloneFile],
                                   expectedFiles: Set[String]): Seq[api.duplication.DuplicationCloneFile] = {
    duplicated.collect {
      case cloneFile if expectedFiles.contains(cloneFile.filePath) =>
        cloneFile
    }
  }
}

object DuplicationToolCollector {

  private val logger: org.log4s.Logger = getLogger

  private val availableTools: List[traits.DuplicationTool] = PluginHelper.dockerDuplicationPlugins

  def fromLanguages(languages: Set[Language]): Set[DuplicationTool] = {
    languages.flatMap { lang =>
      val collectedTools = availableTools.collect {
        case tool if tool.languages.contains(lang) =>
          new DuplicationTool(tool, lang)
      }
      if (collectedTools.isEmpty) {
        logger.info(s"No duplication tools found for language ${lang.name}")
      }
      collectedTools
    }
  }

} 

package com.codacy.analysis.core.tools

import java.nio.file.Paths

import better.files.File
import com.codacy.analysis.core.model.FileMetrics
import com.codacy.plugins.api
import com.codacy.plugins.api.Source
import com.codacy.plugins.api.languages.Language
import com.codacy.plugins.api.metrics.MetricsTool.CodacyConfiguration
import com.codacy.plugins.metrics.traits
import com.codacy.plugins.metrics.traits.{MetricsRequest, MetricsRunner}
import com.codacy.plugins.runners.{BinaryDockerRunner, DockerRunner}
import com.codacy.plugins.utils.PluginHelper
import org.log4s.getLogger

import scala.concurrent.duration.Duration
import scala.util.Try

class MetricsTool(private val metricsTool: traits.MetricsTool, val languageToRun: Language) extends ITool {
  override def name: String = "metrics"

  override def supportedLanguages: Set[Language] = metricsTool.languages.to[Set]

  def run(directory: File,
          files: Option[Set[Source.File]],
          timeout: Option[Duration] = Option.empty[Duration]): Try[List[FileMetrics]] = {
    val request = MetricsRequest(directory.pathAsString)

    val dockerRunner = new BinaryDockerRunner[api.metrics.FileMetrics](metricsTool)
    val runner = new MetricsRunner(metricsTool, dockerRunner)

    val configuration = CodacyConfiguration(files, Some(languageToRun), None)

    val toolFileMetrics =
      runner.run(request, configuration, timeout.getOrElse(DockerRunner.defaultRunTimeout), None)

    toolFileMetrics.map {
      _.collect {
        case fileMetrics if unignoredFile(fileMetrics, files) =>
          FileMetrics(
            filename = Paths.get(fileMetrics.filename),
            complexity = fileMetrics.complexity,
            loc = fileMetrics.loc,
            cloc = fileMetrics.cloc,
            nrMethods = fileMetrics.nrMethods,
            nrClasses = fileMetrics.nrClasses,
            lineComplexities = fileMetrics.lineComplexities)
      }
    }
  }

  def unignoredFile(metrics: api.metrics.FileMetrics, files: Option[Set[Source.File]]): Boolean = {
    files.forall(_.exists(_.path == metrics.filename))
  }
}

object MetricsToolCollector {

  private val logger: org.log4s.Logger = getLogger

  private val availableTools = PluginHelper.dockerMetricsPlugins

  def fromLanguages(languages: Set[Language]): Set[MetricsTool] = {
    languages.flatMap { lang =>
      val collectedTools = availableTools.collect {
        case tool if tool.languages.contains(lang) =>
          new MetricsTool(tool, lang)
      }
      if (collectedTools.isEmpty) {
        logger.info(s"No metrics tools found for language ${lang.name}")
      }
      collectedTools
    }
  }

} 

package com.codacy.analysis.core.analysis

import java.nio.file.Path

import better.files.File
import com.codacy.analysis.core.model.{Configuration, DuplicationClone, FileMetrics, ToolResult}
import com.codacy.analysis.core.tools.{DuplicationTool, MetricsTool, Tool}
import org.log4s.{Logger, getLogger}

import scala.concurrent.duration.Duration
import scala.util.Try

trait AnalyserCompanion[T[_]] {
  def name: String
  def apply(): Analyser[T]
}

trait Analyser[T[_]] {

  def analyse(tool: Tool,
              directory: File,
              files: Set[Path],
              config: Configuration,
              timeout: Option[Duration] = Option.empty[Duration]): T[Set[ToolResult]]

  def metrics(metricsTool: MetricsTool,
              directory: File,
              files: Option[Set[Path]],
              timeout: Option[Duration] = Option.empty[Duration]): T[Set[FileMetrics]]

  def duplication(duplicationTool: DuplicationTool,
                  directory: File,
                  files: Set[Path],
                  timeout: Option[Duration] = Option.empty[Duration]): T[Set[DuplicationClone]]

}

object Analyser {

  private val logger: Logger = getLogger

  val defaultAnalyser: AnalyserCompanion[Try] = CodacyPluginsAnalyser

  val allAnalysers: Set[AnalyserCompanion[Try]] = Set(defaultAnalyser)

  def apply(name: String): Analyser[Try] = {
    val builder = allAnalysers.find(_.name.equalsIgnoreCase(name)).getOrElse {
      logger.warn(s"Could not find analyser for name $name. Using ${defaultAnalyser.name} as fallback.")
      defaultAnalyser
    }

    builder()
  }

  sealed trait Error {
    val message: String
  }

  object Error {

    final case class ToolExecutionFailure(toolType: String, toolName: String) extends Error {
      override val message: String = s"Failed $toolType for $toolName"
    }

    final case class ToolNeedsNetwork(toolName: String) extends Error {
      override val message: String = s"The tool $toolName needs network access to execute."
    }

    final case class NonExistingToolInput(toolName: String, availableTools: Set[String]) extends Error {

      override val message: String =
        s"""The selected tool "$toolName" is not supported or does not exist.
                                        |The tool should be one of (${availableTools.mkString(", ")})""".stripMargin
    }

    case object NoActiveToolInConfiguration extends Error {
      override val message: String = "No active tool found on the remote configuration"
    }

    case object NoToolsFoundForFiles extends Error {
      override val message: String = "No tools found for files provided"
    }

  }
} 

package com.codacy.analysis.core.analysis

import java.nio.file.Path

import better.files.File
import com.codacy.analysis.core.model._
import com.codacy.analysis.core.tools.{DuplicationTool, MetricsTool, Tool}
import com.codacy.plugins.api.Source
import org.log4s.{Logger, getLogger}

import scala.concurrent.duration._
import scala.util.{Failure, Success, Try}

class CodacyPluginsAnalyser extends Analyser[Try] {

  private val logger: Logger = getLogger

  override def analyse(tool: Tool,
                       directory: File,
                       files: Set[Path],
                       config: Configuration,
                       timeout: Option[Duration] = Option.empty[Duration]): Try[Set[ToolResult]] = {
    val result = tool.run(directory, files, config, timeout)

    result match {
      case Success(res) =>
        logger.info(s"Completed analysis for ${tool.name} with ${res.size} results")
      case Failure(e) =>
        logger.error(e)(Analyser.Error.ToolExecutionFailure("analysis", tool.name).message)
    }

    result
  }

  override def metrics(metricsTool: MetricsTool,
                       directory: File,
                       files: Option[Set[Path]],
                       timeout: Option[Duration] = Option.empty[Duration]): Try[Set[FileMetrics]] = {

    val srcFiles = files.map(_.map(filePath => Source.File(filePath.toString)))

    val result = metricsTool.run(directory, srcFiles, timeout)

    result match {
      case Success(res) =>
        logger.info(s"Completed metrics for ${metricsTool.name} with ${res.size} results")
      case Failure(e) =>
        logger.error(e)(Analyser.Error.ToolExecutionFailure("metrics", metricsTool.name).message)
    }

    result.map(_.to[Set])
  }

  override def duplication(duplicationTool: DuplicationTool,
                           directory: File,
                           files: Set[Path],
                           timeout: Option[Duration] = Option.empty[Duration]): Try[Set[DuplicationClone]] = {

    val result = duplicationTool.run(directory, files, timeout)

    result match {
      case Success(res) =>
        logger.info(s"Completed duplication for ${duplicationTool.name} with ${res.size} results")
      case Failure(e) =>
        logger.error(e)(Analyser.Error.ToolExecutionFailure("duplication", duplicationTool.name).message)
    }

    result.map(_.to[Set])
  }

}

object CodacyPluginsAnalyser extends AnalyserCompanion[Try] {

  val name: String = "codacy-plugins"

  override def apply(): Analyser[Try] = new CodacyPluginsAnalyser()

  object errors {

    def missingTool(tool: String): Analyser.Error =
      Analyser.Error.NonExistingToolInput(tool, Tool.allToolShortNames)
  }

} 

package com.codacy.analysis.core.configuration

import better.files.File
import cats.syntax.show._
import com.codacy.analysis.core.files.Glob
import com.codacy.plugins.api.languages.{Language, Languages}
import io.circe.generic.auto._
import io.circe.yaml.parser
import io.circe.{Decoder, Json, _}
import play.api.libs.json.JsValue

import scala.util.{Properties, Try}

final case class LanguageConfiguration(extensions: Option[Set[String]])

final case class EngineConfiguration(excludePaths: Option[Set[Glob]],
                                     baseSubDir: Option[String],
                                     extraValues: Option[Map[String, JsValue]])

final case class CodacyConfigurationFile(engines: Option[Map[String, EngineConfiguration]],
                                         excludePaths: Option[Set[Glob]],
                                         languages: Option[Map[Language, LanguageConfiguration]]) {

  lazy val languageCustomExtensions: Map[Language, Set[String]] =
    languages.fold(Map.empty[Language, Set[String]])(_.map {
      case (lang, config) => (lang, config.extensions.getOrElse(Set.empty[String]))
    })
}

class CodacyConfigurationFileLoader {

  val filenames: Set[String] = Set(".codacy.yaml", ".codacy.yml")

  def load(directory: File): Either[String, CodacyConfigurationFile] = {
    search(directory).flatMap(configDir => parse(configDir.contentAsString))
  }

  def search(root: File): Either[String, File] = {
    filenames
      .map(root / _)
      .find(f => f.exists && f.isRegularFile)
      .fold[Either[String, File]](
        Left(s"Could not find Codacy configuration file. Make sure you have a file named like one of ${filenames
          .mkString(", ")}."))(Right(_))
  }

  def parse(yamlString: String): Either[String, CodacyConfigurationFile] = {
    for {
      json <- parser.parse(yamlString).left.map(_.show)
      cursor = HCursor.fromJson(json)
      configurationEither = Decoder[CodacyConfigurationFile].decodeAccumulating(cursor).toEither
      configuration <- configurationEither.left.map(_.toList.map(_.show).mkString(Properties.lineSeparator))
    } yield configuration
  }

}

object CodacyConfigurationFile {

  implicit val globDecoder: Decoder[Glob] = (c: HCursor) => c.as[String].map(Glob)

  implicit val languageKeyDecoder: KeyDecoder[Language] = (languageStr: String) => Languages.fromName(languageStr)

  implicit val decodeEngineConfiguration: Decoder[EngineConfiguration] =
    new Decoder[EngineConfiguration] {
      val engineConfigurationKeys = Set("enabled", "exclude_paths", "base_sub_dir")

      def apply(c: HCursor): Decoder.Result[EngineConfiguration] = {
        val extraKeys =
          c.keys.fold(List.empty[String])(_.to[List]).filter(key => !engineConfigurationKeys.contains(key))
        for {
          excludePaths <- c.downField("exclude_paths").as[Option[Set[Glob]]]
          baseSubDir <- c.downField("base_sub_dir").as[Option[String]]
        } yield {
          val extraToolConfigurations: Map[String, JsValue] = extraKeys.flatMap { extraKey =>
            c.downField(extraKey)
              .as[Json]
              .fold[Option[JsValue]](
                { _ =>
                  Option.empty
                },
                { json =>
                  Try(play.api.libs.json.Json.parse(json.noSpaces)).toOption
                })
              .map(value => (extraKey, value))
          }(collection.breakOut)

          EngineConfiguration(excludePaths, baseSubDir, Option(extraToolConfigurations).filter(_.nonEmpty))
        }
      }
    }

  implicit val decodeCodacyConfigurationFile: Decoder[CodacyConfigurationFile] =
    Decoder.forProduct3("engines", "exclude_paths", "languages")(CodacyConfigurationFile.apply)

} 

package com.codacy.analysis.core.utils

import java.nio.file.Path

import better.files.File
import org.log4s
import org.log4s.getLogger

import scala.util.Try

object FileHelper {

  private val logger: log4s.Logger = getLogger

  def relativePath(filename: String): Path = File.currentWorkingDirectory.relativize(File(filename))

  def countLoc(filename: String): Option[Int] = {
    Try(File(filename).lineIterator).fold(
      { t =>
        logger.error(t)(s"Failed to read file $filename")
        Option.empty[Int]
      },
      { lines =>
        Some(lines.foldLeft(0) {
          case (counter, line) if line.trim.length >= 3 => counter + 1
          case (counter, _)                             => counter
        })
      })
  }
} 

package com.codacy.analysis.core.git

import better.files.File
import org.eclipse.jgit.internal.storage.file.FileRepository
import org.eclipse.jgit.storage.file.FileRepositoryBuilder
import org.log4s.{Logger, getLogger}

import scala.util.Try

object Git {

  private val logger: Logger = getLogger

  def repository(directory: File): Try[Repository] = {
    Try((directory / ".git").toJava).filter(d => new FileRepository(d.getPath).getObjectDatabase.exists()).flatMap {
      gitDir =>
        Try {
          val builder = new FileRepositoryBuilder

          val repository = builder.setGitDir(gitDir).readEnvironment.findGitDir.build

          new Repository(repository)
        }
    }
  }

  def currentCommitUuid(directory: File): Option[Commit.Uuid] = {
    Git
      .repository(directory)
      .flatMap(_.latestCommit)
      .fold(
        { e =>
          logger.warn(e)(e.getMessage)
          None
        },
        commit => Some(commit.commitUuid))
  }

} 

package com.codacy.analysis.cli.command

import better.files.{File, Resource}
import com.codacy.analysis.cli.analysis.ExitStatus.ExitCodes
import org.specs2.control.NoLanguageFeatures
import org.specs2.matcher.FileMatchers
import org.specs2.mutable.Specification

class ValidateConfigurationCommandSpec extends Specification with NoLanguageFeatures with FileMatchers {

  "ValidateConfigurationExecutor" should {
    "find configuration file" in {
      File
        .temporaryDirectory()
        .map { directory =>
          val resource = Resource.getAsString("com/codacy/analysis/core/configuration/codacy.yaml")
          (directory / ".codacy.yaml").write(resource)

          val command =
            ValidateConfigurationCommand(ValidateConfiguration(CommonOptions(), Option(directory)))

          command.run() mustEqual ExitCodes.success
        }
        .get
    }

    "fail" in {
      File
        .temporaryDirectory()
        .map { directory =>
          val command =
            ValidateConfigurationCommand(ValidateConfiguration(CommonOptions(), Option(directory)))

          command.run() mustEqual ExitCodes.invalidConfigurationFile
        }
        .get
    }
  }

} 

package com.codacy.analysis.cli.command

import better.files.File
import com.codacy.analysis.cli.analysis.ExitStatus
import com.codacy.analysis.cli.analysis.ExitStatus.ExitCodes
import com.codacy.analysis.core.configuration.CodacyConfigurationFileLoader

object ValidateConfigurationCommand {

  def apply(validateConfiguration: ValidateConfiguration): ValidateConfigurationCommand = {
    new ValidateConfigurationCommand(validateConfiguration, new CodacyConfigurationFileLoader())
  }
}

class ValidateConfigurationCommand(validateConfiguration: ValidateConfiguration,
                                   configurationLoader: CodacyConfigurationFileLoader) {

  def run(): ExitStatus.ExitCode = {
    val directory = validateConfiguration.directory.getOrElse(File.currentWorkingDirectory)

    (for {
      file <- configurationLoader.search(directory)
      cfgFile <- configurationLoader.parse(file.contentAsString)
    } yield (file, cfgFile)) match {
      case Left(e) =>
        Console.err.println(e)
        ExitCodes.invalidConfigurationFile

      case Right((file, cfgFile)) =>
        Console.out.println(s"Successfully loaded the Codacy configuration file in ${file.pathAsString}")
        pprint.pprintln(cfgFile)
        ExitCodes.success
    }

  }

} 

package com.codacy.analysis.core.git

import better.files.File
import org.specs2.control.NoLanguageFeatures
import org.specs2.mutable.Specification
import com.codacy.analysis.core.utils.TestUtils._

import scala.sys.process.Process

class GitSpec extends Specification with NoLanguageFeatures {

  "Git" should {
    "create a repository" in {
      (for {
        temporaryDirectory <- File.temporaryDirectory()
      } yield {
        Process(Seq("git", "init"), temporaryDirectory.toJava).!

        Git.repository(temporaryDirectory) must beSuccessfulTry
      }).get
    }

    "get the current commit uuid" in {
      withTemporaryGitRepo(directory => {
        val expectedUuid = Process(Seq("git", "rev-parse", "HEAD"), directory.toJava).!!.trim
        Git.currentCommitUuid(directory) must beLike {
          case Some(commit) => commit.value must beEqualTo(expectedUuid)
        }
      })
    }

    "fail to create a repository" in {
      (for {
        temporaryDirectory <- File.temporaryDirectory()
      } yield {
        Git.repository(temporaryDirectory) must beFailedTry
      }).get
    }
  }

} 

package com.codacy.analysis.cli.configuration

import java.net.URL

import better.files.File
import com.codacy.analysis.core.utils.Implicits._
import org.log4s.{Logger, getLogger}

import scala.util.{Failure, Try}

class Environment(variables: Map[String, String]) {

  private val logger: Logger = getLogger

  def codeDirectoryEnvironmentVariable(): Option[String] = {
    validate("Project directory", "environment variable", "CODACY_CODE")(variables.get("CODACY_CODE"))
  }

  def projectTokenArgument(projectTokenFromArguments: Option[String]): Option[String] = {
    validate("Project token", "argument", "--project-token")(projectTokenFromArguments)
  }

  def projectTokenEnvironmentVariable(): Option[String] = {
    validate("Project token", "environment variable", "CODACY_PROJECT_TOKEN")(variables.get("CODACY_PROJECT_TOKEN"))
  }

  def apiTokenArgument(apiTokenFromArguments: Option[String]): Option[String] = {
    validate("API token", "argument", "--api-token")(apiTokenFromArguments)
  }

  def apiTokenEnvironmentVariable(): Option[String] = {
    validate("API token", "environment variable", "CODACY_API_TOKEN")(variables.get("CODACY_API_TOKEN"))
  }

  def apiBaseUrlArgument(codacyApiBaseURLFromArguments: Option[String]): Option[String] = {
    val apiURL =
      validate("API base URL", "argument", "--codacy-api-base-url")(codacyApiBaseURLFromArguments)
    validateApiBaseUrl(apiURL)
  }

  def apiBaseUrlEnvironmentVariable(): Option[String] = {
    val apiURL =
      validate("API base URL", "environment variable", "CODACY_API_BASE_URL")(variables.get("CODACY_API_BASE_URL"))
    validateApiBaseUrl(apiURL)
  }

  def baseProjectDirectory(directory: Option[File]): File =
    directory.fold(codeDirectoryEnvironmentVariable().map(File(_)).getOrElse(File.currentWorkingDirectory))(dir =>
      if (dir.isDirectory) dir else dir.parent)

  private def validateApiBaseUrl(apiURL: Option[String]): Option[String] = {
    apiURL.flatMap { url =>
      Try(new URL(url)) match {
        case Failure(_) =>
          val error = s"Invalid API base URL: $url"

          val help = if (!url.startsWith("http")) {
            " * Maybe you forgot the http:// or https:// ?"
          }

          logger.warn(s"$error\n$help")
          Option.empty[String]

        case _ =>
          logger.info(s"Using API base URL $url")
          Option(url)
      }
    }
  }

  private def validate(name: String, paramType: String, param: String)(value: Option[String]): Option[String] = {
    value.ifEmpty(logger.info(s"$name not passed through $paramType `$param`")).flatMap {
      case t if t.trim.nonEmpty =>
        logger.info(s"$name found in $paramType `$param`")
        Option(t.trim)
      case _ =>
        logger.warn(s"$name passed through $paramType `$param` is empty")
        Option.empty[String]
    }
  }
} 

package com.orendainx.trucking.simulator.simulators

import akka.actor.{Actor, ActorSystem, Inbox, Props}
import better.files.File
import com.orendainx.trucking.commons.models._
import com.orendainx.trucking.enrichment.WeatherAPI
import com.orendainx.trucking.simulator.coordinators.AutomaticCoordinator
import com.orendainx.trucking.simulator.depots.NoSharingDepot
import com.orendainx.trucking.simulator.flows.SharedFlowManager
import com.orendainx.trucking.simulator.generators.TruckAndTrafficGenerator
import com.orendainx.trucking.simulator.services.DriverFactory
import com.orendainx.trucking.simulator.transmitters.{ActorTransmitter, DataTransmitter, KafkaTransmitter}
import com.typesafe.config.{Config, ConfigFactory}

import scala.concurrent.Await
import scala.concurrent.duration._


  private class EnrichmentActor extends Actor {
    def receive = {
      case td: TruckData =>
        kafkaTruckTransmitter ! DataTransmitter.Transmit(
          EnrichedTruckData(td,
            WeatherAPI.default.getFog(td.eventType),
            WeatherAPI.default.getRain(td.eventType),
            WeatherAPI.default.getWind(td.eventType))
        )
      case td: TrafficData =>
        kafkaTrafficTransmitter ! DataTransmitter.Transmit(td)
    }
  }
} 

package com.wix.bazel.migrator

import java.nio.file.{Files, Path}

import better.files.File
import com.github.marschall.memoryfilesystem.MemoryFileSystemBuilder
import com.wix.bazel.migrator.overrides.{InternalTargetOverride, InternalTargetsOverrides}
import org.specs2.matcher.{Matcher, Scope}
import org.specs2.mutable.SpecificationWithJUnit

class DockerImagesWriterTest extends SpecificationWithJUnit {
  abstract class ctx extends Scope{

    def containExactlyOnce(substr: String): Matcher[String] = {
      {a:String => a.indexOf(substr) must not be_== -1} and {a:String => a.indexOf(substr) must beEqualTo(a.lastIndexOf(substr))}
    }

    val rootfs: Path = MemoryFileSystemBuilder.newLinux().build().getPath("repo-root")
    val overrideWithDockerImages = InternalTargetOverride("some-label", dockerImagesDeps = Option(List("mysql:5.7", "docker-repo.wixpress.com/com.wixpress.whatever/whatever:1.234.5")))
    def overrides: Set[InternalTargetOverride]
    def writer = new DockerImagesWriter(rootfs, InternalTargetsOverrides(overrides))
  }

  "DockerImagesWriter" should {
    "create docker_images.bzl in third_party/docker_images" in new ctx {
      def overrides: Set[InternalTargetOverride] = Set.empty
      writer.write()
      Files.exists(rootfs.resolve("third_party/docker_images/docker_images.bzl")) should beTrue
    }

    "create BUILD.bazel file in third_party/docker_images" in new ctx {
      def overrides: Set[InternalTargetOverride] = Set.empty
      writer.write()
      Files.exists(rootfs.resolve("third_party/docker_images/BUILD.bazel")) should beTrue
    }

    "fill default values in container_pull for short-form image" in new ctx {
      def overrides: Set[InternalTargetOverride] = Set(overrideWithDockerImages)
      writer.write()
      val expected: String =
        s"""|  container_pull(
            |    name = "mysql_5.7",
            |    registry = "index.docker.io",
            |    repository = "library/mysql",
            |    tag = "5.7"
            |  )""".stripMargin
      File(rootfs.resolve("third_party/docker_images/docker_images.bzl")).contentAsString must contain(expected)
    }

    "write values as-is in container_pull for full form image" in new ctx {
      def overrides: Set[InternalTargetOverride] = Set(overrideWithDockerImages)
      writer.write()
      val expected: String =
        s"""|  container_pull(
            |    name = "com.wixpress.whatever_whatever_1.234.5",
            |    registry = "docker-repo.wixpress.com",
            |    repository = "com.wixpress.whatever/whatever",
            |    tag = "1.234.5"
            |  )""".stripMargin
      File(rootfs.resolve("third_party/docker_images/docker_images.bzl")).contentAsString must contain(expected)
    }

    "write container_image in BUILD file" in new ctx {
      def overrides: Set[InternalTargetOverride] = Set(overrideWithDockerImages)
      writer.write()
      val expected: String =
        s"""container_image(name="com.wixpress.whatever_whatever_1.234.5", base="@com.wixpress.whatever_whatever_1.234.5//image")""".stripMargin

      File(rootfs.resolve("third_party/docker_images/BUILD.bazel")).contentAsString must contain(expected)
    }

    "deduplicate images in BUILD file" in new ctx {
      def overrides = Set(overrideWithDockerImages, overrideWithDockerImages.copy(label = "duplicate"))
      writer.write()
      private val fileContent: String = File(rootfs.resolve("third_party/docker_images/BUILD.bazel")).contentAsString
      fileContent must containExactlyOnce("container_image(name=\"mysql_5.7\",")
    }

    "deduplicate images in docker_images.bzl file" in new ctx {

      def overrides = Set(overrideWithDockerImages, overrideWithDockerImages.copy(label = "duplicate"))
      writer.write()
      private val fileContent: String = File(rootfs.resolve("third_party/docker_images/docker_images.bzl")).contentAsString
      fileContent must containExactlyOnce("name = \"mysql_5.7\",")
    }
  }
} 

package com.wixpress.build.git

import better.files.File
import org.eclipse.jgit.api.Git

case class GitRepository(path: File, git: Git) {
  def pathAsString: String = path.pathAsString
}

object GitRepository {
  def newRemote: GitRepository = {
    val remoteRepoDir = newDisposableDir("remote-dir")
    GitRepository(remoteRepoDir, Git.init()
      .setDirectory(remoteRepoDir.toJava)
      .setBare(true)
      .call())
  }

  def newLocal: GitRepository = {
    val localRepoDir = newDisposableDir("local-dir")
    GitRepository(localRepoDir, Git.init()
      .setDirectory(localRepoDir.toJava)
      .call())
  }

  def newLocalCloneOf(remoteRepo: GitRepository): GitRepository = {
    val localCloneDir = newDisposableDir("clone")
    GitRepository(localCloneDir, Git.cloneRepository()
      .setURI(remoteRepo.path.pathAsString)
      .setDirectory(localCloneDir.path.toFile)
      .call())
  }

  private def newDisposableDir(prefix: String): File = {
    val tmpDir = File.newTemporaryDirectory(prefix)
    tmpDir.toJava.deleteOnExit()
    tmpDir
  }
} 

package com.wixpress.build.bazel

import scala.collection.mutable
import ThirdPartyPaths._
import better.files.File

class FakeLocalBazelWorkspace(sourceFiles: mutable.Map[String, String] = mutable.Map.empty, val localWorkspaceName: String = "") extends BazelLocalWorkspace {

  // since FakeLocalBazelWorkspace is already stateful - I allowed another state.
  // on next revision of SynchronizerAcceptanceTest - we will introduce stateless FakeWorkspace
  private var overrides = ThirdPartyOverrides.empty

  def setThirdPartyOverrides(overrides: ThirdPartyOverrides): Unit = {
    this.overrides = overrides
  }

  override def thirdPartyReposFileContent(): String =
    sourceFiles.getOrElse(thirdPartyReposFilePath, "")

  override def overwriteThirdPartyReposFile(skylarkFileContent: String): Unit =
    sourceFiles.put(thirdPartyReposFilePath, skylarkFileContent)

  override def overwriteThirdPartyImportTargetsFile(thirdPartyGroup: String, thirdPartyReposContent: String): Unit = {
    val fileKey = s"$thirdPartyImportFilesPathRoot/$thirdPartyGroup.bzl"
    thirdPartyReposContent match {
      case "" => sourceFiles.remove(fileKey)
      case _ => sourceFiles.put(fileKey, thirdPartyReposContent)
    }
  }

  override def buildFileContent(packageName: String): Option[String] =
    sourceFiles.get(packageName + "/BUILD.bazel")

  override def thirdPartyImportTargetsFileContent(thirdPartyGroup: String): Option[String] =
    sourceFiles.get(s"$thirdPartyImportFilesPathRoot/$thirdPartyGroup.bzl")

  override def allThirdPartyImportTargetsFilesContent(): Set[String] =
    sourceFiles.filter(f => f._1.contains(thirdPartyImportFilesPathRoot + "/")).values.toSet

  override def allThirdPartyImportTargetsFiles(): Map[File, String] =
    sourceFiles.filter(f => f._1.contains(thirdPartyImportFilesPathRoot + "/")).map(pair => (File(pair._1), pair._2)).toMap

  override def overwriteBuildFile(packageName: String, content: String): Unit =
    sourceFiles.put(packageName + "/BUILD.bazel", content)

  override def thirdPartyOverrides(): ThirdPartyOverrides = overrides
}

object FakeLocalBazelWorkspace {
  val thirdPartyReposFilePath: String =  "third_party.bzl"
  val thirdPartyImportFilesPathRoot: String = "third_party"
} 

package com.wixpress.build.bazel

import java.io.FileNotFoundException

import better.files.File
import ThirdPartyPaths._

class FileSystemBazelLocalWorkspace(root: File) extends BazelLocalWorkspace {

  val localWorkspaceName: String = {
    val workspaceFileContent = contentIfExistsOf(root / "WORKSPACE")
    val validWorkspaceWith = """(?s).*workspace\s*\(\s*name\s*=\s*"([^"]+)"\s*\).*""".r

    workspaceFileContent match {
      case Some(validWorkspaceWith(name)) => name
      case _ =>   ""
    }
  }

  private val ThirdPartyOverridesPath = "bazel_migration/third_party_targets.overrides"

  validate()

  override def overwriteBuildFile(packageName: String, content: String): Unit = {
    val buildFilePath = root / packageName / "BUILD.bazel"
    buildFilePath.createIfNotExists(createParents = true)
    buildFilePath.overwrite(content)
  }

  override def overwriteThirdPartyReposFile(thirdPartyReposContent: String): Unit =
    (root / thirdPartyReposFilePath).overwrite(thirdPartyReposContent)

  override def overwriteThirdPartyImportTargetsFile(thirdPartyGroup: String, content: String): Unit = {
    val targetsFile = root / s"$thirdPartyImportFilesPathRoot/$thirdPartyGroup.bzl"
    content match {
      case "" => if (targetsFile.exists) targetsFile.delete()
      case _ => {
        targetsFile.createIfNotExists(createParents = true)
        targetsFile.overwrite(content)
      }
    }
  }

  override def thirdPartyReposFileContent(): String = contentIfExistsOf(root / thirdPartyReposFilePath).getOrElse("")

  override def buildFileContent(packageName: String): Option[String] = contentIfExistsOf(root / packageName / "BUILD.bazel")

  override def thirdPartyImportTargetsFileContent(thirdPartyGroup: String): Option[String] = contentIfExistsOf(root / thirdPartyImportFilesPathRoot / s"$thirdPartyGroup.bzl")

  override def allThirdPartyImportTargetsFilesContent(): Set[String] = {
    allThirdPartyImportTargetsFiles().values.toSet
  }

  override def allThirdPartyImportTargetsFiles(): Map[File, String] = {
    val thirdPartyLocation = root / thirdPartyImportFilesPathRoot
    thirdPartyLocation.createIfNotExists(asDirectory = true, createParents = true)
    val files = thirdPartyLocation.glob("**/*.bzl")
    val withNoCustomVersions = files.filterNot(f => f.path.startsWith(thirdPartyLocation + "/custom/"))
    withNoCustomVersions.map(f => f -> contentIfExistsOf(f).get) toMap
  }

  override def thirdPartyOverrides(): ThirdPartyOverrides = {
    contentIfExistsOf(root / ThirdPartyOverridesPath)
      .map(ThirdPartyOverridesReader.from)
      .getOrElse(ThirdPartyOverrides.empty)
  }

  private def contentIfExistsOf(filePath: File) =
    if (filePath.exists) Some(filePath.contentAsString) else None


  private def validate(): Unit = {
    if (!root.exists)
      throw new FileNotFoundException(root.pathAsString)
  }

} 

package it.teamdigitale.miniclusters

import better.files.File
import org.apache.logging.log4j.LogManager
import org.apache.hadoop.hdfs.HdfsConfiguration
import org.apache.hadoop.test.PathUtils
import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.hdfs.{HdfsConfiguration, MiniDFSCluster}
import better.files._

import scala.util.{Failure, Try}

class HDFSMiniCluster extends AutoCloseable {
  val alogger = LogManager.getLogger(this.getClass)

  var hdfsCluster: Try[MiniDFSCluster] = Failure[MiniDFSCluster](new Exception)
  var fileSystem: Try[FileSystem] = Failure[FileSystem](new Exception)

  val (testDataPath, confPath) = {
    val testDataPath = s"${PathUtils.getTestDir(classOf[HDFSMiniCluster]).getCanonicalPath}/MiniCluster"
    val confPath = s"$testDataPath/conf"
    (
      testDataPath.toFile.createIfNotExists(asDirectory = true, createParents = false),
      confPath.toFile.createIfNotExists(asDirectory = true, createParents = false)
    )
  }


  def start(): Unit = {

    alogger.info("Starting HDFS mini cluster")
    val conf = new HdfsConfiguration()
    conf.setBoolean("dfs.permissions", true)
    System.clearProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA)

    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, testDataPath.pathAsString)
    //FileUtil.fullyDelete(testDataPath.toJava)

    conf.set(s"hadoop.proxyuser.${System.getProperties.get("user.name")}.groups", "*")
    conf.set(s"hadoop.proxyuser.${System.getProperties.get("user.name")}.hosts", "*")

    val builder = new MiniDFSCluster.Builder(conf)
    hdfsCluster = Try(builder.build())
    fileSystem = hdfsCluster.map(_.getFileSystem)
    fileSystem.foreach(fs => {
      val confFile: File = confPath / "hdfs-site.xml"
      for {os <- confFile.newOutputStream.autoClosed} fs.getConf.writeXml(os)
    })

  }


  override def close() = {
    alogger.info("Stopping HDFS mini cluster")
    hdfsCluster.foreach(_.shutdown(true))
    val _ = testDataPath.parent.parent.delete(true)
  }
} 

package daf.util

import better.files.{ File, _ }
import daf.util.DataFrameClasses.{ Address, Person }
import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.hdfs.{ HdfsConfiguration, MiniDFSCluster }
import org.apache.hadoop.test.PathUtils
import org.apache.spark.sql.{ SaveMode, SparkSession }
import org.scalatest.{ BeforeAndAfterAll, FlatSpec, Matchers }
import org.slf4j.LoggerFactory

import scala.util.{ Failure, Random, Try }

abstract class HDFSBase extends FlatSpec with Matchers with BeforeAndAfterAll {

  var miniCluster: Try[MiniDFSCluster] = Failure[MiniDFSCluster](new Exception)

  var fileSystem: Try[FileSystem] = Failure[FileSystem](new Exception)

  val sparkSession: SparkSession = SparkSession.builder().master("local").getOrCreate()

  val alogger = LoggerFactory.getLogger(this.getClass)

  val (testDataPath, confPath) = {
    val testDataPath = s"${PathUtils.getTestDir(this.getClass).getCanonicalPath}/MiniCluster"
    val confPath = s"$testDataPath/conf"
    (
      testDataPath.toFile.createIfNotExists(asDirectory = true, createParents = false),
      confPath.toFile.createIfNotExists(asDirectory = true, createParents = false)
    )
  }

  def pathAvro = "opendata/test.avro"
  def pathParquet = "opendata/test.parquet"
  def pathCsv = "opendata/test.csv"

  def getSparkSession = sparkSession

  override def beforeAll(): Unit = {

    val conf = new HdfsConfiguration()
    conf.setBoolean("dfs.permissions", true)
    System.clearProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA)

    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, testDataPath.pathAsString)
    //FileUtil.fullyDelete(testDataPath.toJava)

    conf.set(s"hadoop.proxyuser.${System.getProperties.get("user.name")}.groups", "*")
    conf.set(s"hadoop.proxyuser.${System.getProperties.get("user.name")}.hosts", "*")

    val builder = new MiniDFSCluster.Builder(conf)
    miniCluster = Try(builder.build())
    fileSystem = miniCluster.map(_.getFileSystem)
    fileSystem.foreach(fs => {
      val confFile: File = confPath / "hdfs-site.xml"
      for { os <- confFile.newOutputStream.autoClosed } fs.getConf.writeXml(os)
    })

    writeDf()
  }

  override def afterAll(): Unit = {
    miniCluster.foreach(_.shutdown(true))
    val _ = testDataPath.parent.parent.delete(true)
    sparkSession.stop()
  }

  
  private def writeDf(): Unit = {
    import sparkSession.implicits._

    alogger.info(s"TestDataPath ${testDataPath.toJava.getAbsolutePath}")
    alogger.info(s"ConfPath ${confPath.toJava.getAbsolutePath}")
    val persons = (1 to 10).map(i => Person(s"Andy$i", Random.nextInt(85), Address("Via Ciccio Cappuccio")))
    val caseClassDS = persons.toDS()
    caseClassDS.write.format("parquet").mode(SaveMode.Overwrite).save(pathParquet)
    caseClassDS.write.format("com.databricks.spark.avro").mode(SaveMode.Overwrite).save(pathAvro)
    //writing directly the Person dataframe generates an exception
    caseClassDS.toDF.select("name", "age").write.format("csv").mode(SaveMode.Overwrite).option("header", "true").save(pathCsv)
  }
}

object DataFrameClasses {

  final case class Address(street: String)

  final case class Person(name: String, age: Int, address: Address)
} 

package jbok.core.config

import java.net.InetSocketAddress
import java.nio.file.{Path, Paths}

import better.files.File
import cats.effect.IO
import cats.implicits._
import io.circe.syntax._
import jbok.common.config.Config
import jbok.core.keystore.KeyStorePlatform
import jbok.core.models.Address
import jbok.core.peer.PeerUri
import jbok.crypto.signature.KeyPair
import monocle.macros.syntax.lens._

import sys.process.{ProcessLogger, stringSeqToProcess}
import scala.concurrent.duration._

final case class NetworkBuilder(
    base: FullConfig,
    configs: List[FullConfig] = Nil,
) {
  val home = System.getProperty("user.home")
  val root = Paths.get(home).resolve(".jbok")

  def withBlockPeriod(n: Int): NetworkBuilder =
    copy(base = base.lens(_.mining.period).set(n.millis))

  def createCert(ip: String, cn: String, caDir: Path, certDir: Path): IO[String] = IO {
    val path = File(".")
    val projectDir = path.path.toAbsolutePath
    val processLogger = new ProcessLogger {
      override def out(s: => String): Unit = println(s)
      override def err(s: => String): Unit = println(s)
      override def buffer[T](f: => T): T = f
    }

    Seq("bash", "-c", s"${projectDir.resolve("bin/create-cert.sh")} ${ip} ${cn} ${projectDir.resolve("bin").toAbsolutePath} ${caDir.toAbsolutePath} ${certDir.toAbsolutePath}")
      .lineStream_!(processLogger)
      .mkString("\n")
  }

  def addNode(keyPair: KeyPair, coinbase: Address, rootPath: Path, host: String): NetworkBuilder = {
    val config = base
      .lens(_.rootPath).set(rootPath.toAbsolutePath.toString)
      .lens(_.peer.host).set(host)
      .lens(_.service.local).set(host)
      .lens(_.service.enableMetrics).set(true)
//      .lens(_.service.secure).set(true)
      .lens(_.mining.enabled).set(true)
      .lens(_.mining.address).set(Address(keyPair))
      .lens(_.mining.coinbase).set(coinbase)
//      .lens(_.ssl.enabled).set(true)
      .lens(_.ssl.trustStorePath).set(rootPath.resolve("cert/cacert.jks").toAbsolutePath.toString)
      .lens(_.ssl.keyStorePath).set(rootPath.resolve("cert/server.jks").toAbsolutePath.toString)
      .lens(_.persist.driver).set("rocksdb")
      .lens(_.persist.path).set(s"${rootPath.resolve("data").toAbsolutePath}")
      .lens(_.log.logDir).set(s"${rootPath.resolve("logs").toAbsolutePath}")
      .lens(_.keystore.dir).set(s"${rootPath.resolve("keystore").toAbsolutePath}")
      .lens(_.db.driver).set("org.sqlite.JDBC")
      .lens(_.db.url).set(s"jdbc:sqlite:${rootPath.resolve(s"service.db")}")

    val keystore = new KeyStorePlatform[IO](config.keystore)
    keystore.importPrivateKey(keyPair.secret.bytes, "changeit").unsafeRunSync()

    createCert(host, host, root.resolve("ca"), rootPath.resolve("cert")).unsafeRunSync()
    copy(configs = config :: configs)
  }

  def build: List[FullConfig] = {
    val reversed = configs.reverse
    val seeds = reversed.map(_.peer).map { peer =>
      PeerUri.fromTcpAddr(new InetSocketAddress(peer.host, peer.port)).uri
    }

    reversed.zipWithIndex.map { case (config, i) => config.lens(_.peer.seeds).set(seeds.take(i) ++ seeds.drop(i + 1)) }
  }

  def dump: IO[Unit] =
    build.traverse_(config => Config[IO].dump(config.asJson, Paths.get(config.rootPath).resolve(s"config.yaml")))
} 

package jbok.common

import better.files.File
import cats.effect.IO
import jbok.common.FileUtil.FileLockErr

class FileUtilSpec extends CommonSpec {
  "FileUtil" should {
    "release lock whatever use" in {
      val file = File.newTemporaryFile()
      val p = FileUtil[IO]
        .lock(file.path)
        .use { _ =>
          IO.raiseError(new Exception("boom"))
        }
        .attempt
      p.unsafeRunSync()
      file.exists shouldBe false
    }

    "raise FileLockErr if already locked" in {
      val file = File.newTemporaryFile()
      val p = FileUtil[IO]
        .lock(file.path)
        .use { _ =>
          FileUtil[IO]
            .lock(file.path)
            .use { _ =>
              IO.unit
            }
            .attempt
            .map(x => x.left.get shouldBe FileLockErr(file.path))
        }
        .attempt
      p.unsafeRunSync().isRight shouldBe true
      file.exists shouldBe false
    }

    "lock with content" in {
      val file = File.newTemporaryFile()
      val p = FileUtil[IO]
        .lock(file.path, "oho")
        .use { _ =>
          IO(println(file.lines.head)).flatMap(_ => IO.raiseError(new Exception("boom")))
        }
        .attempt
      p.unsafeRunSync()
      file.exists shouldBe false
    }
  }
} 

package org.scalasteward.core.buildtool.maven

import better.files.File
import cats.Monad
import cats.implicits._
import org.scalasteward.core.application.Config
import org.scalasteward.core.buildtool.BuildToolAlg
import org.scalasteward.core.data._
import org.scalasteward.core.io.{FileAlg, ProcessAlg, WorkspaceAlg}
import org.scalasteward.core.scalafix.Migration
import org.scalasteward.core.util.Nel
import org.scalasteward.core.vcs.data.Repo

trait MavenAlg[F[_]] extends BuildToolAlg[F]

object MavenAlg {
  def create[F[_]](implicit
      config: Config,
      fileAlg: FileAlg[F],
      processAlg: ProcessAlg[F],
      workspaceAlg: WorkspaceAlg[F],
      F: Monad[F]
  ): MavenAlg[F] =
    new MavenAlg[F] {
      override def containsBuild(repo: Repo): F[Boolean] =
        workspaceAlg.repoDir(repo).flatMap(repoDir => fileAlg.isRegularFile(repoDir / "pom.xml"))

      override def getDependencies(repo: Repo): F[List[Scope.Dependencies]] =
        for {
          repoDir <- workspaceAlg.repoDir(repo)
          dependenciesRaw <- exec(mvnCmd(command.listDependencies), repoDir)
          repositoriesRaw <- exec(mvnCmd(command.listRepositories), repoDir)
          dependencies = parser.parseDependencies(dependenciesRaw).distinct
          resolvers = parser.parseResolvers(repositoriesRaw).distinct
        } yield List(Scope(dependencies, resolvers))

      override def runMigrations(repo: Repo, migrations: Nel[Migration]): F[Unit] =
        F.unit

      def exec(command: Nel[String], repoDir: File): F[List[String]] =
        maybeIgnoreOptsFiles(repoDir)(processAlg.execSandboxed(command, repoDir))

      def mvnCmd(commands: String*): Nel[String] =
        Nel("mvn", "--batch-mode" :: commands.toList)

      def maybeIgnoreOptsFiles[A](dir: File)(fa: F[A]): F[A] =
        if (config.ignoreOptsFiles) ignoreOptsFiles(dir)(fa) else fa

      def ignoreOptsFiles[A](dir: File)(fa: F[A]): F[A] =
        fileAlg.removeTemporarily(dir / ".jvmopts")(fa)
    }
} 

package org.scalasteward.core.io

import better.files.File
import cats.effect.{Blocker, IO}
import java.util.concurrent.Executors
import org.scalasteward.core.TestInstances._
import org.scalasteward.core.io.ProcessAlgTest.ioProcessAlg
import org.scalasteward.core.mock.MockContext._
import org.scalasteward.core.mock.MockState
import org.scalasteward.core.util.Nel
import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers

class ProcessAlgTest extends AnyFunSuite with Matchers {
  test("exec echo") {
    ioProcessAlg
      .exec(Nel.of("echo", "-n", "hello"), File.currentWorkingDirectory)
      .unsafeRunSync() shouldBe List("hello")
  }

  test("exec false") {
    ioProcessAlg
      .exec(Nel.of("ls", "--foo"), File.currentWorkingDirectory)
      .attempt
      .map(_.isLeft)
      .unsafeRunSync()
  }

  test("respect the disableSandbox setting") {
    val cfg = config.copy(disableSandbox = true)
    val processAlg = new MockProcessAlg()(cfg)

    val state = processAlg
      .execSandboxed(Nel.of("echo", "hello"), File.temp)
      .runS(MockState.empty)
      .unsafeRunSync()

    state shouldBe MockState.empty.copy(
      commands = Vector(
        List("TEST_VAR=GREAT", "ANOTHER_TEST_VAR=ALSO_GREAT", File.temp.toString, "echo", "hello")
      )
    )
  }

  test("execSandboxed echo") {
    val state = processAlg
      .execSandboxed(Nel.of("echo", "hello"), File.temp)
      .runS(MockState.empty)
      .unsafeRunSync()

    state shouldBe MockState.empty.copy(
      commands = Vector(
        List(
          "TEST_VAR=GREAT",
          "ANOTHER_TEST_VAR=ALSO_GREAT",
          File.temp.toString,
          "firejail",
          s"--whitelist=${File.temp}",
          "echo",
          "hello"
        )
      )
    )
  }
}

object ProcessAlgTest {
  val blocker: Blocker = Blocker.liftExecutorService(Executors.newCachedThreadPool())
  implicit val ioProcessAlg: ProcessAlg[IO] = ProcessAlg.create[IO](blocker)
} 

package org.scalasteward.core.io

import better.files.File
import cats.data.StateT
import cats.effect.IO
import fs2.Stream
import org.scalasteward.core.mock.{applyPure, MockEff, MockState}

class MockFileAlg extends FileAlg[MockEff] {
  override def createTemporarily[A](file: File, content: String)(fa: MockEff[A]): MockEff[A] =
    for {
      _ <- StateT.modify[IO, MockState](_.exec(List("create", file.pathAsString)))
      a <- fa
      _ <- StateT.modify[IO, MockState](_.exec(List("rm", file.pathAsString)))
    } yield a

  override def deleteForce(file: File): MockEff[Unit] =
    StateT.modify(_.exec(List("rm", "-rf", file.pathAsString)).rm(file))

  override def ensureExists(dir: File): MockEff[File] =
    applyPure(s => (s.exec(List("mkdir", "-p", dir.pathAsString)), dir))

  override def home: MockEff[File] =
    StateT.pure(File.root / "tmp" / "steward")

  override def isDirectory(file: File): MockEff[Boolean] =
    StateT.pure(false)

  override def isRegularFile(file: File): MockEff[Boolean] =
    for {
      _ <- StateT.modify[IO, MockState](_.exec(List("test", "-f", file.pathAsString)))
      s <- StateT.get[IO, MockState]
      exists = s.files.contains(file)
    } yield exists

  override def removeTemporarily[A](file: File)(fa: MockEff[A]): MockEff[A] =
    for {
      _ <- StateT.modify[IO, MockState](_.exec(List("rm", file.pathAsString)))
      a <- fa
      _ <- StateT.modify[IO, MockState](_.exec(List("restore", file.pathAsString)))
    } yield a

  override def readFile(file: File): MockEff[Option[String]] =
    applyPure(s => (s.exec(List("read", file.pathAsString)), s.files.get(file)))

  override def readResource(resource: String): MockEff[String] =
    for {
      _ <- StateT.modify[IO, MockState](_.exec(List("read", s"classpath:$resource")))
      content <- StateT.liftF(FileAlgTest.ioFileAlg.readResource(resource))
    } yield content

  override def walk(dir: File): Stream[MockEff, File] = {
    val dirAsString = dir.pathAsString
    val state: MockEff[List[File]] = StateT.inspect {
      _.files.keys.filter(_.pathAsString.startsWith(dirAsString)).toList
    }
    Stream.eval(state).flatMap(Stream.emits[MockEff, File])
  }

  override def writeFile(file: File, content: String): MockEff[Unit] =
    StateT.modify(_.exec(List("write", file.pathAsString)).add(file, content))
} 

package org.scalasteward.core.scalafix

import better.files.File
import org.scalasteward.core.data.{GroupId, Version}
import org.scalasteward.core.mock.MockContext._
import org.scalasteward.core.mock.{MockEff, MockState}
import org.scalasteward.core.util.Nel
import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers

class MigrationAlgTest extends AnyFunSuite with Matchers {
  val extraFile: File = config.workspace / "extra-migrations.conf"

  test("loadMigrations with extra file") {
    val content =
      """|migrations = [
         |  {
         |    groupId: "org.ice.cream",
         |    artifactIds: ["yumyum-.*"],
         |    newVersion: "1.0.0",
         |    rewriteRules: ["awesome rewrite rule"],
         |    doc: "https://scalacenter.github.io/scalafix/"
         |  }
         |]""".stripMargin
    val initialState = MockState.empty.add(extraFile, content)
    val migrations =
      MigrationAlg.loadMigrations[MockEff](Some(extraFile)).runA(initialState).unsafeRunSync

    migrations.size should be > 1
    (migrations should contain).oneElementOf(
      List(
        Migration(
          GroupId("org.ice.cream"),
          Nel.of("yumyum-.*"),
          Version("1.0.0"),
          Nel.of("awesome rewrite rule"),
          Some("https://scalacenter.github.io/scalafix/")
        )
      )
    )
  }

  test("loadMigrations with extra file and disableDefaults = true") {
    val content =
      """|disableDefaults = true
         |migrations = [
         |  {
         |    groupId: "org.ice.cream",
         |    artifactIds: ["yumyum-.*"],
         |    newVersion: "1.0.0",
         |    rewriteRules: ["awesome rewrite rule"]
         |  }
         |]""".stripMargin
    val initialState = MockState.empty.add(extraFile, content)
    val migrations =
      MigrationAlg.loadMigrations[MockEff](Some(extraFile)).runA(initialState).unsafeRunSync

    migrations shouldBe List(
      Migration(
        GroupId("org.ice.cream"),
        Nel.of("yumyum-.*"),
        Version("1.0.0"),
        Nel.of("awesome rewrite rule"),
        None
      )
    )
  }

  test("loadMigrations with extra file and disableDefaults = true only") {
    val initialState = MockState.empty.add(extraFile, "disableDefaults = true")
    val migrations =
      MigrationAlg.loadMigrations[MockEff](Some(extraFile)).runA(initialState).unsafeRunSync
    migrations.isEmpty shouldBe true
  }

  test("loadMigrations with malformed extra file") {
    val initialState = MockState.empty.add(extraFile, """{"key": "i'm not a valid Migration}""")
    val migrations =
      MigrationAlg.loadMigrations[MockEff](Some(extraFile)).runA(initialState).attempt.unsafeRunSync
    migrations.isLeft shouldBe true
  }

  test("loadMigrations without extra file") {
    val migrations =
      MigrationAlg.loadMigrations[MockEff](None).runA(MockState.empty).unsafeRunSync()
    migrations.size shouldBe 13
  }
} 

package org.scalasteward.core.buildtool.maven

import better.files.File
import org.scalasteward.core.mock.MockContext._
import org.scalasteward.core.mock.MockState
import org.scalasteward.core.vcs.data.Repo
import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers

class MavenAlgTest extends AnyFunSuite with Matchers {
  val var1 = "TEST_VAR=GREAT"
  val var2 = "ANOTHER_TEST_VAR=ALSO_GREAT"

  test("getDependencies") {
    val repo = Repo("namespace", "repo-name")
    val repoDir = config.workspace / repo.show
    val files: Map[File, String] = Map.empty

    val state =
      mavenAlg.getDependencies(repo).runS(MockState.empty.copy(files = files)).unsafeRunSync()

    state shouldBe MockState(
      files = files,
      logs = Vector.empty,
      commands = Vector(
        List(
          var1,
          var2,
          repoDir.toString,
          "firejail",
          s"--whitelist=$repoDir",
          "mvn",
          "--batch-mode",
          command.listDependencies
        ),
        List(
          var1,
          var2,
          repoDir.toString,
          "firejail",
          s"--whitelist=$repoDir",
          "mvn",
          "--batch-mode",
          command.listRepositories
        )
      )
    )
  }
} 

package org.scalasteward.core.mock

import better.files.File
import cats.Parallel
import cats.effect.Sync
import org.http4s.Uri
import org.scalasteward.core.TestInstances.ioContextShift
import org.scalasteward.core.application.Cli.EnvVar
import org.scalasteward.core.application.{Config, SupportedVCS}
import org.scalasteward.core.buildtool.BuildToolDispatcher
import org.scalasteward.core.buildtool.maven.MavenAlg
import org.scalasteward.core.buildtool.mill.MillAlg
import org.scalasteward.core.buildtool.sbt.SbtAlg
import org.scalasteward.core.coursier.{CoursierAlg, VersionsCache}
import org.scalasteward.core.edit.EditAlg
import org.scalasteward.core.git.{Author, GitAlg}
import org.scalasteward.core.io.{MockFileAlg, MockProcessAlg, MockWorkspaceAlg}
import org.scalasteward.core.nurture.PullRequestRepository
import org.scalasteward.core.persistence.JsonKeyValueStore
import org.scalasteward.core.repocache.RepoCacheRepository
import org.scalasteward.core.repoconfig.RepoConfigAlg
import org.scalasteward.core.scalafix.MigrationAlg
import org.scalasteward.core.scalafmt.ScalafmtAlg
import org.scalasteward.core.update.{FilterAlg, GroupMigrations, PruningAlg, UpdateAlg}
import org.scalasteward.core.util.uri._
import org.scalasteward.core.util.{BracketThrowable, DateTimeAlg}
import org.scalasteward.core.vcs.VCSRepoAlg
import org.scalasteward.core.vcs.data.AuthenticatedUser

import scala.concurrent.duration._

object MockContext {
  implicit val config: Config = Config(
    workspace = File.temp / "ws",
    reposFile = File.temp / "repos.md",
    defaultRepoConfigFile = Some(File.temp / "default.scala-steward.conf"),
    gitAuthor = Author("Bot Doe", "[email protected]"),
    vcsType = SupportedVCS.GitHub,
    vcsApiHost = Uri(),
    vcsLogin = "bot-doe",
    gitAskPass = File.temp / "askpass.sh",
    signCommits = false,
    whitelistedDirectories = Nil,
    readOnlyDirectories = Nil,
    disableSandbox = false,
    doNotFork = false,
    ignoreOptsFiles = false,
    envVars = List(
      EnvVar("TEST_VAR", "GREAT"),
      EnvVar("ANOTHER_TEST_VAR", "ALSO_GREAT")
    ),
    processTimeout = 10.minutes,
    scalafixMigrations = None,
    groupMigrations = None,
    cacheTtl = 1.hour,
    cacheMissDelay = 0.milliseconds,
    bitbucketServerUseDefaultReviewers = false
  )

  implicit val mockEffBracketThrowable: BracketThrowable[MockEff] = Sync[MockEff]
  implicit val mockEffParallel: Parallel[MockEff] = Parallel.identity

  implicit val fileAlg: MockFileAlg = new MockFileAlg
  implicit val mockLogger: MockLogger = new MockLogger
  implicit val processAlg: MockProcessAlg = new MockProcessAlg
  implicit val workspaceAlg: MockWorkspaceAlg = new MockWorkspaceAlg

  implicit val coursierAlg: CoursierAlg[MockEff] = CoursierAlg.create
  implicit val dateTimeAlg: DateTimeAlg[MockEff] = DateTimeAlg.create
  implicit val gitAlg: GitAlg[MockEff] = GitAlg.create
  implicit val user: AuthenticatedUser = AuthenticatedUser("scala-steward", "token")
  implicit val vcsRepoAlg: VCSRepoAlg[MockEff] = VCSRepoAlg.create(config, gitAlg)
  implicit val scalafmtAlg: ScalafmtAlg[MockEff] = ScalafmtAlg.create
  implicit val migrationAlg: MigrationAlg =
    MigrationAlg.create[MockEff](config.scalafixMigrations).runA(MockState.empty).unsafeRunSync()
  implicit val cacheRepository: RepoCacheRepository[MockEff] =
    new RepoCacheRepository[MockEff](new JsonKeyValueStore("repo_cache", "1"))
  implicit val filterAlg: FilterAlg[MockEff] = new FilterAlg[MockEff]
  implicit val versionsCache: VersionsCache[MockEff] =
    new VersionsCache[MockEff](config.cacheTtl, new JsonKeyValueStore("versions", "1"))
  implicit val groupMigrations: GroupMigrations =
    GroupMigrations.create[MockEff].runA(MockState.empty).unsafeRunSync()
  implicit val updateAlg: UpdateAlg[MockEff] = new UpdateAlg[MockEff]
  implicit val mavenAlg: MavenAlg[MockEff] = MavenAlg.create
  implicit val sbtAlg: SbtAlg[MockEff] = SbtAlg.create
  implicit val millAlg: MillAlg[MockEff] = MillAlg.create
  implicit val buildToolDispatcher: BuildToolDispatcher[MockEff] = BuildToolDispatcher.create
  implicit val editAlg: EditAlg[MockEff] = new EditAlg[MockEff]
  implicit val repoConfigAlg: RepoConfigAlg[MockEff] = new RepoConfigAlg[MockEff]
  implicit val pullRequestRepository: PullRequestRepository[MockEff] =
    new PullRequestRepository[MockEff](new JsonKeyValueStore("pull_requests", "1"))
  implicit val pruningAlg: PruningAlg[MockEff] = new PruningAlg[MockEff]
} 

package org.scalasteward.core.mock

import better.files.File

final case class MockState(
    commands: Vector[List[String]],
    logs: Vector[(Option[Throwable], String)],
    files: Map[File, String]
) {
  def add(file: File, content: String): MockState =
    copy(files = files + (file -> content))

  def addFiles(newFiles: Map[File, String]): MockState =
    copy(files = files ++ newFiles)

  def rm(file: File): MockState =
    copy(files = files - file)

  def exec(cmd: List[String], env: (String, String)*): MockState =
    copy(commands = commands :+ (env.map { case (k, v) => s"$k=$v" }.toList ++ cmd))

  def log(maybeThrowable: Option[Throwable], msg: String): MockState =
    copy(logs = logs :+ ((maybeThrowable, msg)))
}

object MockState {
  def empty: MockState =
    MockState(commands = Vector.empty, logs = Vector.empty, files = Map.empty)
} 

package org.scalasteward.core

import better.files.File
import cats.implicits._
import org.scalasteward.core.data.{GroupId, Update}

package object io {
  def isSourceFile(update: Update, fileExtensions: Set[String])(file: File): Boolean = {
    val notInGitDir = !file.pathAsString.contains(".git/")
    notInGitDir && isSpecificOrGenericSourceFile(update, fileExtensions)(file)
  }

  private def isSpecificOrGenericSourceFile(update: Update, fileExtensions: Set[String])(
      file: File
  ): Boolean =
    () match {
      case _ if isSbtUpdate(update)          => file.name === "build.properties"
      case _ if isScalafmtCoreUpdate(update) => file.name === ".scalafmt.conf"
      case _                                 => isGenericSourceFile(file, fileExtensions)
    }

  private def isGenericSourceFile(file: File, fileExtensions: Set[String]): Boolean =
    fileExtensions.exists(file.name.endsWith)

  private def isSbtUpdate(update: Update): Boolean =
    update.groupId === GroupId("org.scala-sbt") &&
      update.artifactIds.exists(_.name === "sbt")

  private def isScalafmtCoreUpdate(update: Update): Boolean =
    update.groupId === GroupId("org.scalameta") &&
      update.artifactIds.exists(_.name === "scalafmt-core")
} 

package org.scalasteward.core.io

import better.files.File
import cats.effect.{Blocker, Concurrent, ContextShift, Timer}
import cats.implicits._
import io.chrisdavenport.log4cats.Logger
import org.scalasteward.core.application.Cli.EnvVar
import org.scalasteward.core.application.Config
import org.scalasteward.core.util.Nel

trait ProcessAlg[F[_]] {
  def exec(command: Nel[String], cwd: File, extraEnv: (String, String)*): F[List[String]]

  def execSandboxed(command: Nel[String], cwd: File): F[List[String]]
}

object ProcessAlg {
  abstract class UsingFirejail[F[_]](config: Config) extends ProcessAlg[F] {
    override def execSandboxed(command: Nel[String], cwd: File): F[List[String]] = {
      val envVars = config.envVars.map(EnvVar.unapply(_).get)
      if (config.disableSandbox)
        exec(command, cwd, envVars: _*)
      else {
        val whitelisted = (cwd.pathAsString :: config.whitelistedDirectories)
          .map(dir => s"--whitelist=$dir")
        val readOnly = config.readOnlyDirectories
          .map(dir => s"--read-only=$dir")
        exec(Nel("firejail", whitelisted ++ readOnly) ::: command, cwd, envVars: _*)
      }
    }
  }

  def create[F[_]](blocker: Blocker)(implicit
      config: Config,
      contextShift: ContextShift[F],
      logger: Logger[F],
      timer: Timer[F],
      F: Concurrent[F]
  ): ProcessAlg[F] =
    new UsingFirejail[F](config) {
      override def exec(
          command: Nel[String],
          cwd: File,
          extraEnv: (String, String)*
      ): F[List[String]] =
        logger.debug(s"Execute ${command.mkString_(" ")}") >>
          process.slurp[F](
            command,
            Some(cwd.toJava),
            extraEnv.toMap,
            config.processTimeout,
            logger.trace(_),
            blocker
          )
    }
} 

package org.scalasteward.core.io

import better.files.File
import cats.FlatMap
import cats.implicits._
import io.chrisdavenport.log4cats.Logger
import org.scalasteward.core.application.Config
import org.scalasteward.core.vcs.data.Repo

trait WorkspaceAlg[F[_]] {
  def cleanWorkspace: F[Unit]

  def rootDir: F[File]

  def repoDir(repo: Repo): F[File]
}

object WorkspaceAlg {
  def create[F[_]](implicit
      fileAlg: FileAlg[F],
      logger: Logger[F],
      config: Config,
      F: FlatMap[F]
  ): WorkspaceAlg[F] =
    new WorkspaceAlg[F] {
      private[this] val reposDir = config.workspace / "repos"

      override def cleanWorkspace: F[Unit] =
        for {
          _ <- logger.info(s"Clean workspace ${config.workspace}")
          _ <- fileAlg.deleteForce(reposDir)
          _ <- rootDir
        } yield ()

      override def rootDir: F[File] =
        fileAlg.ensureExists(config.workspace)

      override def repoDir(repo: Repo): F[File] =
        fileAlg.ensureExists(reposDir / repo.owner / repo.repo)
    }
} 

package org.scalasteward.core.scalafix

import better.files.File
import cats.data.OptionT
import cats.effect.Sync
import cats.implicits._
import io.circe.config.parser.decode
import org.scalasteward.core.data.{Update, Version}
import org.scalasteward.core.io.FileAlg
import org.scalasteward.core.util.{ApplicativeThrowable, MonadThrowable}

trait MigrationAlg {
  def findMigrations(update: Update): List[Migration]
}

object MigrationAlg {
  def create[F[_]](extraMigrations: Option[File])(implicit
      fileAlg: FileAlg[F],
      F: Sync[F]
  ): F[MigrationAlg] =
    loadMigrations(extraMigrations).map { migrations =>
      new MigrationAlg {
        override def findMigrations(update: Update): List[Migration] =
          findMigrationsImpl(migrations, update)
      }
    }

  def loadMigrations[F[_]](
      extraMigrations: Option[File]
  )(implicit fileAlg: FileAlg[F], F: MonadThrowable[F]): F[List[Migration]] =
    for {
      default <-
        fileAlg
          .readResource("scalafix-migrations.conf")
          .flatMap(decodeMigrations[F](_, "default"))
      maybeExtra <- OptionT(extraMigrations.flatTraverse(fileAlg.readFile))
        .semiflatMap(decodeMigrations[F](_, "extra"))
        .value
      migrations = maybeExtra match {
        case Some(extra) if extra.disableDefaults => extra.migrations
        case Some(extra)                          => default.migrations ++ extra.migrations
        case None                                 => default.migrations
      }
    } yield migrations

  private def decodeMigrations[F[_]](content: String, tpe: String)(implicit
      F: ApplicativeThrowable[F]
  ): F[ScalafixMigrations] =
    F.fromEither(decode[ScalafixMigrations](content))
      .adaptErr(new Throwable(s"Failed to load $tpe Scalafix migrations", _))

  private def findMigrationsImpl(
      givenMigrations: List[Migration],
      update: Update
  ): List[Migration] =
    givenMigrations.filter { migration =>
      update.groupId === migration.groupId &&
      migration.artifactIds.exists(re =>
        update.artifactIds.exists(artifactId => re.r.findFirstIn(artifactId.name).isDefined)
      ) &&
      Version(update.currentVersion) < migration.newVersion &&
      Version(update.newerVersions.head) >= migration.newVersion
    }
} 

package org.scalasteward.core.persistence

import better.files.File
import cats.Monad
import cats.implicits._
import io.chrisdavenport.log4cats.Logger
import io.circe.parser.decode
import io.circe.syntax._
import io.circe.{Decoder, Encoder, KeyEncoder}
import org.scalasteward.core.io.{FileAlg, WorkspaceAlg}

final class JsonKeyValueStore[F[_], K, V](
    name: String,
    schemaVersion: String,
    maybePrefix: Option[String] = None
)(implicit
    fileAlg: FileAlg[F],
    keyEncoder: KeyEncoder[K],
    logger: Logger[F],
    valueDecoder: Decoder[V],
    valueEncoder: Encoder[V],
    workspaceAlg: WorkspaceAlg[F],
    F: Monad[F]
) extends KeyValueStore[F, K, V] {
  override def get(key: K): F[Option[V]] =
    jsonFile(key).flatMap { file =>
      fileAlg.readFile(file).flatMap {
        case Some(content) =>
          decode[Option[V]](content) match {
            case Right(maybeValue) => F.pure(maybeValue)
            case Left(error) =>
              logger.error(error)(s"Failed to parse or decode JSON from $file").as(Option.empty[V])
          }
        case None => F.pure(Option.empty[V])
      }
    }

  override def put(key: K, value: V): F[Unit] =
    write(key, Some(value))

  override def modifyF(key: K)(f: Option[V] => F[Option[V]]): F[Option[V]] =
    get(key).flatMap(maybeValue => f(maybeValue).flatTap(write(key, _)))

  private def jsonFile(key: K): F[File] = {
    val keyPath = maybePrefix.fold("")(_ + "/") + keyEncoder(key)
    workspaceAlg.rootDir.map(_ / "store" / name / s"v$schemaVersion" / keyPath / s"$name.json")
  }

  private def write(key: K, value: Option[V]): F[Unit] =
    jsonFile(key).flatMap(fileAlg.writeFile(_, value.asJson.toString))
} 

package org.scalasteward.core.repoconfig

import better.files.File
import cats.data.OptionT
import cats.implicits._
import io.chrisdavenport.log4cats.Logger
import io.circe.config.parser
import org.scalasteward.core.application.Config
import org.scalasteward.core.data.Update
import org.scalasteward.core.io.{FileAlg, WorkspaceAlg}
import org.scalasteward.core.repoconfig.RepoConfigAlg._
import org.scalasteward.core.util.MonadThrowable
import org.scalasteward.core.vcs.data.Repo

final class RepoConfigAlg[F[_]](implicit
    config: Config,
    fileAlg: FileAlg[F],
    logger: Logger[F],
    workspaceAlg: WorkspaceAlg[F],
    F: MonadThrowable[F]
) {
  def readRepoConfigOrDefault(repo: Repo): F[RepoConfig] =
    readRepoConfig(repo).flatMap { config =>
      config.map(F.pure).getOrElse(defaultRepoConfig)
    }

  
  val defaultRepoConfig: F[RepoConfig] =
    OptionT
      .fromOption[F](config.defaultRepoConfigFile)
      .flatMap(readRepoConfigFromFile)
      .getOrElse(RepoConfig.empty)

  def readRepoConfig(repo: Repo): F[Option[RepoConfig]] =
    workspaceAlg
      .repoDir(repo)
      .flatMap(dir => readRepoConfigFromFile(dir / repoConfigBasename).value)

  private def readRepoConfigFromFile(configFile: File): OptionT[F, RepoConfig] =
    OptionT(fileAlg.readFile(configFile)).map(parseRepoConfig).flatMapF {
      case Right(repoConfig) => logger.info(s"Parsed $repoConfig").as(repoConfig.some)
      case Left(errorMsg)    => logger.info(errorMsg).as(none[RepoConfig])
    }
}

object RepoConfigAlg {
  val repoConfigBasename: String = ".scala-steward.conf"

  def parseRepoConfig(input: String): Either[String, RepoConfig] =
    parser.decode[RepoConfig](input).leftMap { error =>
      s"Failed to parse $repoConfigBasename: ${error.getMessage}"
    }

  def configToIgnoreFurtherUpdates(update: Update): String =
    update match {
      case s: Update.Single =>
        s"""updates.ignore = [ { groupId = "${s.groupId}", artifactId = "${s.artifactId.name}" } ]"""
      case g: Update.Group =>
        s"""updates.ignore = [ { groupId = "${g.groupId}" } ]"""
    }
} 

package org.scalasteward.core.application

import better.files.File
import cats.Monad
import cats.effect.ExitCode
import cats.implicits._
import fs2.Stream
import io.chrisdavenport.log4cats.Logger
import org.scalasteward.core.buildtool.sbt.SbtAlg
import org.scalasteward.core.git.GitAlg
import org.scalasteward.core.io.{FileAlg, WorkspaceAlg}
import org.scalasteward.core.nurture.NurtureAlg
import org.scalasteward.core.repocache.RepoCacheAlg
import org.scalasteward.core.update.PruningAlg
import org.scalasteward.core.util
import org.scalasteward.core.util.DateTimeAlg
import org.scalasteward.core.util.logger.LoggerOps
import org.scalasteward.core.vcs.data.Repo

final class StewardAlg[F[_]](implicit
    config: Config,
    dateTimeAlg: DateTimeAlg[F],
    fileAlg: FileAlg[F],
    gitAlg: GitAlg[F],
    logger: Logger[F],
    nurtureAlg: NurtureAlg[F],
    pruningAlg: PruningAlg[F],
    repoCacheAlg: RepoCacheAlg[F],
    sbtAlg: SbtAlg[F],
    selfCheckAlg: SelfCheckAlg[F],
    streamCompiler: Stream.Compiler[F, F],
    workspaceAlg: WorkspaceAlg[F],
    F: Monad[F]
) {
  private def printBanner: F[Unit] = {
    val banner =
      """|  ____            _         ____  _                             _
         | / ___|  ___ __ _| | __ _  / ___|| |_ _____      ____ _ _ __ __| |
         | \___ \ / __/ _` | |/ _` | \___ \| __/ _ \ \ /\ / / _` | '__/ _` |
         |  ___) | (_| (_| | | (_| |  ___) | ||  __/\ V  V / (_| | | | (_| |
         | |____/ \___\__,_|_|\__,_| |____/ \__\___| \_/\_/ \__,_|_|  \__,_|""".stripMargin
    val msg = List(" ", banner, s" v${org.scalasteward.core.BuildInfo.version}", " ")
      .mkString(System.lineSeparator())
    logger.info(msg)
  }

  private def readRepos(reposFile: File): F[List[Repo]] =
    fileAlg.readFile(reposFile).map { maybeContent =>
      val regex = """-\s+(.+)/([^/]+)""".r
      val content = maybeContent.getOrElse("")
      content.linesIterator.collect {
        case regex(owner, repo) => Repo(owner.trim, repo.trim)
      }.toList
    }

  private def steward(repo: Repo): F[Either[Throwable, Unit]] = {
    val label = s"Steward ${repo.show}"
    logger.infoTotalTime(label) {
      for {
        _ <- logger.info(util.string.lineLeftRight(label))
        _ <- repoCacheAlg.checkCache(repo)
        (attentionNeeded, updates) <- pruningAlg.needsAttention(repo)
        result <- {
          if (attentionNeeded) nurtureAlg.nurture(repo, updates)
          else gitAlg.removeClone(repo).as(().asRight[Throwable])
        }
      } yield result
    }
  }

  def runF: F[ExitCode] =
    logger.infoTotalTime("run") {
      for {
        _ <- printBanner
        _ <- selfCheckAlg.checkAll
        exitCode <- sbtAlg.addGlobalPlugins {
          for {
            _ <- workspaceAlg.cleanWorkspace
            repos <- readRepos(config.reposFile)
            result <- Stream.emits(repos).evalMap(steward).compile.foldMonoid
          } yield result.fold(_ => ExitCode.Error, _ => ExitCode.Success)
        }
      } yield exitCode
    }
} 

package org.scalasteward.core.edit

import better.files.File
import cats.Traverse
import cats.implicits._
import fs2.Stream
import io.chrisdavenport.log4cats.Logger
import org.scalasteward.core.buildtool.BuildToolDispatcher
import org.scalasteward.core.data.Update
import org.scalasteward.core.io.{isSourceFile, FileAlg, WorkspaceAlg}
import org.scalasteward.core.scalafix.MigrationAlg
import org.scalasteward.core.util._
import org.scalasteward.core.vcs.data.Repo

final class EditAlg[F[_]](implicit
    buildToolDispatcher: BuildToolDispatcher[F],
    fileAlg: FileAlg[F],
    logger: Logger[F],
    migrationAlg: MigrationAlg,
    streamCompiler: Stream.Compiler[F, F],
    workspaceAlg: WorkspaceAlg[F],
    F: MonadThrowable[F]
) {
  def applyUpdate(repo: Repo, update: Update, fileExtensions: Set[String]): F[Unit] =
    for {
      _ <- applyScalafixMigrations(repo, update).handleErrorWith(e =>
        logger.warn(s"Could not apply ${update.show} : $e")
      )
      repoDir <- workspaceAlg.repoDir(repo)
      files <- fileAlg.findFilesContaining(
        repoDir,
        update.currentVersion,
        isSourceFile(update, fileExtensions)
      )
      noFilesFound = logger.warn("No files found that contain the current version")
      _ <- files.toNel.fold(noFilesFound)(applyUpdateTo(_, update))
    } yield ()

  def applyUpdateTo[G[_]: Traverse](files: G[File], update: Update): F[Unit] = {
    val actions = UpdateHeuristic.all.map { heuristic =>
      logger.info(s"Trying heuristic '${heuristic.name}'") >>
        fileAlg.editFiles(files, heuristic.replaceVersion(update))
    }
    bindUntilTrue(actions).void
  }

  def applyScalafixMigrations(repo: Repo, update: Update): F[Unit] =
    Nel.fromList(migrationAlg.findMigrations(update)).traverse_ { migrations =>
      logger.info(s"Applying migrations: $migrations") >>
        buildToolDispatcher.runMigrations(repo, migrations)
    }
} 

package com.codacy.analysis.core.files

import better.files.File
import cats.instances.try_.catsStdInstancesForTry
import org.specs2.control.NoLanguageFeatures
import org.specs2.mutable.Specification

import scala.util.{Failure, Success, Try}

class FallbackFileCollectorSpec extends Specification with NoLanguageFeatures {

  private val failingCompanion: FileCollectorCompanion[Try] = new FileCollectorCompanion[Try] {
    override def name: String = ""

    override def apply(): FileCollector[Try] =
      new FileCollector[Try] {

        override def list(directory: File): Try[FilesTarget] = {
          Failure(new Exception("because fail"))
        }
      }
  }

  private val successfulCompanion: FileCollectorCompanion[Try] = new FileCollectorCompanion[Try] {
    override def name: String = ""

    override def apply(): FileCollector[Try] =
      new FileCollector[Try] {

        override def list(directory: File): Try[FilesTarget] = {
          Success(FilesTarget(directory, Set.empty, Set.empty))
        }
      }
  }

  "FallbackFileCollectorSpec" should {
    "not fallback" in {
      new FallbackFileCollector(List(successfulCompanion, failingCompanion)).list(File("")) must beSuccessfulTry
    }

    "fallback" in {
      new FallbackFileCollector(List(failingCompanion, successfulCompanion)).list(File("")) must beSuccessfulTry
    }

    "fail when all fail" in {
      new FallbackFileCollector(List(failingCompanion, failingCompanion)).list(File("")) must beFailedTry
    }
  }
} 

package com.codacy.analysis.core.utils

import better.files.File
import com.codacy.analysis.core.files.FilesTarget
import com.codacy.plugins.api.languages.Languages
import org.specs2.control.NoLanguageFeatures
import org.specs2.mutable.Specification

class LanguagesHelperSpec extends Specification with NoLanguageFeatures {

  "LanguagesHelper" should {
    "detect the languages from a given set of files" in {

      val filesTarget =
        FilesTarget(File(""), Set(File("test.js").path, File("Test.java").path, File("SomeClazz.rb").path), Set.empty)

      val languages = LanguagesHelper.fromFileTarget(filesTarget, Map.empty)

      languages should containTheSameElementsAs(Seq(Languages.Java, Languages.Ruby, Languages.Javascript))
    }

    "detect the languages from a given set of files, considering custom extensions for some of them" in {
      val filesTarget = FilesTarget(
        File(""),
        Set(
          File("test.css").path,
          File("test.js-that_will_be_kotlin").path,
          File("Test-1.java-that_will_be_ruby").path,
          File("test-rb.resource").path),
        Set.empty)

      val languages = LanguagesHelper.fromFileTarget(
        filesTarget,
        Map(
          Languages.Ruby -> Set("-rb.resource", "-1.java-that_will_be_ruby"),
          Languages.Kotlin -> Set(".js-that_will_be_kotlin")))

      languages should containTheSameElementsAs(Seq(Languages.Kotlin, Languages.Ruby, Languages.CSS))
    }

    "return an empty set of languages for extensions that do not match any language" in {
      val filesTarget =
        FilesTarget(
          File(""),
          Set(File("test.exotericLanguage").path, File("test-rb.anotherLanguageThatNoOneUses").path),
          Set.empty)

      val languages = LanguagesHelper.fromFileTarget(filesTarget, Map.empty)

      languages should beEmpty
    }
  }
} 

package com.codacy.analysis.core.utils

import java.nio.file.attribute.PosixFilePermission
import java.nio.file.{Path, Paths}

import better.files.File
import com.codacy.plugins.api.results
import io.circe.Decoder
import org.specs2.concurrent.ExecutionEnv
import org.specs2.matcher.MatchResult

import scala.sys.process.Process

object TestUtils {

  implicit val categoryDecoder: Decoder[results.Pattern.Category.Value] =
    Decoder.decodeEnumeration(results.Pattern.Category)

  implicit val levelDecoder: Decoder[results.Result.Level.Value] =
    Decoder.decodeEnumeration(results.Result.Level)
  implicit val fileDecoder: Decoder[Path] = Decoder[String].map(Paths.get(_))
  implicit val executionEnv: ExecutionEnv = ExecutionEnv.fromGlobalExecutionContext

  def withClonedRepo[T](gitUrl: String, commitUUid: String)(block: (File, File) => MatchResult[T]): MatchResult[T] =
    (for {
      directory <- File.temporaryDirectory()
      file <- File.temporaryFile()
    } yield {
      directory
        .addPermission(PosixFilePermission.OWNER_READ)
        .addPermission(PosixFilePermission.GROUP_READ)
        .addPermission(PosixFilePermission.OTHERS_READ)
        .addPermission(PosixFilePermission.OWNER_EXECUTE)
        .addPermission(PosixFilePermission.GROUP_EXECUTE)
        .addPermission(PosixFilePermission.OTHERS_EXECUTE)
      Process(Seq("git", "clone", gitUrl, directory.pathAsString)).!
      Process(Seq("git", "reset", "--hard", commitUUid), directory.toJava).!
      block(file, directory)
    }).get()

  def withTemporaryGitRepo[T](fn: File => MatchResult[T]): MatchResult[T] = {
    (for {
      temporaryDirectory <- File.temporaryDirectory()
    } yield {
      Process(Seq("git", "init"), temporaryDirectory.toJava).!
      Process(Seq("git", "commit", "--allow-empty", "-m", "initial commit"), temporaryDirectory.toJava).!
      fn(temporaryDirectory)
    }).get

  }
} 

package com.codacy.analysis.core.git

import better.files.File
import org.specs2.control.NoLanguageFeatures
import org.specs2.mutable.Specification
import com.codacy.analysis.core.utils.TestUtils._

import scala.sys.process.Process
import scala.util.Success

class CommitSpec extends Specification with NoLanguageFeatures {

  "Commit" should {
    "get all files" in {
      withTemporaryGitRepo { temporaryDirectory =>
        (for {
          tempFile1 <- File.temporaryFile(parent = Some(temporaryDirectory))
          tempFile2 <- File.temporaryFile(parent = Some(temporaryDirectory))
          tempFile3 <- File.temporaryFile(parent = Some(temporaryDirectory))
        } yield {

          def addFile(file: File) = {
            Process(Seq("git", "add", temporaryDirectory.relativize(file).toString), temporaryDirectory.toJava).!
          }
          addFile(tempFile1)
          addFile(tempFile2)
          addFile(tempFile3)
          Process(Seq("git", "commit", "-m", "tmp"), temporaryDirectory.toJava).!

          val expectedFiles =
            List(tempFile1, tempFile2, tempFile3).map(temporaryDirectory.relativize)
          Git.repository(temporaryDirectory).flatMap(_.latestCommit).flatMap(_.files) must beLike {
            case Success(fileSet) => fileSet must containTheSameElementsAs(expectedFiles)
          }
        }).get()
      }
    }
  }
} 

package com.codacy.analysis.core.git

import better.files.File
import com.codacy.analysis.core.utils.TestUtils._
import org.specs2.control.NoLanguageFeatures
import org.specs2.mutable.Specification

import scala.sys.process.Process
import scala.util.Success

class RepositorySpec extends Specification with NoLanguageFeatures {

  "Repository" should {
    "get latest commit" in {

      "when it exists" in {
        (for {
          temporaryDirectory <- File.temporaryDirectory()
          temporaryFile <- File.temporaryFile(parent = Some(temporaryDirectory))
        } yield {
          Process(Seq("git", "init"), temporaryDirectory.toJava).!
          Process(Seq("git", "add", temporaryDirectory.relativize(temporaryFile).toString), temporaryDirectory.toJava).!
          Process(Seq("git", "commit", "-m", "tmp"), temporaryDirectory.toJava).!

          Git.repository(temporaryDirectory).flatMap(_.latestCommit) must beSuccessfulTry
        }).get
      }

      "when it doesn't exist" in {
        (for {
          temporaryDirectory <- File.temporaryDirectory()
        } yield {
          Process(Seq("git", "init"), temporaryDirectory.toJava).!

          Git.repository(temporaryDirectory).flatMap(_.latestCommit) must beFailedTry
        }).get
      }
    }

    "get all uncommitted changes" in {

      "changed files" in {
        withTemporaryGitRepo { directory =>
          val file = directory / "random_file.file"
          file.createFileIfNotExists(createParents = true)

          Process(Seq("git", "add", "."), directory.toJava).!
          Process(Seq("git", "commit", "-m", "added a new file!"), directory.toJava).!

          file.write("Random file contents")

          Git.repository(directory).flatMap(_.uncommitedFiles) must beLike {
            case Success(uncommited) =>
              uncommited must contain(exactly(relativePath(file, directory)))
          }
        }
      }

      "untracked files" in {
        "with an untracked folder that contains an untracked file" in {
          withTemporaryGitRepo { directory =>
            val deepFile = directory / "mainFolder" / "subFolder" / "deepFile.sc"
            deepFile.createFileIfNotExists(createParents = true)

            Git.repository(directory).flatMap(_.uncommitedFiles) must beLike {
              case Success(uncommited) =>
                uncommited must contain(exactly(relativePath(deepFile, directory)))
            }
          }
        }

        "with an untracked folder with no content" in {
          withTemporaryGitRepo { directory =>
            val noContentsFolder = directory / "mainFolder" / "noContents"
            noContentsFolder.createDirectoryIfNotExists(createParents = true)

            Git.repository(directory).flatMap(_.uncommitedFiles) must beLike {
              case Success(uncommited) =>
                uncommited must beEmpty
            }
          }
        }
      }
    }

  }

  private def relativePath(targetFile: File, directory: File): String = {
    targetFile.pathAsString.stripPrefix(s"${directory.pathAsString}/")
  }
} 

package jbok.common.config

import java.nio.file.Path

import better.files.File
import cats.effect.Sync
import cats.implicits._
import io.circe
import io.circe.{Decoder, Json}
import jbok.common.FileUtil
import jbok.common.log.Logger

trait Config[F[_]] {
  def read[A: Decoder](path: Path): F[A]

  def read[A: Decoder](text: String): F[A]

  def readResource[A: Decoder](name: String): F[A]

  def dump(json: Json, path: Path): F[Unit]
}

object Config {
  def apply[F[_]](implicit ev: Config[F]): Config[F] = ev

  implicit def instance[F[_]](implicit F: Sync[F]): Config[F] = new Config[F] {
    private val log = Logger[F]

    override def read[A: Decoder](path: Path): F[A] =
      log.i(s"reading config from path=${path}") >>
        (File(path).extension match {
          case Some(".json") =>
            FileUtil[F].read(path).flatMap(text => F.fromEither(circe.parser.decode[A](text)))
          case Some(".yml") | Some(".yaml") =>
            FileUtil[F].read(path).flatMap(text => F.fromEither(circe.yaml.parser.parse(text).flatMap(_.as[A])))
          case Some(ext) =>
            F.raiseError(new Exception(s"Unknown extension path=${path},ext=${ext}"))
          case None =>
            F.raiseError(new Exception(s"No extension path=${path}"))
        })

    override def read[A: Decoder](text: String): F[A] =
      log.i(s"parsing config from text") >>
        F.fromEither(circe.yaml.parser.parse(text).flatMap(_.as[A]))

    override def readResource[A: Decoder](name: String): F[A] =
      log.i(s"reading config from resource=${name}") >>
        FileUtil[F].readResource(name) >>= read[A]

    override def dump(json: Json, path: Path): F[Unit] =
      log.i(s"dumping config to path=${path}") >>
        (File(path).extension match {
          case Some(".json") =>
            FileUtil[F].dump(json.spaces2, path)
          case Some(".yml") | Some(".yaml") =>
            FileUtil[F].dump(circe.yaml.Printer.spaces2.copy(preserveOrder = true, indicatorIndent = 2).pretty(json), path)
          case Some(ext) =>
            F.raiseError(new Exception(s"Unknown extension path=${path},ext=${ext}"))
          case None =>
            F.raiseError(new Exception(s"No extension path=${path}"))
        })
  }
}