org.apache.mesos.Protos Scala Examples

package org.apache.spark.scheduler.cluster.mesos

import scala.collection.JavaConverters._

import org.apache.mesos.Protos

import org.apache.spark.SparkException
import org.apache.spark.internal.Logging

object MesosProtoUtils extends Logging {

  
  def mesosLabels(labelsStr: String): Protos.Labels.Builder = {
    val labels: Seq[Protos.Label] = if (labelsStr == "") {
      Seq()
    } else {
      labelsStr.split("""(?<!\\),""").toSeq.map { labelStr =>
        val parts = labelStr.split("""(?<!\\):""")
        if (parts.length != 2) {
          throw new SparkException(s"Malformed label: ${labelStr}")
        }

        val cleanedParts = parts
          .map(part => part.replaceAll("""\\,""", ","))
          .map(part => part.replaceAll("""\\:""", ":"))

        Protos.Label.newBuilder()
          .setKey(cleanedParts(0))
          .setValue(cleanedParts(1))
          .build()
      }
    }

    Protos.Labels.newBuilder().addAllLabels(labels.asJava)
  }
} 

package org.apache.amaterasu.leader.mesos

import java.io.FileInputStream

import org.apache.amaterasu.common.configuration.ClusterConfig
import org.apache.amaterasu.common.logging.Logging
import org.apache.amaterasu.leader.Kami
import org.apache.amaterasu.leader.mesos.schedulers.ClusterScheduler
import org.apache.mesos.{MesosSchedulerDriver, Protos}

object Launcher extends App with Logging {

  println(
    """
       Apache
           (                      )
           )\        )      )   ( /(   (   (       )        (
          ((_)(     (     ( /(  )\()  ))\  )(   ( /(  (    ))\
         )\ _ )\    )\  ' )(_))(_))/ /((_)(()\  )(_)) )\  /((_)
         (_)_\(_) _((_)) ((_) _ | |_ (_))   ((_)((_)_ ((_)(_))(
          / _ \  | '   \()/ _` ||  _|/ -_) | '_|/ _` |(_-<| || |
         /_/ \_\ |_|_|_|  \__,_| \__|\___| |_|  \__,_|/__/ \_,_|

         Durable Dataflow Cluster
         Version 0.1.0
    """
  )

  val config = ClusterConfig(new FileInputStream("scripts/amaterasu.properties"))
  val kami = Kami(Seq("https://github.com/roadan/amaterasu-job-sample.git"))

  // for multi-tenancy reasons the name of the framework is composed out of the username ( which defaults
  // to empty string concatenated with - Amaterasu
  val framework = Protos.FrameworkInfo.newBuilder()
    .setName(s"${config.user} - Amaterasu")
    .setFailoverTimeout(config.timeout)
    .setUser(config.user).build()

  log.debug(s"The framework user is ${config.user}")
  val masterAddress = s"${config.master}:${config.masterPort}"
  val scheduler = ClusterScheduler(kami, config)
  val driver = new MesosSchedulerDriver(scheduler, framework, masterAddress)

  log.debug(s"Connecting to master on: $masterAddress")
  driver.run()

} 

package org.apache.amaterasu.leader.mesos

import java.io.FileInputStream
import java.nio.file.Paths

import org.apache.amaterasu.common.logging.Logging
import org.apache.amaterasu.common.configuration.ClusterConfig
import org.apache.amaterasu.leader.mesos.schedulers.JobScheduler
import org.apache.mesos.Protos.FrameworkID
import org.apache.mesos.{MesosSchedulerDriver, Protos}

case class Args(
                 repo: String = "",
                 branch: String = "master",
                 env: String = "default",
                 name: String = "amaterasu-job",
                 jobId: String = null,
                 report: String = "code",
                 home: String = ""
               )


object JobLauncher extends App with Logging {


  val parser = new scopt.OptionParser[Args]("amaterasu job") {
    head("amaterasu job", "0.2.0-incubating") //TODO: Get the version from the build

    opt[String]('r', "repo") action { (x, c) =>
      c.copy(repo = x)
    } text "The git repo containing the job"
    opt[String]('b', "branch") action { (x, c) =>
      c.copy(branch = x)
    } text "The branch to be executed (default is master)"
    opt[String]('e', "env") action { (x, c) =>
      c.copy(env = x)
    } text "The environment to be executed (test, prod, etc. values from the default env are taken if np env specified)"
    opt[String]('n', "name") action { (x, c) =>
      c.copy(name = x)
    } text "The name of the job"
    opt[String]('i', "job-id") action { (x, c) =>
      c.copy(jobId = x)
    } text "The jobId - should be passed only when resuming a job"
    opt[String]('r', "report") action { (x, c) =>
      c.copy(report = x)
    }
    opt[String]('h', "home") action { (x, c) =>
      c.copy(home = x)
    } text "The level of reporting"

  }


  parser.parse(args, Args()) match {

    case Some(arguments) =>

      val config = ClusterConfig(new FileInputStream(s"${arguments.home}/amaterasu.properties"))

      val frameworkBuilder = Protos.FrameworkInfo.newBuilder()
        .setName(s"${arguments.name} - Amaterasu Job")
        .setFailoverTimeout(config.timeout)
        .setUser(config.user)

      // TODO: test this
      val resume = arguments.jobId != null
      if (resume) {
        frameworkBuilder.setId(FrameworkID.newBuilder().setValue(arguments.jobId))
      }

      val framework = frameworkBuilder.build()

      val masterAddress = s"${config.master}:${config.masterPort}"

      val scheduler = JobScheduler(
        arguments.repo,
        arguments.branch,
        arguments.env,
        resume,
        config,
        arguments.report,
        arguments.home
      )

      val driver = new MesosSchedulerDriver(scheduler, framework, masterAddress)

      log.debug(s"Connecting to master on: $masterAddress")
      driver.run()

    case None =>
    // arguments are bad, error message will have been displayed
  }

} 

package mesosphere.marathon.example.plugin.env

import mesosphere.marathon.plugin.{ApplicationSpec, PodSpec}
import mesosphere.marathon.plugin.task._
import mesosphere.marathon.plugin.plugin.PluginConfiguration
import org.apache.mesos.Protos
import org.apache.mesos.Protos.{ExecutorInfo, TaskGroupInfo, TaskInfo}

class EnvVarExtenderPlugin extends RunSpecTaskProcessor with PluginConfiguration {
  private[env] var envVariables = Map.empty[String, String]

  def initialize(marathonInfo: Map[String, Any], configuration: play.api.libs.json.JsObject): Unit = {
    envVariables = (configuration \ "env").as[Map[String, String]]
  }

  override def taskInfo(appSpec: ApplicationSpec, builder: TaskInfo.Builder): Unit = {
    val envBuilder = builder.getCommand.getEnvironment.toBuilder
    envVariables.foreach {
      case (key, value) =>
        val envVariable = Protos.Environment.Variable.newBuilder()
        envVariable.setName(key)
        envVariable.setValue(value)
        envBuilder.addVariables(envVariable)
    }
    val commandBuilder = builder.getCommand.toBuilder
    commandBuilder.setEnvironment(envBuilder)
    builder.setCommand(commandBuilder)
  }

  override def taskGroup(podSpec: PodSpec, executor: ExecutorInfo.Builder, taskGroup: TaskGroupInfo.Builder): Unit = ???
} 

package mesosphere.marathon.example.plugin.env

import mesosphere.marathon.plugin.ApplicationSpec
import org.apache.mesos.Protos
import org.scalatest.{FlatSpec, Matchers}
import play.api.libs.json.{JsObject, Json}


class EnvVarExtenderPluginTest extends FlatSpec with Matchers {
  "Initialization with a configuration" should "work" in {
    val f = new Fixture
    f.envVarExtender.envVariables should be(Map("foo" -> "bar", "key" -> "value"))
  }

  "Applying the plugin" should "work" in {
    val f = new Fixture
    val runSpec: ApplicationSpec = null
    val builder = Protos.TaskInfo.newBuilder()
    f.envVarExtender.taskInfo(runSpec, builder)
    builder.getCommand.getEnvironment.getVariablesList.get(0).getName should be("foo")
  }

  class Fixture {
    val json =
      """{
        |    "env": {
        |        "foo": "bar",
        |        "key": "value"
        |    }
        |}
      """.stripMargin
    val config = Json.parse(json).as[JsObject]
    val envVarExtender = new EnvVarExtenderPlugin()
    envVarExtender.initialize(Map.empty, config)
  }
}