org.apache.spark.scheduler.LiveListenerBus Scala Examples
The following examples show how to use org.apache.spark.scheduler.LiveListenerBus.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
Example 1
Source File: StreamingQueryListenerBus.scala From drizzle-spark with Apache License 2.0 | 5 votes |
package org.apache.spark.sql.execution.streaming import org.apache.spark.scheduler.{LiveListenerBus, SparkListener, SparkListenerEvent} import org.apache.spark.sql.streaming.StreamingQueryListener import org.apache.spark.util.ListenerBus def post(event: StreamingQueryListener.Event) { event match { case s: QueryStartedEvent => postToAll(s) case _ => sparkListenerBus.post(event) } } override def onOtherEvent(event: SparkListenerEvent): Unit = { event match { case e: StreamingQueryListener.Event => postToAll(e) case _ => } } override protected def doPostEvent( listener: StreamingQueryListener, event: StreamingQueryListener.Event): Unit = { event match { case queryStarted: QueryStartedEvent => listener.onQueryStarted(queryStarted) case queryProgress: QueryProgressEvent => listener.onQueryProgress(queryProgress) case queryTerminated: QueryTerminatedEvent => listener.onQueryTerminated(queryTerminated) case _ => } } }
Example 2
Source File: OapRpcManagerMaster.scala From OAP with Apache License 2.0 | 5 votes |
package org.apache.spark.sql.oap.rpc import scala.collection.mutable import org.apache.spark.internal.Logging import org.apache.spark.rpc.{RpcCallContext, RpcEndpointRef, RpcEnv, ThreadSafeRpcEndpoint} import org.apache.spark.scheduler.LiveListenerBus import org.apache.spark.sql.oap.listener.SparkListenerCustomInfoUpdate import org.apache.spark.sql.oap.rpc.OapMessages._ private[spark] class OapRpcManagerMaster(oapRpcManagerMasterEndpoint: OapRpcManagerMasterEndpoint) extends OapRpcManager with Logging { private def sendOneWayMessageToExecutors(message: OapMessage): Unit = { oapRpcManagerMasterEndpoint.rpcEndpointRefByExecutor.foreach { case (_, slaveEndpoint) => slaveEndpoint.send(message) } } override private[spark] def send(message: OapMessage): Unit = { sendOneWayMessageToExecutors(message) } } private[spark] object OapRpcManagerMaster { val DRIVER_ENDPOINT_NAME = "OapRpcManagerMaster" } private[spark] class OapRpcManagerMasterEndpoint( override val rpcEnv: RpcEnv, listenerBus: LiveListenerBus) extends ThreadSafeRpcEndpoint with Logging { // Mapping from executor ID to RpcEndpointRef. private[rpc] val rpcEndpointRefByExecutor = new mutable.HashMap[String, RpcEndpointRef] override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = { case RegisterOapRpcManager(executorId, slaveEndpoint) => context.reply(handleRegistration(executorId, slaveEndpoint)) case _ => } override def receive: PartialFunction[Any, Unit] = { case heartbeat: Heartbeat => handleHeartbeat(heartbeat) case message: OapMessage => handleNormalOapMessage(message) case _ => } private def handleRegistration(executorId: String, ref: RpcEndpointRef): Boolean = { rpcEndpointRefByExecutor += ((executorId, ref)) true } private def handleNormalOapMessage(message: OapMessage) = message match { case _: Heartbeat => throw new IllegalArgumentException( "This is only to deal with non-heartbeat messages") case DummyMessage(id, someContent) => val c = this.getClass.getMethods logWarning(s"Dummy message received on Driver with id: $id, content: $someContent") case _ => } private def handleHeartbeat(heartbeat: Heartbeat) = heartbeat match { case FiberCacheHeartbeat(executorId, blockManagerId, content) => listenerBus.post(SparkListenerCustomInfoUpdate( blockManagerId.host, executorId, "OapFiberCacheHeartBeatMessager", content)) case FiberCacheMetricsHeartbeat(executorId, blockManagerId, content) => listenerBus.post(SparkListenerCustomInfoUpdate( blockManagerId.host, executorId, "FiberCacheManagerMessager", content)) case _ => } }
Example 3
Source File: StreamingQueryListenerBus.scala From XSQL with Apache License 2.0 | 4 votes |
package org.apache.spark.sql.execution.streaming import java.util.UUID import scala.collection.mutable import org.apache.spark.scheduler.{LiveListenerBus, SparkListener, SparkListenerEvent} import org.apache.spark.sql.streaming.StreamingQueryListener import org.apache.spark.util.ListenerBus override protected def doPostEvent( listener: StreamingQueryListener, event: StreamingQueryListener.Event): Unit = { def shouldReport(runId: UUID): Boolean = { activeQueryRunIds.synchronized { activeQueryRunIds.contains(runId) } } event match { case queryStarted: QueryStartedEvent => if (shouldReport(queryStarted.runId)) { listener.onQueryStarted(queryStarted) } case queryProgress: QueryProgressEvent => if (shouldReport(queryProgress.progress.runId)) { listener.onQueryProgress(queryProgress) } case queryTerminated: QueryTerminatedEvent => if (shouldReport(queryTerminated.runId)) { listener.onQueryTerminated(queryTerminated) } case _ => } } } object StreamingQueryListenerBus { val STREAM_EVENT_QUERY = "streams" }