org.apache.spark.streaming.scheduler.StreamingListener Scala Examples

The following examples show how to use org.apache.spark.streaming.scheduler.StreamingListener. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.
Example 1
Source File: SubscriberListener.scala    From incubator-s2graph   with Apache License 2.0 5 votes vote down vote up 
package org.apache.s2graph.spark.spark

import org.apache.spark.Logging
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.scheduler.{StreamingListener, StreamingListenerReceiverError, StreamingListenerReceiverStarted, StreamingListenerReceiverStopped}

class SubscriberListener(ssc: StreamingContext) extends StreamingListener with Logging {
  override def onReceiverError(receiverError: StreamingListenerReceiverError): Unit = {
    logInfo("onReceiverError")
  }

  override def onReceiverStarted(receiverStarted: StreamingListenerReceiverStarted): Unit = {
    logInfo("onReceiverStarted")
  }

  override def onReceiverStopped(receiverStopped: StreamingListenerReceiverStopped): Unit = {
    logInfo("onReceiverStopped")
    ssc.stop()
  }
}
Example 2
Source File: StreamingContextUtils.scala    From sscheck   with Apache License 2.0 5 votes vote down vote up 
package es.ucm.fdi.sscheck.spark.streaming

import scala.concurrent._
import scala.concurrent.duration._
import scala.language.postfixOps
import ExecutionContext.Implicits.global
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.scheduler.{StreamingListener, StreamingListenerReceiverStarted, StreamingListenerBatchCompleted}

// Inline alternative implementation based on SyncVar
object StreamingContextUtils {
  
  def awaitForNBatchesCompleted(numBatches : Int, atMost : scala.concurrent.duration.Duration = 30 seconds)
                               (ssc : StreamingContext) : Unit = {
    val onBatchCompletedSyncVar = new SyncVar[Unit]
    ssc.addStreamingListener(new StreamingListener {
      override def onBatchCompleted(batchCompleted: StreamingListenerBatchCompleted) : Unit =  {  
        if (! onBatchCompletedSyncVar.isSet) {
          // note only this threads makes puts, so no problem with concurrency
          onBatchCompletedSyncVar.put(())
        }
      }
    })
    val waitingForBatches = Future {
      for (_ <- 1 to numBatches) {
        onBatchCompletedSyncVar.take()
      }  
    }
    Await.result(waitingForBatches, atMost)
  }
}
Example 3
Source File: SparkStream.scala    From infinispan-spark   with Apache License 2.0 5 votes vote down vote up 
package org.infinispan.spark.test

import java.lang.Thread._

import org.apache.spark.streaming.scheduler.{StreamingListener, StreamingListenerReceiverStarted}
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
import org.scalatest.{BeforeAndAfterEach, Suite}


trait SparkStream extends BeforeAndAfterEach {
   this: Suite with RemoteTest =>

   protected var sc: SparkContext = _
   protected var ssc: StreamingContext = _

   private lazy val config: SparkConf = new SparkConf().setMaster("local[8]").setAppName(this.getClass.getName).set("spark.driver.host","127.0.0.1")

   override protected def beforeEach(): Unit = {
      sc = new SparkContext(config)
      ssc = new StreamingContext(sc, Seconds(1))
      super.beforeEach()
   }

   override protected def afterEach(): Unit = {
      ssc.stop(stopSparkContext = true)
      sc.stop()
      super.afterEach()
   }

   protected def executeAfterReceiverStarted(block: => Unit) = {
      ssc.addStreamingListener(new StreamingListener {
         override def onReceiverStarted(receiverStarted: StreamingListenerReceiverStarted): Unit = {
            sleep(1000)
            block
         }
      })
   }

}