java.net.Socket Scala Examples
The following examples show how to use java.net.Socket.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
Example 1
| Source File: LauncherBackend.scala From drizzle-spark with Apache License 2.0 | 6 votes |
|
package org.apache.spark.launcher
import java.net.{InetAddress, Socket}
import org.apache.spark.SPARK_VERSION
import org.apache.spark.launcher.LauncherProtocol._
import org.apache.spark.util.{ThreadUtils, Utils}
protected def onDisconnected() : Unit = { }
private def fireStopRequest(): Unit = {
val thread = LauncherBackend.threadFactory.newThread(new Runnable() {
override def run(): Unit = Utils.tryLogNonFatalError {
onStopRequest()
}
})
thread.start()
}
private class BackendConnection(s: Socket) extends LauncherConnection(s) {
override protected def handle(m: Message): Unit = m match {
case _: Stop =>
fireStopRequest()
case _ =>
throw new IllegalArgumentException(s"Unexpected message type: ${m.getClass().getName()}")
}
override def close(): Unit = {
try {
super.close()
} finally {
onDisconnected()
_isConnected = false
}
}
}
}
private object LauncherBackend {
val threadFactory = ThreadUtils.namedThreadFactory("LauncherBackend")
}
Example 2
| Source File: PythonGatewayServer.scala From BigDatalog with Apache License 2.0 | 5 votes |
|
package org.apache.spark.api.python
import java.io.DataOutputStream
import java.net.Socket
import py4j.GatewayServer
import org.apache.spark.Logging
import org.apache.spark.util.Utils
private[spark] object PythonGatewayServer extends Logging {
def main(args: Array[String]): Unit = Utils.tryOrExit {
// Start a GatewayServer on an ephemeral port
val gatewayServer: GatewayServer = new GatewayServer(null, 0)
gatewayServer.start()
val boundPort: Int = gatewayServer.getListeningPort
if (boundPort == -1) {
logError("GatewayServer failed to bind; exiting")
System.exit(1)
} else {
logDebug(s"Started PythonGatewayServer on port $boundPort")
}
// Communicate the bound port back to the caller via the caller-specified callback port
val callbackHost = sys.env("_PYSPARK_DRIVER_CALLBACK_HOST")
val callbackPort = sys.env("_PYSPARK_DRIVER_CALLBACK_PORT").toInt
logDebug(s"Communicating GatewayServer port to Python driver at $callbackHost:$callbackPort")
val callbackSocket = new Socket(callbackHost, callbackPort)
val dos = new DataOutputStream(callbackSocket.getOutputStream)
dos.writeInt(boundPort)
dos.close()
callbackSocket.close()
// Exit on EOF or broken pipe to ensure that this process dies when the Python driver dies:
while (System.in.read() != -1) {
// Do nothing
}
logDebug("Exiting due to broken pipe from Python driver")
System.exit(0)
}
}
Example 3
| Source File: PythonGatewayServer.scala From iolap with Apache License 2.0 | 5 votes |
|
package org.apache.spark.api.python
import java.io.DataOutputStream
import java.net.Socket
import py4j.GatewayServer
import org.apache.spark.Logging
import org.apache.spark.util.Utils
private[spark] object PythonGatewayServer extends Logging {
def main(args: Array[String]): Unit = Utils.tryOrExit {
// Start a GatewayServer on an ephemeral port
val gatewayServer: GatewayServer = new GatewayServer(null, 0)
gatewayServer.start()
val boundPort: Int = gatewayServer.getListeningPort
if (boundPort == -1) {
logError("GatewayServer failed to bind; exiting")
System.exit(1)
} else {
logDebug(s"Started PythonGatewayServer on port $boundPort")
}
// Communicate the bound port back to the caller via the caller-specified callback port
val callbackHost = sys.env("_PYSPARK_DRIVER_CALLBACK_HOST")
val callbackPort = sys.env("_PYSPARK_DRIVER_CALLBACK_PORT").toInt
logDebug(s"Communicating GatewayServer port to Python driver at $callbackHost:$callbackPort")
val callbackSocket = new Socket(callbackHost, callbackPort)
val dos = new DataOutputStream(callbackSocket.getOutputStream)
dos.writeInt(boundPort)
dos.close()
callbackSocket.close()
// Exit on EOF or broken pipe to ensure that this process dies when the Python driver dies:
while (System.in.read() != -1) {
// Do nothing
}
logDebug("Exiting due to broken pipe from Python driver")
System.exit(0)
}
}
Example 4
| Source File: CustomReceiver.scala From spark1.52 with Apache License 2.0 | 5 votes |
|
// scalastyle:off println
package org.apache.spark.examples.streaming
import java.io.{InputStreamReader, BufferedReader, InputStream}
import java.net.Socket
import org.apache.spark.{SparkConf, Logging}
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.receiver.Receiver
import java.io.File
import java.io.FileInputStream
private def receive() {
var socket: Socket = null
var userInput: String = null
try {
logInfo("Connecting to " + host + ":" + port)
socket = new Socket(host, port) //连接机器
logInfo("Connected to " + host + ":" + port)
//获取网络连接输入流
println("isConnected:"+socket.isConnected())
val socketInput=socket.getInputStream()
//
//val inputFile=new File("../data/mllib/als/testCustomReceiver.data")
// val in = new FileInputStream(inputFile)
// val in = new FileInputStream(socketInput)
val reader = new BufferedReader(new InputStreamReader(socketInput, "UTF-8"))
userInput = reader.readLine()
while(!isStopped && userInput != null) {
store(userInput)//存储数据
userInput = reader.readLine()//读取数据
println("userInput:"+userInput)
}
reader.close()//关闭流
socket.close()//关闭连接
logInfo("Stopped receiving")
restart("Trying to connect again")
} catch {
case e: java.net.ConnectException =>
restart("Error connecting to " + host + ":" + port, e)
case t: Throwable =>
restart("Error receiving data", t)
}
}
}
// scalastyle:on println
Example 5
| Source File: ImageInputDStream.scala From spark1.52 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.examples.streaming
import java.io.InputStream
import java.net.Socket
import org.apache.hadoop.io.BytesWritable
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.receiver.Receiver
import scala.collection.mutable.ArrayBuffer
import org.apache.spark.Logging
class ImageInputDStream(@transient ssc_ : StreamingContext, host: String, port:
Int, storageLevel: StorageLevel) extends
ReceiverInputDStream[BytesWritable](ssc_) with Logging {
override def getReceiver(): Receiver[BytesWritable] = {
new ImageRecevier(host, port, storageLevel)
}
}
class ImageRecevier(host: String, port: Int, storageLevel: StorageLevel) extends
Receiver[BytesWritable](storageLevel) with Logging {
override def onStart(): Unit = {
new Thread("Image Socket") {
setDaemon(true)
override def run(): Unit = {
receive()
}
}.start()
}
def receive(): Unit = {
var socket: Socket = null
var in: InputStream = null
try {
log.info("Connecting to " + host + ":" + port)
socket = new Socket(host, port)
log.info("Connected to " + host + ":" + port)
in = socket.getInputStream
val buf = new ArrayBuffer[Byte]()
var bytes = new Array[Byte](1024)
var len = 0
while (-1 < len) {
len = in.read(bytes)
if (len > 0) {
buf ++= bytes
}
}
val bw = new BytesWritable(buf.toArray)
log.error("byte:::::" + bw.getLength)
store(bw)
log.info("Stopped receiving")
restart("Retrying connecting to " + host + ":" + port)
} catch {
case e: java.net.ConnectException =>
restart("Error connecting to " + host + ":" + port, e)
case t: Throwable =>
restart("Error receiving data", t)
} finally {
if (in != null) {
in.close()
}
if (socket != null) {
socket.close()
log.info("Closed socket to " + host + ":" + port)
}
}
}
override def onStop(): Unit = {
}
}
Example 6
| Source File: SocketInputDStream.scala From spark1.52 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.dstream
import scala.util.control.NonFatal
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.storage.StorageLevel
import org.apache.spark.util.NextIterator
import scala.reflect.ClassTag
import java.io._
import java.net.{UnknownHostException, Socket}
import org.apache.spark.Logging
import org.apache.spark.streaming.receiver.Receiver
private[streaming]
class SocketInputDStream[T: ClassTag](
@transient ssc_ : StreamingContext,
host: String,
port: Int,
bytesToObjects: InputStream => Iterator[T],
storageLevel: StorageLevel
) extends ReceiverInputDStream[T](ssc_) {
def getReceiver(): Receiver[T] = {
new SocketReceiver(host, port, bytesToObjects, storageLevel)
}
}
private[streaming]
class SocketReceiver[T: ClassTag](
host: String,
port: Int,
bytesToObjects: InputStream => Iterator[T],
storageLevel: StorageLevel
) extends Receiver[T](storageLevel) with Logging {
def onStart() {
// Start the thread that receives data over a connection
//启动接收到连接上的数据的线程
new Thread("Socket Receiver") {
setDaemon(true)
override def run() { receive() }
}.start()
}
def onStop() {
// There is nothing much to do as the thread calling receive()
//没有什么可做的线程调用receive()
// is designed to stop by itself isStopped() returns false
//是为了阻止自己isstopped()返回false
}
def bytesToLines(inputStream: InputStream): Iterator[String] = {
val dataInputStream = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"))
new NextIterator[String] {
protected override def getNext() = {
val nextValue = dataInputStream.readLine()
if (nextValue == null) {
finished = true
}
nextValue
}
protected override def close() {
dataInputStream.close()
}
}
}
}
Example 7
| Source File: PythonGatewayServer.scala From spark1.52 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.api.python
import java.io.DataOutputStream
import java.net.Socket
import py4j.GatewayServer
import org.apache.spark.Logging
import org.apache.spark.util.Utils
private[spark] object PythonGatewayServer extends Logging {
def main(args: Array[String]): Unit = Utils.tryOrExit {
// Start a GatewayServer on an ephemeral port
val gatewayServer: GatewayServer = new GatewayServer(null, 0)
gatewayServer.start()
val boundPort: Int = gatewayServer.getListeningPort
if (boundPort == -1) {
logError("GatewayServer failed to bind; exiting")
System.exit(1)
} else {
logDebug(s"Started PythonGatewayServer on port $boundPort")
}
// Communicate the bound port back to the caller via the caller-specified callback port
//System.getenv()和System.getProperties()的区别
//System.getenv() 返回系统环境变量值 设置系统环境变量：当前登录用户主目录下的".bashrc"文件中可以设置系统环境变量
//System.getProperties() 返回Java进程变量值 通过命令行参数的"-D"选项
val callbackHost = sys.env("_PYSPARK_DRIVER_CALLBACK_HOST")
val callbackPort = sys.env("_PYSPARK_DRIVER_CALLBACK_PORT").toInt
logDebug(s"Communicating GatewayServer port to Python driver at $callbackHost:$callbackPort")
val callbackSocket = new Socket(callbackHost, callbackPort)
val dos = new DataOutputStream(callbackSocket.getOutputStream)
dos.writeInt(boundPort)
dos.close()
callbackSocket.close()
// Exit on EOF or broken pipe to ensure that this process dies when the Python driver dies:
while (System.in.read() != -1) {
// Do nothing
}
logDebug("Exiting due to broken pipe from Python driver")
System.exit(0)
}
}
Example 8
| Source File: CustomReceiver.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
// scalastyle:off println
package org.apache.spark.examples.streaming
import java.io.{BufferedReader, InputStreamReader}
import java.net.Socket
import java.nio.charset.StandardCharsets
import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.receiver.Receiver
private def receive() {
var socket: Socket = null
var userInput: String = null
try {
logInfo(s"Connecting to $host : $port")
socket = new Socket(host, port)
logInfo(s"Connected to $host : $port")
val reader = new BufferedReader(
new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))
userInput = reader.readLine()
while(!isStopped && userInput != null) {
store(userInput)
userInput = reader.readLine()
}
reader.close()
socket.close()
logInfo("Stopped receiving")
restart("Trying to connect again")
} catch {
case e: java.net.ConnectException =>
restart(s"Error connecting to $host : $port", e)
case t: Throwable =>
restart("Error receiving data", t)
}
}
}
// scalastyle:on println
Example 9
| Source File: SocketInputDStream.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.dstream
import java.io._
import java.net.{ConnectException, Socket}
import java.nio.charset.StandardCharsets
import scala.reflect.ClassTag
import scala.util.control.NonFatal
import org.apache.spark.internal.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.receiver.Receiver
import org.apache.spark.util.NextIterator
private[streaming]
class SocketInputDStream[T: ClassTag](
_ssc: StreamingContext,
host: String,
port: Int,
bytesToObjects: InputStream => Iterator[T],
storageLevel: StorageLevel
) extends ReceiverInputDStream[T](_ssc) {
def getReceiver(): Receiver[T] = {
new SocketReceiver(host, port, bytesToObjects, storageLevel)
}
}
private[streaming]
class SocketReceiver[T: ClassTag](
host: String,
port: Int,
bytesToObjects: InputStream => Iterator[T],
storageLevel: StorageLevel
) extends Receiver[T](storageLevel) with Logging {
private var socket: Socket = _
def onStart() {
logInfo(s"Connecting to $host:$port")
try {
socket = new Socket(host, port)
} catch {
case e: ConnectException =>
restart(s"Error connecting to $host:$port", e)
return
}
logInfo(s"Connected to $host:$port")
// Start the thread that receives data over a connection
new Thread("Socket Receiver") {
setDaemon(true)
override def run() { receive() }
}.start()
}
def onStop() {
// in case restart thread close it twice
synchronized {
if (socket != null) {
socket.close()
socket = null
logInfo(s"Closed socket to $host:$port")
}
}
}
def bytesToLines(inputStream: InputStream): Iterator[String] = {
val dataInputStream = new BufferedReader(
new InputStreamReader(inputStream, StandardCharsets.UTF_8))
new NextIterator[String] {
protected override def getNext() = {
val nextValue = dataInputStream.readLine()
if (nextValue == null) {
finished = true
}
nextValue
}
protected override def close() {
dataInputStream.close()
}
}
}
}
Example 10
| Source File: SocketAuthHelper.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.security
import java.io.{DataInputStream, DataOutputStream, InputStream}
import java.net.Socket
import java.nio.charset.StandardCharsets.UTF_8
import org.apache.spark.SparkConf
import org.apache.spark.network.util.JavaUtils
import org.apache.spark.util.Utils
def authToServer(s: Socket): Unit = {
writeUtf8(secret, s)
val reply = readUtf8(s)
if (reply != "ok") {
JavaUtils.closeQuietly(s)
throw new IllegalArgumentException("Authentication failed.")
}
}
protected def readUtf8(s: Socket): String = {
val din = new DataInputStream(s.getInputStream())
val len = din.readInt()
val bytes = new Array[Byte](len)
din.readFully(bytes)
new String(bytes, UTF_8)
}
protected def writeUtf8(str: String, s: Socket): Unit = {
val bytes = str.getBytes(UTF_8)
val dout = new DataOutputStream(s.getOutputStream())
dout.writeInt(bytes.length)
dout.write(bytes, 0, bytes.length)
dout.flush()
}
}
Example 11
| Source File: RAuthHelper.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.api.r
import java.io.{DataInputStream, DataOutputStream}
import java.net.Socket
import org.apache.spark.SparkConf
import org.apache.spark.security.SocketAuthHelper
private[spark] class RAuthHelper(conf: SparkConf) extends SocketAuthHelper(conf) {
override protected def readUtf8(s: Socket): String = {
SerDe.readString(new DataInputStream(s.getInputStream()))
}
override protected def writeUtf8(str: String, s: Socket): Unit = {
val out = s.getOutputStream()
SerDe.writeString(new DataOutputStream(out), str)
out.flush()
}
}
Example 12
| Source File: LauncherBackend.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.launcher
import java.net.{InetAddress, Socket}
import org.apache.spark.{SPARK_VERSION, SparkConf}
import org.apache.spark.launcher.LauncherProtocol._
import org.apache.spark.util.{ThreadUtils, Utils}
protected def onDisconnected() : Unit = { }
private def fireStopRequest(): Unit = {
val thread = LauncherBackend.threadFactory.newThread(new Runnable() {
override def run(): Unit = Utils.tryLogNonFatalError {
onStopRequest()
}
})
thread.start()
}
private class BackendConnection(s: Socket) extends LauncherConnection(s) {
override protected def handle(m: Message): Unit = m match {
case _: Stop =>
fireStopRequest()
case _ =>
throw new IllegalArgumentException(s"Unexpected message type: ${m.getClass().getName()}")
}
override def close(): Unit = {
try {
super.close()
} finally {
onDisconnected()
_isConnected = false
}
}
}
}
private object LauncherBackend {
val threadFactory = ThreadUtils.namedThreadFactory("LauncherBackend")
}
Example 13
| Source File: CustomReceiver.scala From BigDatalog with Apache License 2.0 | 5 votes |
|
// scalastyle:off println
package org.apache.spark.examples.streaming
import java.io.{InputStreamReader, BufferedReader, InputStream}
import java.net.Socket
import org.apache.spark.{SparkConf, Logging}
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.receiver.Receiver
private def receive() {
var socket: Socket = null
var userInput: String = null
try {
logInfo("Connecting to " + host + ":" + port)
socket = new Socket(host, port)
logInfo("Connected to " + host + ":" + port)
val reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), "UTF-8"))
userInput = reader.readLine()
while(!isStopped && userInput != null) {
store(userInput)
userInput = reader.readLine()
}
reader.close()
socket.close()
logInfo("Stopped receiving")
restart("Trying to connect again")
} catch {
case e: java.net.ConnectException =>
restart("Error connecting to " + host + ":" + port, e)
case t: Throwable =>
restart("Error receiving data", t)
}
}
}
// scalastyle:on println
Example 14
| Source File: SocketInputDStream.scala From BigDatalog with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.dstream
import scala.util.control.NonFatal
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.storage.StorageLevel
import org.apache.spark.util.NextIterator
import scala.reflect.ClassTag
import java.io._
import java.net.{UnknownHostException, Socket}
import org.apache.spark.Logging
import org.apache.spark.streaming.receiver.Receiver
private[streaming]
class SocketInputDStream[T: ClassTag](
ssc_ : StreamingContext,
host: String,
port: Int,
bytesToObjects: InputStream => Iterator[T],
storageLevel: StorageLevel
) extends ReceiverInputDStream[T](ssc_) {
def getReceiver(): Receiver[T] = {
new SocketReceiver(host, port, bytesToObjects, storageLevel)
}
}
private[streaming]
class SocketReceiver[T: ClassTag](
host: String,
port: Int,
bytesToObjects: InputStream => Iterator[T],
storageLevel: StorageLevel
) extends Receiver[T](storageLevel) with Logging {
def onStart() {
// Start the thread that receives data over a connection
new Thread("Socket Receiver") {
setDaemon(true)
override def run() { receive() }
}.start()
}
def onStop() {
// There is nothing much to do as the thread calling receive()
// is designed to stop by itself isStopped() returns false
}
def bytesToLines(inputStream: InputStream): Iterator[String] = {
val dataInputStream = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"))
new NextIterator[String] {
protected override def getNext() = {
val nextValue = dataInputStream.readLine()
if (nextValue == null) {
finished = true
}
nextValue
}
protected override def close() {
dataInputStream.close()
}
}
}
}
Example 15
| Source File: CustomReceiver.scala From iolap with Apache License 2.0 | 5 votes |
|
package org.apache.spark.examples.streaming
import java.io.{InputStreamReader, BufferedReader, InputStream}
import java.net.Socket
import org.apache.spark.{SparkConf, Logging}
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.receiver.Receiver
private def receive() {
var socket: Socket = null
var userInput: String = null
try {
logInfo("Connecting to " + host + ":" + port)
socket = new Socket(host, port)
logInfo("Connected to " + host + ":" + port)
val reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), "UTF-8"))
userInput = reader.readLine()
while(!isStopped && userInput != null) {
store(userInput)
userInput = reader.readLine()
}
reader.close()
socket.close()
logInfo("Stopped receiving")
restart("Trying to connect again")
} catch {
case e: java.net.ConnectException =>
restart("Error connecting to " + host + ":" + port, e)
case t: Throwable =>
restart("Error receiving data", t)
}
}
}
Example 16
| Source File: LauncherBackend.scala From BigDatalog with Apache License 2.0 | 5 votes |
|
package org.apache.spark.launcher
import java.net.{InetAddress, Socket}
import org.apache.spark.SPARK_VERSION
import org.apache.spark.launcher.LauncherProtocol._
import org.apache.spark.util.{ThreadUtils, Utils}
protected def onDisconnected() : Unit = { }
private def fireStopRequest(): Unit = {
val thread = LauncherBackend.threadFactory.newThread(new Runnable() {
override def run(): Unit = Utils.tryLogNonFatalError {
onStopRequest()
}
})
thread.start()
}
private class BackendConnection(s: Socket) extends LauncherConnection(s) {
override protected def handle(m: Message): Unit = m match {
case _: Stop =>
fireStopRequest()
case _ =>
throw new IllegalArgumentException(s"Unexpected message type: ${m.getClass().getName()}")
}
override def close(): Unit = {
try {
super.close()
} finally {
onDisconnected()
_isConnected = false
}
}
}
}
private object LauncherBackend {
val threadFactory = ThreadUtils.namedThreadFactory("LauncherBackend")
}
Example 17
| Source File: SocksProxyChecker.scala From ProxyCrawler with Apache License 2.0 | 5 votes |
|
package org.crowdcrawler.proxycrawler.checker
import java.net
import java.net.{InetSocketAddress, Socket, URI}
import java.nio.charset.StandardCharsets
import javax.net.ssl.{HostnameVerifier, SSLContext}
import org.apache.http.annotation.ThreadSafe
import org.apache.http.client.methods.HttpGet
import org.apache.http.client.protocol.HttpClientContext
import org.apache.http.config.RegistryBuilder
import org.apache.http.conn.socket.{ConnectionSocketFactory, PlainConnectionSocketFactory}
import org.apache.http.conn.ssl.{NoopHostnameVerifier, SSLConnectionSocketFactory}
import org.apache.http.impl.client.HttpClients
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager
import org.apache.http.protocol.HttpContext
import org.apache.http.util.EntityUtils
@ThreadSafe
private[checker] object SocksProxyChecker extends AbstractProxyChecker {
private class MyHttpConnectionSocketFactory extends PlainConnectionSocketFactory {
override def createSocket(context: HttpContext): Socket = {
val socksAddress = context.getAttribute("socks.address").asInstanceOf[InetSocketAddress]
val proxy = new net.Proxy(net.Proxy.Type.SOCKS, socksAddress)
new Socket(proxy)
}
}
private class MyHttpsConnectionSocketFactory(sslContext: SSLContext, verifier: HostnameVerifier)
extends SSLConnectionSocketFactory(sslContext) {
override def createSocket(context: HttpContext): Socket = {
val socksAddress = context.getAttribute("socks.address").asInstanceOf[InetSocketAddress]
val proxy = new net.Proxy(net.Proxy.Type.SOCKS, socksAddress)
new Socket(proxy)
}
}
private val CLIENT = {
val reg = RegistryBuilder.create[ConnectionSocketFactory]()
.register("http", new MyHttpConnectionSocketFactory())
.register("https",
new MyHttpsConnectionSocketFactory(HttpsProxyChecker.SSL_CONTEXT, NoopHostnameVerifier.INSTANCE))
.build()
val cm = new PoolingHttpClientConnectionManager(reg)
cm.setMaxTotal(AbstractProxyChecker.MAX_CONN)
HttpClients.custom().setConnectionManager(cm).disableRedirectHandling().build()
}
private val TARGET_URL = new URI("http://www.baidu.com")
def check(host: String, port: Int): (Int, Int) = {
val request = new HttpGet(TARGET_URL)
AbstractProxyChecker.configureRequest(request)
val httpContext = {
val socksAddress = new InetSocketAddress(host, port)
val context = HttpClientContext.create()
context.setAttribute("socks.address", socksAddress)
context
}
val response = CLIENT.execute(request, httpContext)
val statusCode = response.getStatusLine.getStatusCode
val html = EntityUtils.toString(response.getEntity, StandardCharsets.UTF_8)
if (statusCode == 200 && html.contains("<title>百度一下")) (statusCode, html.getBytes.length) else (statusCode, -1)
}
}
Example 18
| Source File: MetricImplicits.scala From Mastering-Spark-for-Data-Science with MIT License | 5 votes |
|
package io.gzet.timeseries.timely
import java.io.PrintStream
import java.net.Socket
import java.nio.charset.StandardCharsets
import io.gzet.timeseries.SimpleConfig
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.{Logging, Partitioner}
object MetricImplicits extends Logging with SimpleConfig {
def nonNegativeMod(x: Int, mod: Int): Int = {
val rawMod = x % mod
rawMod + (if (rawMod < 0) mod else 0)
}
class MetricPartitioner(partitions: Int) extends Partitioner {
require(partitions >= 0, s"Number of partitions ($partitions) cannot be negative.")
override def numPartitions: Int = partitions
override def getPartition(key: Any): Int = {
val k = key.asInstanceOf[MetricKey]
nonNegativeMod(k.metricName.hashCode, partitions)
}
}
implicit class Metrics(rdd: RDD[Metric]) {
val partitions = rdd.partitions.length
val partitioner = new MetricPartitioner(partitions)
def publish() = {
val sSortedMetricRDD = rdd filter { metric =>
metric.tags.nonEmpty
} map { metric =>
(MetricKey(metric.name, metric.time), metric)
} repartitionAndSortWithinPartitions partitioner
sSortedMetricRDD.values foreachPartition { it: Iterator[Metric] =>
val sock = new Socket(timelyHost, timelyPort)
val writer = new PrintStream(sock.getOutputStream, true, StandardCharsets.UTF_8.name)
it foreach { metric =>
writer.println(metric.toPut)
}
writer.flush()
}
}
}
implicit class MetricStream(stream: DStream[Metric]) {
def publish() = {
stream foreachRDD {
rdd => rdd.publish()
}
}
}
}
case class Metric(name: String, time: Long, value: Double, tags: Map[String, String], viz: Option[String] = None) {
def toPut = {
val vizMap = if(viz.isDefined) List("viz" -> viz.get) else List[(String, String)]()
val strTags = vizMap.union(tags.toList).map({ case (k, v) =>
s"$k=$v"
}).mkString(" ")
s"put $name $time $value $strTags"
}
}
case class MetricKey(metricName: String, metricTime: Long)
object MetricKey {
implicit def orderingByMetricDate[A <: MetricKey] : Ordering[A] = {
Ordering.by(fk => (fk.metricName, fk.metricTime))
}
}
Example 19
package skuber.api.security
import java.net.Socket
import javax.net.ssl._
import java.security.cert.X509Certificate
import java.security.SecureRandom
import skuber.api.client.{AuthInfo, CertAuth, Context, PathOrData}
object TLS {
// This trust manager supports the InsecureSkipTLSVerify flag in kubeconfig files -
// it always trusts the server i.e. skips verifying the server cert for a TLS connection
object InsecureSkipTLSVerifyTrustManager extends X509ExtendedTrustManager
{
def getAcceptedIssuers = Array.empty[X509Certificate]
def checkClientTrusted(certs: Array[X509Certificate], authType: String) : Unit = {}
def checkServerTrusted(certs: Array[X509Certificate], authType: String) : Unit = {}
def checkClientTrusted(certs: Array[X509Certificate], s: String, socket: Socket): Unit = {}
def checkClientTrusted(certs: Array[X509Certificate], s: String, sslEngine: SSLEngine): Unit = {}
def checkServerTrusted(certs: Array[X509Certificate], s: String, socket: Socket): Unit = {}
def checkServerTrusted(certs: Array[X509Certificate], s: String, sslEngine: SSLEngine): Unit = {}
}
val skipTLSTrustManagers = Array[TrustManager](InsecureSkipTLSVerifyTrustManager)
val HttpsPattern = "https:.*".r
val HttpPattern = "http:.*".r
def establishSSLContext(k8sContext: Context): Option[SSLContext] = {
k8sContext.cluster.server match {
case HttpPattern(_*) => None // not using SSL so return no context
case HttpsPattern(_*) => Some(buildSSLContext(k8sContext))
case _ => throw new Exception("Kubernetes cluster API server URL does not begin with either http or https : "
+ k8sContext.cluster.server)
}
}
private def buildSSLContext(k8sContext: Context): SSLContext = {
val sslContext = SSLContext.getInstance("TLS")
val skipTLSVerify = k8sContext.cluster.insecureSkipTLSVerify
val clusterCertConfig = k8sContext.cluster.certificateAuthority
val trustManagers = getTrustManagers(skipTLSVerify,clusterCertConfig)
val keyManagers = getKeyManagers(k8sContext.authInfo)
sslContext.init(keyManagers.orNull, trustManagers.orNull, new SecureRandom())
sslContext
}
private def getTrustManagers(skipTLSVerify: Boolean, serverCertConfig: Option[PathOrData]) : Option[Array[TrustManager]] =
if (skipTLSVerify)
Some(skipTLSTrustManagers)
else
serverCertConfig map { certPathOrData =>
val clusterServerCerts = SecurityHelper.getCertificates(certPathOrData)
val trustStore = SecurityHelper.createTrustStore(clusterServerCerts)
val tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
tmf.init(trustStore)
tmf.getTrustManagers
}
private def getKeyManagers(authInfo: AuthInfo) : Option[Array[KeyManager]] =
authInfo match {
case CertAuth(clientCert, clientKey, userName) =>
val certs = SecurityHelper.getCertificates(clientCert)
val key = SecurityHelper.getPrivateKey(clientKey)
val keyStore = SecurityHelper.createKeyStore(userName.getOrElse("skuber"), certs, key)
val kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm)
kmf.init(keyStore, "changeit".toCharArray)
Some(kmf.getKeyManagers)
case _ => None
}
}
Example 20
| Source File: TCPConnector.scala From scala-loci with Apache License 2.0 | 5 votes |
|
package loci
package communicator
package tcp
import java.net.Socket
import scala.util.{Failure, Success}
import scala.util.control.NonFatal
private class TCPConnector(
host: String, port: Int, properties: TCP.Properties)
extends Connector[TCP] {
protected def connect(connectionEstablished: Connected[TCP]): Unit = {
new Thread() {
override def run() =
try TCPHandler.handleConnection(
new Socket(host, port), properties, TCPConnector.this, { connection =>
connectionEstablished.set(Success(connection))
})
catch {
case NonFatal(exception) =>
connectionEstablished.set(Failure(exception))
}
}.start()
}
}
Example 21
| Source File: SchemaRegistryService.scala From kafka-testing with Apache License 2.0 | 5 votes |
|
package com.landoop.kafka.testing
import java.net.{Socket, SocketException}
import java.util.Properties
import com.typesafe.scalalogging.StrictLogging
import io.confluent.kafka.schemaregistry.avro.AvroCompatibilityLevel
import io.confluent.kafka.schemaregistry.client.rest.RestService
import io.confluent.kafka.schemaregistry.rest.{SchemaRegistryConfig, SchemaRegistryRestApplication}
import io.confluent.kafka.schemaregistry.storage.{SchemaRegistry, SchemaRegistryIdentity}
import org.eclipse.jetty.server.Server
class SchemaRegistryService(val port: Int,
val zookeeperConnection: String,
val kafkaTopic: String,
val avroCompatibilityLevel: AvroCompatibilityLevel,
val masterEligibility: Boolean) extends StrictLogging {
private val app = new SchemaRegistryRestApplication({
val prop = new Properties
prop.setProperty("port", port.asInstanceOf[Integer].toString)
prop.setProperty(SchemaRegistryConfig.KAFKASTORE_CONNECTION_URL_CONFIG, zookeeperConnection)
prop.put(SchemaRegistryConfig.KAFKASTORE_TOPIC_CONFIG, kafkaTopic)
prop.put(SchemaRegistryConfig.COMPATIBILITY_CONFIG, avroCompatibilityLevel.toString)
prop.put(SchemaRegistryConfig.MASTER_ELIGIBILITY, masterEligibility.asInstanceOf[AnyRef])
prop
})
val restServer = startServer(port)
var Endpoint: String = getEndpoint(restServer)
val restClient = new RestService(Endpoint)
def startServer(port: Int, retries: Int = 5): Option[Server] = {
var retry = retries > 0
var restServer: Option[Server] = None
if (retry) {
if (isPortInUse(port)) {
logger.info(s"Schema Registry Port $port is already in use")
Thread.sleep(2000)
startServer(port, retries - 1)
} else {
restServer = Some(app.createServer)
restServer.get.start()
}
}
restServer
}
def getEndpoint(restServer: Option[Server]): String = {
if (restServer.isDefined) {
val uri = restServer.get.getURI.toString
if (uri.endsWith("/")) {
uri.substring(0, uri.length - 1)
} else {
uri
}
} else ""
}
private def isPortInUse(port: Integer): Boolean = try {
new Socket("127.0.0.1", port).close()
true
}
catch {
case e: SocketException => false
}
def close() {
if (restServer.isDefined) {
restServer.get.stop()
restServer.get.join()
}
}
def isMaster: Boolean = app.schemaRegistry.isMaster
def setMaster(schemaRegistryIdentity: SchemaRegistryIdentity): Unit =
app.schemaRegistry.setMaster(schemaRegistryIdentity)
def myIdentity: SchemaRegistryIdentity = app.schemaRegistry.myIdentity
def masterIdentity: SchemaRegistryIdentity = app.schemaRegistry.masterIdentity
def schemaRegistry: SchemaRegistry = app.schemaRegistry
}
Example 22
| Source File: ToxCoreTestBase.scala From jvm-toxcore-c with GNU General Public License v3.0 | 5 votes |
|
package im.tox.tox4j
import java.io.IOException
import java.net.{ InetAddress, Socket }
import java.util.Random
import org.jetbrains.annotations.NotNull
import org.scalatest.Assertions
object ToxCoreTestBase extends Assertions {
private[tox4j] val nodeCandidates = Seq(
new DhtNode("tox.initramfs.io", "tox.initramfs.io", 33445, "3F0A45A268367C1BEA652F258C85F4A66DA76BCAA667A49E770BCC4917AB6A25"),
new DhtNode("tox.verdict.gg", null, 33445, "1C5293AEF2114717547B39DA8EA6F1E331E5E358B35F9B6B5F19317911C5F976")
)
@NotNull def randomBytes(length: Int): Array[Byte] = {
val array = new Array[Byte](length)
new Random().nextBytes(array)
array
}
@NotNull
def readablePublicKey(@NotNull id: Array[Byte]): String = {
val str = new StringBuilder
id foreach { c => str.append(f"$c%02X") }
str.toString()
}
@NotNull
def parsePublicKey(@NotNull id: String): Array[Byte] = {
val publicKey = new Array[Byte](id.length / 2)
publicKey.indices foreach { i =>
publicKey(i) =
((fromHexDigit(id.charAt(i * 2)) << 4) +
fromHexDigit(id.charAt(i * 2 + 1))).toByte
}
publicKey
}
private def fromHexDigit(c: Char): Byte = {
val digit =
if (false) { 0 }
else if ('0' to '9' contains c) { c - '0' }
else if ('A' to 'F' contains c) { c - 'A' + 10 }
else if ('a' to 'f' contains c) { c - 'a' + 10 }
else { throw new IllegalArgumentException(s"Non-hex digit character: $c") }
digit.toByte
}
@SuppressWarnings(Array("org.wartremover.warts.Equals"))
private def hasConnection(ip: String, port: Int): Option[String] = {
var socket: Socket = null
try {
socket = new Socket(InetAddress.getByName(ip), port)
if (socket.getInputStream == null) {
Some("Socket input stream is null")
} else {
None
}
} catch {
case e: IOException =>
Some(s"A network connection can't be established to $ip:$port: ${e.getMessage}")
} finally {
if (socket != null) {
socket.close()
}
}
}
def checkIPv4: Option[String] = {
hasConnection("8.8.8.8", 53)
}
def checkIPv6: Option[String] = {
hasConnection("2001:4860:4860::8888", 53)
}
protected[tox4j] def assumeIPv4(): Unit = {
assume(checkIPv4.isEmpty)
}
protected[tox4j] def assumeIPv6(): Unit = {
assume(checkIPv6.isEmpty)
}
}
Example 23
| Source File: DhtNodeSelector.scala From jvm-toxcore-c with GNU General Public License v3.0 | 5 votes |
|
package im.tox.tox4j
import java.io.IOException
import java.net.{ InetAddress, Socket }
import com.typesafe.scalalogging.Logger
import im.tox.tox4j.core.ToxCore
import im.tox.tox4j.impl.jni.ToxCoreImplFactory
import org.scalatest.Assertions
import org.slf4j.LoggerFactory
object DhtNodeSelector extends Assertions {
private val logger = Logger(LoggerFactory.getLogger(this.getClass))
private var selectedNode: Option[DhtNode] = Some(ToxCoreTestBase.nodeCandidates(0))
@SuppressWarnings(Array("org.wartremover.warts.Equals"))
private def tryConnect(node: DhtNode): Option[DhtNode] = {
var socket: Socket = null
try {
socket = new Socket(InetAddress.getByName(node.ipv4), node.udpPort.value)
assume(socket.getInputStream != null)
Some(node)
} catch {
case e: IOException =>
logger.info(s"TCP connection failed (${e.getMessage})")
None
} finally {
if (socket != null) {
socket.close()
}
}
}
private def tryBootstrap(
withTox: (Boolean, Boolean) => (ToxCore => Option[DhtNode]) => Option[DhtNode],
node: DhtNode,
udpEnabled: Boolean
): Option[DhtNode] = {
val protocol = if (udpEnabled) "UDP" else "TCP"
val port = if (udpEnabled) node.udpPort else node.tcpPort
logger.info(s"Trying to bootstrap with ${node.ipv4}:$port using $protocol")
withTox(true, udpEnabled) { tox =>
val status = new ConnectedListener
if (!udpEnabled) {
tox.addTcpRelay(node.ipv4, port, node.dhtId)
}
tox.bootstrap(node.ipv4, port, node.dhtId)
// Try bootstrapping for 10 seconds.
(0 to 10000 / tox.iterationInterval) find { _ =>
tox.iterate(status)(())
Thread.sleep(tox.iterationInterval)
status.isConnected
} match {
case Some(time) =>
logger.info(s"Bootstrapped successfully after ${time * tox.iterationInterval}ms using $protocol")
Some(node)
case None =>
logger.info(s"Unable to bootstrap with $protocol")
None
}
}
}
private def findNode(withTox: (Boolean, Boolean) => (ToxCore => Option[DhtNode]) => Option[DhtNode]): DhtNode = {
DhtNodeSelector.selectedNode match {
case Some(node) => node
case None =>
logger.info("Looking for a working bootstrap node")
DhtNodeSelector.selectedNode = ToxCoreTestBase.nodeCandidates find { node =>
logger.info(s"Trying to establish a TCP connection to ${node.ipv4}")
(for {
node <- tryConnect(node)
node <- tryBootstrap(withTox, node, udpEnabled = true)
node <- tryBootstrap(withTox, node, udpEnabled = false)
} yield node).isDefined
}
assume(DhtNodeSelector.selectedNode.nonEmpty, "No viable nodes for bootstrap found; cannot test")
DhtNodeSelector.selectedNode.get
}
}
def node: DhtNode = findNode(ToxCoreImplFactory.withToxUnit[Option[DhtNode]])
}
Example 24
| Source File: EmbeddedKafkaSpecSupport.scala From embedded-kafka-schema-registry with MIT License | 5 votes |
|
package net.manub.embeddedkafka.schemaregistry
import java.net.{InetAddress, Socket}
import net.manub.embeddedkafka.schemaregistry.EmbeddedKafkaSpecSupport.{
Available,
NotAvailable,
ServerStatus
}
import org.scalatest.Assertion
import org.scalatest.concurrent.{Eventually, IntegrationPatience}
import org.scalatest.matchers.should.Matchers
import org.scalatest.time.{Milliseconds, Seconds, Span}
import org.scalatest.wordspec.AnyWordSpecLike
import scala.util.{Failure, Success, Try}
trait EmbeddedKafkaSpecSupport
extends AnyWordSpecLike
with Matchers
with Eventually
with IntegrationPatience {
implicit val config: PatienceConfig =
PatienceConfig(Span(1, Seconds), Span(100, Milliseconds))
def expectedServerStatus(port: Int, expectedStatus: ServerStatus): Assertion =
eventually {
status(port) shouldBe expectedStatus
}
private def status(port: Int): ServerStatus = {
Try(new Socket(InetAddress.getByName("localhost"), port)) match {
case Failure(_) => NotAvailable
case Success(_) => Available
}
}
}
object EmbeddedKafkaSpecSupport {
sealed trait ServerStatus
case object Available extends ServerStatus
case object NotAvailable extends ServerStatus
}
Example 25
| Source File: CountStreamProducer.scala From Apache-Spark-2x-Machine-Learning-Cookbook with MIT License | 5 votes |
|
package spark.ml.cookbook.chapter13
import java.io.{BufferedOutputStream, PrintWriter}
import java.net.Socket
import java.net.ServerSocket
import scala.util.Random
class CountSreamThread(socket: Socket) extends Thread {
val villians = Array("Bane", "Thanos", "Loki", "Apocalypse", "Red Skull", "The Governor", "Sinestro", "Galactus",
"Doctor Doom", "Lex Luthor", "Joker", "Magneto", "Darth Vader")
override def run(): Unit = {
println("Connection accepted")
val out = new PrintWriter(new BufferedOutputStream(socket.getOutputStream()))
println("Producing Data")
while (true) {
out.println(villians(Random.nextInt(villians.size)))
Thread.sleep(10)
}
println("Done Producing")
}
}
object CountStreamProducer {
def main(args: Array[String]): Unit = {
val ss = new ServerSocket(9999)
while (true) {
println("Accepting Connection...")
new CountSreamThread(ss.accept()).start()
}
}
}
Example 26
| Source File: LauncherBackend.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.launcher
import java.net.{InetAddress, Socket}
import org.apache.spark.SPARK_VERSION
import org.apache.spark.launcher.LauncherProtocol._
import org.apache.spark.util.{ThreadUtils, Utils}
protected def onDisconnected() : Unit = { }
private def fireStopRequest(): Unit = {
val thread = LauncherBackend.threadFactory.newThread(new Runnable() {
override def run(): Unit = Utils.tryLogNonFatalError {
onStopRequest()
}
})
thread.start()
}
private class BackendConnection(s: Socket) extends LauncherConnection(s) {
override protected def handle(m: Message): Unit = m match {
case _: Stop =>
fireStopRequest()
case _ =>
throw new IllegalArgumentException(s"Unexpected message type: ${m.getClass().getName()}")
}
override def close(): Unit = {
try {
super.close()
} finally {
onDisconnected()
_isConnected = false
}
}
}
}
private object LauncherBackend {
val threadFactory = ThreadUtils.namedThreadFactory("LauncherBackend")
}
Example 27
| Source File: SinkFunctionExample.scala From examples-scala with Apache License 2.0 | 5 votes |
|
package io.github.streamingwithflink.chapter8
import java.io.PrintStream
import java.net.{InetAddress, Socket}
import io.github.streamingwithflink.util.{SensorReading, SensorSource, SensorTimeAssigner}
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction}
import org.apache.flink.streaming.api.scala._
/**
* Example program that writes sensor readings to a socket.
*
* NOTE: Before starting the program, you need to start a process that listens on a socket at localhost:9191.
* On Linux, you can do that with nc (netcat) with the following command:
*
* nc -l localhost 9191
*/
object SinkFunctionExample {
def main(args: Array[String]): Unit = {
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
// use event time for the application
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
// configure watermark interval
env.getConfig.setAutoWatermarkInterval(1000L)
// ingest sensor stream
val readings: DataStream[SensorReading] = env
// SensorSource generates random temperature readings
.addSource(new SensorSource)
// assign timestamps and watermarks which are required for event time
.assignTimestampsAndWatermarks(new SensorTimeAssigner)
// write the sensor readings to a socket
readings.addSink(new SimpleSocketSink("localhost", 9191))
// set parallelism to 1 because only one thread can write to a socket
.setParallelism(1)
env.execute()
}
}
/**
* Writes a stream of [[SensorReading]] to a socket.
*/
class SimpleSocketSink(val host: String, val port: Int)
extends RichSinkFunction[SensorReading] {
var socket: Socket = _
var writer: PrintStream = _
override def open(config: Configuration): Unit = {
// open socket and writer
socket = new Socket(InetAddress.getByName(host), port)
writer = new PrintStream(socket.getOutputStream)
}
override def invoke(
value: SensorReading,
ctx: SinkFunction.Context[_]): Unit = {
// write sensor reading to socket
writer.println(value.toString)
writer.flush()
}
override def close(): Unit = {
// close writer and socket
writer.close()
socket.close()
}
}
Example 28
| Source File: CustomReceiver.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
// scalastyle:off println
package org.apache.spark.examples.streaming
import java.io.{BufferedReader, InputStreamReader}
import java.net.Socket
import java.nio.charset.StandardCharsets
import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.receiver.Receiver
private def receive() {
var socket: Socket = null
var userInput: String = null
try {
logInfo("Connecting to " + host + ":" + port)
socket = new Socket(host, port)
logInfo("Connected to " + host + ":" + port)
val reader = new BufferedReader(
new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))
userInput = reader.readLine()
while(!isStopped && userInput != null) {
store(userInput)
userInput = reader.readLine()
}
reader.close()
socket.close()
logInfo("Stopped receiving")
restart("Trying to connect again")
} catch {
case e: java.net.ConnectException =>
restart("Error connecting to " + host + ":" + port, e)
case t: Throwable =>
restart("Error receiving data", t)
}
}
}
// scalastyle:on println
Example 29
| Source File: SocketInputDStream.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.dstream
import java.io._
import java.net.{ConnectException, Socket}
import java.nio.charset.StandardCharsets
import scala.reflect.ClassTag
import scala.util.control.NonFatal
import org.apache.spark.internal.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.receiver.Receiver
import org.apache.spark.util.NextIterator
private[streaming]
class SocketInputDStream[T: ClassTag](
_ssc: StreamingContext,
host: String,
port: Int,
bytesToObjects: InputStream => Iterator[T],
storageLevel: StorageLevel
) extends ReceiverInputDStream[T](_ssc) {
def getReceiver(): Receiver[T] = {
new SocketReceiver(host, port, bytesToObjects, storageLevel)
}
}
private[streaming]
class SocketReceiver[T: ClassTag](
host: String,
port: Int,
bytesToObjects: InputStream => Iterator[T],
storageLevel: StorageLevel
) extends Receiver[T](storageLevel) with Logging {
private var socket: Socket = _
def onStart() {
logInfo(s"Connecting to $host:$port")
try {
socket = new Socket(host, port)
} catch {
case e: ConnectException =>
restart(s"Error connecting to $host:$port", e)
return
}
logInfo(s"Connected to $host:$port")
// Start the thread that receives data over a connection
new Thread("Socket Receiver") {
setDaemon(true)
override def run() { receive() }
}.start()
}
def onStop() {
// in case restart thread close it twice
synchronized {
if (socket != null) {
socket.close()
socket = null
logInfo(s"Closed socket to $host:$port")
}
}
}
def bytesToLines(inputStream: InputStream): Iterator[String] = {
val dataInputStream = new BufferedReader(
new InputStreamReader(inputStream, StandardCharsets.UTF_8))
new NextIterator[String] {
protected override def getNext() = {
val nextValue = dataInputStream.readLine()
if (nextValue == null) {
finished = true
}
nextValue
}
protected override def close() {
dataInputStream.close()
}
}
}
}
Example 30
| Source File: PythonGatewayServer.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.api.python
import java.io.DataOutputStream
import java.net.Socket
import py4j.GatewayServer
import org.apache.spark.internal.Logging
import org.apache.spark.util.Utils
private[spark] object PythonGatewayServer extends Logging {
initializeLogIfNecessary(true)
def main(args: Array[String]): Unit = Utils.tryOrExit {
// Start a GatewayServer on an ephemeral port
val gatewayServer: GatewayServer = new GatewayServer(null, 0)
gatewayServer.start()
val boundPort: Int = gatewayServer.getListeningPort
if (boundPort == -1) {
logError("GatewayServer failed to bind; exiting")
System.exit(1)
} else {
logDebug(s"Started PythonGatewayServer on port $boundPort")
}
// Communicate the bound port back to the caller via the caller-specified callback port
val callbackHost = sys.env("_PYSPARK_DRIVER_CALLBACK_HOST")
val callbackPort = sys.env("_PYSPARK_DRIVER_CALLBACK_PORT").toInt
logDebug(s"Communicating GatewayServer port to Python driver at $callbackHost:$callbackPort")
val callbackSocket = new Socket(callbackHost, callbackPort)
val dos = new DataOutputStream(callbackSocket.getOutputStream)
dos.writeInt(boundPort)
dos.close()
callbackSocket.close()
// Exit on EOF or broken pipe to ensure that this process dies when the Python driver dies:
while (System.in.read() != -1) {
// Do nothing
}
logDebug("Exiting due to broken pipe from Python driver")
System.exit(0)
}
}
Example 31
| Source File: CustomReceiver.scala From Learning-Spark-SQL with MIT License | 5 votes |
|
import java.io.{BufferedReader, InputStreamReader}
import java.net.Socket
import java.nio.charset.StandardCharsets
import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.receiver.Receiver
private def receive() {
var socket: Socket = null
var userInput: String = null
try {
println("Connecting to " + host + ":" + port)
socket = new Socket(host, port)
println("Connected to " + host + ":" + port)
val reader = new BufferedReader(
new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))
userInput = reader.readLine()
while(!isStopped && userInput != null) {
store(userInput)
userInput = reader.readLine()
}
reader.close()
socket.close()
println("Stopped receiving")
restart("Trying to connect again")
} catch {
case e: java.net.ConnectException =>
restart("Error connecting to " + host + ":" + port, e)
case t: Throwable =>
restart("Error receiving data", t)
}
}
}
Example 32
| Source File: TestZooKeeper.scala From mango with Apache License 2.0 | 5 votes |
|
package com.kakao.mango.zk
import java.io.{File, IOException}
import java.net.{ServerSocket, Socket}
import java.util.concurrent.TimeUnit
import com.kakao.mango.concurrent.NamedExecutors
import com.kakao.mango.logging.{LogLevelOverrider, Logging}
import com.kakao.shaded.guava.io.Files
import org.apache.zookeeper.server.persistence.FileTxnSnapLog
import org.apache.zookeeper.server.{ServerCnxnFactory, ServerConfig, ZooKeeperServer}
import org.scalatest.{BeforeAndAfterAll, Suite}
trait TestZooKeeper extends BeforeAndAfterAll with Logging { this: Suite =>
val zkServerPort = 2181
val zkServerExecutor = NamedExecutors.single("zookeeper-server")
var zk: ZooKeeperConnection = _
override protected def beforeAll(): Unit = {
logger.info("Launching a standalone ZooKeeper server for testing...")
try {
val socket = new ServerSocket(zkServerPort)
socket.close()
} catch {
case e: IOException =>
throw new RuntimeException(s"TCP port $zkServerPort is required for tests but not available")
}
zkServerExecutor.submit {
LogLevelOverrider.error("org.apache.zookeeper")
val datadir = Files.createTempDir().getAbsolutePath
val config = new ServerConfig
config.parse(Array(zkServerPort.toString, datadir))
val zkServer = new ZooKeeperServer
zkServer.setTxnLogFactory(new FileTxnSnapLog(new File(datadir), new File(datadir)))
zkServer.setTickTime(6000)
zkServer.setMinSessionTimeout(6000)
zkServer.setMaxSessionTimeout(6000)
val cnxnFactory = ServerCnxnFactory.createFactory
try {
cnxnFactory.configure(config.getClientPortAddress, 60)
cnxnFactory.startup(zkServer)
cnxnFactory.join()
} catch {
case _: InterruptedException =>
logger.info("ZooKeeper server interrupted; shutting down...")
cnxnFactory.shutdown()
cnxnFactory.join()
if (zkServer.isRunning) {
zkServer.shutdown()
}
logger.info("ZooKeeper server stopped")
}
}
var connected = false
while (!connected) {
logger.info("Waiting for ZooKeeper server to launch...")
try {
val socket = new Socket("localhost", zkServerPort)
logger.info("ZooKeeper server is available")
socket.close()
zk = ZooKeeperConnection(s"localhost:$zkServerPort")
connected = true
} catch {
case _: IOException => Thread.sleep(1000) // retry
}
}
super.beforeAll()
}
override protected def afterAll(): Unit = {
try super.afterAll()
finally {
zk.close()
logger.info("Interrupting ZooKeeper server...")
zkServerExecutor.shutdownNow()
while (!zkServerExecutor.awaitTermination(1, TimeUnit.SECONDS)) {
logger.info("awaiting ZooKeeper server termination...")
}
logger.info("ZooKeeper server terminated")
}
}
}
Example 33
| Source File: EmbeddedKafkaSpecSupport.scala From embedded-kafka with MIT License | 5 votes |
|
package net.manub.embeddedkafka
import java.net.{InetAddress, Socket}
import net.manub.embeddedkafka.EmbeddedKafkaSpecSupport.{
Available,
NotAvailable,
ServerStatus
}
import org.scalatest.Assertion
import org.scalatest.concurrent.{Eventually, IntegrationPatience}
import org.scalatest.matchers.should.Matchers
import org.scalatest.time.{Milliseconds, Seconds, Span}
import org.scalatest.wordspec.AnyWordSpecLike
import scala.util.{Failure, Success, Try}
trait EmbeddedKafkaSpecSupport
extends AnyWordSpecLike
with Matchers
with Eventually
with IntegrationPatience {
implicit val config: PatienceConfig =
PatienceConfig(Span(1, Seconds), Span(100, Milliseconds))
def expectedServerStatus(port: Int, expectedStatus: ServerStatus): Assertion =
eventually {
status(port) shouldBe expectedStatus
}
private def status(port: Int): ServerStatus = {
Try(new Socket(InetAddress.getByName("localhost"), port)) match {
case Failure(_) => NotAvailable
case Success(_) => Available
}
}
}
object EmbeddedKafkaSpecSupport {
sealed trait ServerStatus
case object Available extends ServerStatus
case object NotAvailable extends ServerStatus
}
Example 34
| Source File: ResourceManagerLinkHelper.scala From sparta with Apache License 2.0 | 5 votes |
|
package com.stratio.sparta.serving.core.helpers
import java.net.Socket
import scala.IllegalArgumentException
import scala.util._
import akka.event.slf4j.SLF4JLogging
import com.stratio.sparta.serving.core.config.SpartaConfig
import com.stratio.sparta.serving.core.constants.AppConstant
object ResourceManagerLinkHelper extends SLF4JLogging {
def getLink(executionMode : String, monitoringLink: Option[String] = None): Option[String] = {
val (host: String, port: Int) = (monitoringLink, executionMode) match {
case (None, AppConstant.ConfigMesos) | (None, AppConstant.ConfigMarathon) => mesosLink
case (None, AppConstant.ConfigLocal) => localLink
case (Some(uri), _) => userLink(uri)
case _ => throw new IllegalArgumentException(s"Wrong value in property executionMode: $executionMode")
}
checkConnectivity(host, port)
}
def checkConnectivity(host: String, port: Int, monitoringLink: Option[String] = None): Option[String] = {
Try {
new Socket(host, port)
} match {
case Success(socket) =>
if (socket.isConnected) {
socket.close()
monitoringLink.orElse(Option(s"http://$host:$port"))
} else {
log.debug(s"Cannot connect to http://$host:$port")
socket.close()
monitoringLink
}
case Failure(_) =>
log.debug(s"Cannot connect to http://$host:$port")
monitoringLink
}
}
private def mesosLink = {
val mesosDispatcherUrl = SpartaConfig.getClusterConfig().get.getString(AppConstant.MesosMasterDispatchers)
val host = mesosDispatcherUrl.replace("mesos://", "").replaceAll(":\\d+", "")
val port = 5050
(host, port)
}
private def localLink = {
val localhostName = java.net.InetAddress.getLocalHost.getHostName
(localhostName, 4040)
}
private def userLink(uri: String) = {
val host = uri.replace("http://", "").replace("https://", "").replaceAll(":\\d+", "")
val port = uri.split(":").lastOption.getOrElse("4040").toInt
(host, port)
}
}
Example 35
| Source File: CustomReceiver.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
// scalastyle:off println
package org.apache.spark.examples.streaming
import java.io.{BufferedReader, InputStreamReader}
import java.net.Socket
import java.nio.charset.StandardCharsets
import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.receiver.Receiver
private def receive() {
var socket: Socket = null
var userInput: String = null
try {
logInfo("Connecting to " + host + ":" + port)
socket = new Socket(host, port)
logInfo("Connected to " + host + ":" + port)
val reader = new BufferedReader(
new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))
userInput = reader.readLine()
while(!isStopped && userInput != null) {
store(userInput)
userInput = reader.readLine()
}
reader.close()
socket.close()
logInfo("Stopped receiving")
restart("Trying to connect again")
} catch {
case e: java.net.ConnectException =>
restart("Error connecting to " + host + ":" + port, e)
case t: Throwable =>
restart("Error receiving data", t)
}
}
}
// scalastyle:on println
Example 36
| Source File: SocketInputDStream.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.dstream
import java.io._
import java.net.{ConnectException, Socket}
import java.nio.charset.StandardCharsets
import scala.reflect.ClassTag
import scala.util.control.NonFatal
import org.apache.spark.internal.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.receiver.Receiver
import org.apache.spark.util.NextIterator
private[streaming]
class SocketInputDStream[T: ClassTag](
_ssc: StreamingContext,
host: String,
port: Int,
bytesToObjects: InputStream => Iterator[T],
storageLevel: StorageLevel
) extends ReceiverInputDStream[T](_ssc) {
def getReceiver(): Receiver[T] = {
new SocketReceiver(host, port, bytesToObjects, storageLevel)
}
}
private[streaming]
class SocketReceiver[T: ClassTag](
host: String,
port: Int,
bytesToObjects: InputStream => Iterator[T],
storageLevel: StorageLevel
) extends Receiver[T](storageLevel) with Logging {
private var socket: Socket = _
def onStart() {
logInfo(s"Connecting to $host:$port")
try {
socket = new Socket(host, port)
} catch {
case e: ConnectException =>
restart(s"Error connecting to $host:$port", e)
return
}
logInfo(s"Connected to $host:$port")
// Start the thread that receives data over a connection
new Thread("Socket Receiver") {
setDaemon(true)
override def run() { receive() }
}.start()
}
def onStop() {
// in case restart thread close it twice
synchronized {
if (socket != null) {
socket.close()
socket = null
logInfo(s"Closed socket to $host:$port")
}
}
}
def bytesToLines(inputStream: InputStream): Iterator[String] = {
val dataInputStream = new BufferedReader(
new InputStreamReader(inputStream, StandardCharsets.UTF_8))
new NextIterator[String] {
protected override def getNext() = {
val nextValue = dataInputStream.readLine()
if (nextValue == null) {
finished = true
}
nextValue
}
protected override def close() {
dataInputStream.close()
}
}
}
}
Example 37
| Source File: PythonGatewayServer.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.api.python
import java.io.DataOutputStream
import java.net.Socket
import py4j.GatewayServer
import org.apache.spark.internal.Logging
import org.apache.spark.util.Utils
private[spark] object PythonGatewayServer extends Logging {
initializeLogIfNecessary(true)
def main(args: Array[String]): Unit = Utils.tryOrExit {
// Start a GatewayServer on an ephemeral port
val gatewayServer: GatewayServer = new GatewayServer(null, 0)
gatewayServer.start()
val boundPort: Int = gatewayServer.getListeningPort
if (boundPort == -1) {
logError("GatewayServer failed to bind; exiting")
System.exit(1)
} else {
logDebug(s"Started PythonGatewayServer on port $boundPort")
}
// Communicate the bound port back to the caller via the caller-specified callback port
val callbackHost = sys.env("_PYSPARK_DRIVER_CALLBACK_HOST")
val callbackPort = sys.env("_PYSPARK_DRIVER_CALLBACK_PORT").toInt
logDebug(s"Communicating GatewayServer port to Python driver at $callbackHost:$callbackPort")
val callbackSocket = new Socket(callbackHost, callbackPort)
val dos = new DataOutputStream(callbackSocket.getOutputStream)
dos.writeInt(boundPort)
dos.close()
callbackSocket.close()
// Exit on EOF or broken pipe to ensure that this process dies when the Python driver dies:
while (System.in.read() != -1) {
// Do nothing
}
logDebug("Exiting due to broken pipe from Python driver")
System.exit(0)
}
}
Example 38
| Source File: SocketInputDStream.scala From iolap with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.dstream
import scala.util.control.NonFatal
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.storage.StorageLevel
import org.apache.spark.util.NextIterator
import scala.reflect.ClassTag
import java.io._
import java.net.{UnknownHostException, Socket}
import org.apache.spark.Logging
import org.apache.spark.streaming.receiver.Receiver
private[streaming]
class SocketInputDStream[T: ClassTag](
@transient ssc_ : StreamingContext,
host: String,
port: Int,
bytesToObjects: InputStream => Iterator[T],
storageLevel: StorageLevel
) extends ReceiverInputDStream[T](ssc_) {
def getReceiver(): Receiver[T] = {
new SocketReceiver(host, port, bytesToObjects, storageLevel)
}
}
private[streaming]
class SocketReceiver[T: ClassTag](
host: String,
port: Int,
bytesToObjects: InputStream => Iterator[T],
storageLevel: StorageLevel
) extends Receiver[T](storageLevel) with Logging {
def onStart() {
// Start the thread that receives data over a connection
new Thread("Socket Receiver") {
setDaemon(true)
override def run() { receive() }
}.start()
}
def onStop() {
// There is nothing much to do as the thread calling receive()
// is designed to stop by itself isStopped() returns false
}
def bytesToLines(inputStream: InputStream): Iterator[String] = {
val dataInputStream = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"))
new NextIterator[String] {
protected override def getNext() = {
val nextValue = dataInputStream.readLine()
if (nextValue == null) {
finished = true
}
nextValue
}
protected override def close() {
dataInputStream.close()
}
}
}
}
Example 39
| Source File: PythonGatewayServer.scala From SparkCore with Apache License 2.0 | 5 votes |
|
package org.apache.spark.api.python
import java.io.DataOutputStream
import java.net.Socket
import py4j.GatewayServer
import org.apache.spark.Logging
import org.apache.spark.util.Utils
private[spark] object PythonGatewayServer extends Logging {
def main(args: Array[String]): Unit = Utils.tryOrExit {
// Start a GatewayServer on an ephemeral port
val gatewayServer: GatewayServer = new GatewayServer(null, 0)
gatewayServer.start()
val boundPort: Int = gatewayServer.getListeningPort
if (boundPort == -1) {
logError("GatewayServer failed to bind; exiting")
System.exit(1)
} else {
logDebug(s"Started PythonGatewayServer on port $boundPort")
}
// Communicate the bound port back to the caller via the caller-specified callback port
val callbackHost = sys.env("_PYSPARK_DRIVER_CALLBACK_HOST")
val callbackPort = sys.env("_PYSPARK_DRIVER_CALLBACK_PORT").toInt
logDebug(s"Communicating GatewayServer port to Python driver at $callbackHost:$callbackPort")
val callbackSocket = new Socket(callbackHost, callbackPort)
val dos = new DataOutputStream(callbackSocket.getOutputStream)
dos.writeInt(boundPort)
dos.close()
callbackSocket.close()
// Exit on EOF or broken pipe to ensure that this process dies when the Python driver dies:
while (System.in.read() != -1) {
// Do nothing
}
logDebug("Exiting due to broken pipe from Python driver")
System.exit(0)
}
}
Example 40
| Source File: ProcessOverSocketStreamConnectionProvider.scala From intellij-lsp with Apache License 2.0 | 5 votes |
|
package com.github.gtache.lsp.client.connection
import java.io.{IOException, InputStream, OutputStream}
import java.net.{ServerSocket, Socket}
import java.util.Objects
import com.intellij.openapi.diagnostic.Logger
class ProcessOverSocketStreamConnectionProvider(commands: Seq[String], workingDir: String, port: Int = 0) extends ProcessStreamConnectionProvider(commands, workingDir) {
import ProcessOverSocketStreamConnectionProvider._
private var socket: Socket = _
private var inputStream: InputStream = _
private var outputStream: OutputStream = _
@throws[IOException]
override def start(): Unit = {
val serverSocket = new ServerSocket(port)
val socketThread = new Thread(() => {
try
socket = serverSocket.accept
catch {
case e: IOException =>
LOG.error(e)
} finally try
serverSocket.close()
catch {
case e: IOException =>
LOG.error(e)
}
})
socketThread.start()
super.start()
try {
socketThread.join(5000)
}
catch {
case e: InterruptedException =>
LOG.error(e)
}
if (socket == null) throw new IOException("Unable to make socket connection: " + toString) //$NON-NLS-1$
inputStream = socket.getInputStream
outputStream = socket.getOutputStream
}
override def getInputStream: InputStream = inputStream
override def getOutputStream: OutputStream = outputStream
override def getErrorStream: InputStream = inputStream
override def stop(): Unit = {
super.stop()
if (socket != null) try
socket.close()
catch {
case e: IOException =>
LOG.error(e)
}
}
override def hashCode: Int = {
val result = super.hashCode
result ^ Objects.hashCode(this.port)
}
}
object ProcessOverSocketStreamConnectionProvider {
private val LOG = Logger.getInstance(classOf[ProcessOverSocketStreamConnectionProvider])
}
Example 41
| Source File: RconConnector.scala From chatoverflow with Eclipse Public License 2.0 | 5 votes |
|
package org.codeoverflow.chatoverflow.requirement.service.rcon
import java.io.{DataInputStream, IOException, InputStream, OutputStream}
import java.net.{Socket, SocketException}
import java.nio.{ByteBuffer, ByteOrder}
import java.util.Random
import org.codeoverflow.chatoverflow.WithLogger
import org.codeoverflow.chatoverflow.connector.Connector
class RconConnector(override val sourceIdentifier: String) extends Connector(sourceIdentifier) with WithLogger {
override protected var requiredCredentialKeys: List[String] = List("password", "address")
override protected var optionalCredentialKeys: List[String] = List("port")
private var socket: Socket = _
private var outputStream: OutputStream = _
private var inputStream: InputStream = _
private var requestId: Int = 0
def sendCommand(command: String): String = {
logger debug s"Sending $command to RCON"
requestId += 1
if (write(2, command.getBytes("ASCII"))) {
return read()
}
null
}
override def stop(): Boolean = {
logger info s"Stopped RCON connector to ${credentials.get.getValue("address").get}!"
socket.close()
true
}
}
Example 42
| Source File: WebappTestSupports.scala From pizza-auth-3 with MIT License | 5 votes |
|
package moe.pizza.auth.webapp
import java.net.{Socket, InetSocketAddress, ServerSocket}
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import moe.pizza.auth.config.ConfigFile.ConfigFile
import scala.concurrent.{Future, Await}
import scala.io.Source
import scala.util.Try
import scala.concurrent.duration._
import scala.concurrent.ExecutionContext.Implicits.global
object WebappTestSupports {
val OM = new ObjectMapper(new YAMLFactory())
OM.registerModule(DefaultScalaModule)
def readTestConfig(): ConfigFile = {
val config = Source
.fromURL(getClass.getResource("/config.yml"))
.getLines()
.mkString("\n")
val conf = OM.readValue[ConfigFile](config, classOf[ConfigFile])
conf
}
}
Example 43
| Source File: StartDynamoDBLocal.scala From sbt-dynamodb with MIT License | 5 votes |
|
package com.localytics.sbt.dynamodb
import java.net.Socket
import sbt.File
import sbt.Keys._
import scala.sys.process._
import scala.util.Try
object StartDynamoDBLocal {
private[dynamodb] def isDynamoDBLocalRunning(port: Int): Boolean = Try(new Socket("localhost", port).close()).isSuccess
def apply(baseDir: File, port: Int, heapSize: Option[Int], dbPath: Option[String], inMem: Boolean, shared: Boolean, streamz: TaskStreams): String = {
val jar = new File(baseDir, "DynamoDBLocal.jar")
val lib = new File(baseDir, "DynamoDBLocal_lib")
val args = Seq("java", s"-Djava.library.path=${lib.getAbsolutePath}") ++
heapSize.map(mb => Seq(s"-Xms${mb}m", s"-Xmx${mb}m")).getOrElse(Nil) ++
Seq("-jar", jar.getAbsolutePath) ++
Seq("-port", port.toString) ++
dbPath.map(db => Seq("-dbPath", db)).getOrElse(Nil) ++
(if (inMem) Seq("-inMemory") else Nil) ++
(if (shared) Seq("-sharedDb") else Nil)
if (isDynamoDBLocalRunning(port)) {
streamz.log.warn(s"dynamodb local is already running on port $port")
} else {
streamz.log.info("Starting dynamodb local")
Process(args).run()
do {
streamz.log.info(s"Waiting for dynamodb local to boot on port $port")
Thread.sleep(500)
} while (!isDynamoDBLocalRunning(port))
}
PidUtils.extractPid("jps -ml".!!, port, jar).getOrElse {
sys.error(s"Cannot find dynamodb local PID running on $port")
}
}
}
Example 44
| Source File: CustomReceiver.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
// scalastyle:off println
package org.apache.spark.examples.streaming
import java.io.{BufferedReader, InputStreamReader}
import java.net.Socket
import java.nio.charset.StandardCharsets
import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.receiver.Receiver
private def receive() {
var socket: Socket = null
var userInput: String = null
try {
logInfo("Connecting to " + host + ":" + port)
socket = new Socket(host, port)
logInfo("Connected to " + host + ":" + port)
val reader = new BufferedReader(
new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))
userInput = reader.readLine()
while(!isStopped && userInput != null) {
store(userInput)
userInput = reader.readLine()
}
reader.close()
socket.close()
logInfo("Stopped receiving")
restart("Trying to connect again")
} catch {
case e: java.net.ConnectException =>
restart("Error connecting to " + host + ":" + port, e)
case t: Throwable =>
restart("Error receiving data", t)
}
}
}
// scalastyle:on println
Example 45
| Source File: SocketInputDStream.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.dstream
import java.io._
import java.net.{ConnectException, Socket}
import java.nio.charset.StandardCharsets
import scala.reflect.ClassTag
import scala.util.control.NonFatal
import org.apache.spark.internal.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.receiver.Receiver
import org.apache.spark.util.NextIterator
private[streaming]
class SocketInputDStream[T: ClassTag](
_ssc: StreamingContext,
host: String,
port: Int,
bytesToObjects: InputStream => Iterator[T],
storageLevel: StorageLevel
) extends ReceiverInputDStream[T](_ssc) {
def getReceiver(): Receiver[T] = {
new SocketReceiver(host, port, bytesToObjects, storageLevel)
}
}
private[streaming]
class SocketReceiver[T: ClassTag](
host: String,
port: Int,
bytesToObjects: InputStream => Iterator[T],
storageLevel: StorageLevel
) extends Receiver[T](storageLevel) with Logging {
private var socket: Socket = _
def onStart() {
logInfo(s"Connecting to $host:$port")
try {
socket = new Socket(host, port)
} catch {
case e: ConnectException =>
restart(s"Error connecting to $host:$port", e)
return
}
logInfo(s"Connected to $host:$port")
// Start the thread that receives data over a connection
new Thread("Socket Receiver") {
setDaemon(true)
override def run() { receive() }
}.start()
}
def onStop() {
// in case restart thread close it twice
synchronized {
if (socket != null) {
socket.close()
socket = null
logInfo(s"Closed socket to $host:$port")
}
}
}
def bytesToLines(inputStream: InputStream): Iterator[String] = {
val dataInputStream = new BufferedReader(
new InputStreamReader(inputStream, StandardCharsets.UTF_8))
new NextIterator[String] {
protected override def getNext() = {
val nextValue = dataInputStream.readLine()
if (nextValue == null) {
finished = true
}
nextValue
}
protected override def close() {
dataInputStream.close()
}
}
}
}
Example 46
| Source File: PythonGatewayServer.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.api.python
import java.io.DataOutputStream
import java.net.Socket
import py4j.GatewayServer
import org.apache.spark.internal.Logging
import org.apache.spark.util.Utils
private[spark] object PythonGatewayServer extends Logging {
initializeLogIfNecessary(true)
def main(args: Array[String]): Unit = Utils.tryOrExit {
// Start a GatewayServer on an ephemeral port
val gatewayServer: GatewayServer = new GatewayServer(null, 0)
gatewayServer.start()
val boundPort: Int = gatewayServer.getListeningPort
if (boundPort == -1) {
logError("GatewayServer failed to bind; exiting")
System.exit(1)
} else {
logDebug(s"Started PythonGatewayServer on port $boundPort")
}
// Communicate the bound port back to the caller via the caller-specified callback port
val callbackHost = sys.env("_PYSPARK_DRIVER_CALLBACK_HOST")
val callbackPort = sys.env("_PYSPARK_DRIVER_CALLBACK_PORT").toInt
logDebug(s"Communicating GatewayServer port to Python driver at $callbackHost:$callbackPort")
val callbackSocket = new Socket(callbackHost, callbackPort)
val dos = new DataOutputStream(callbackSocket.getOutputStream)
dos.writeInt(boundPort)
dos.close()
callbackSocket.close()
// Exit on EOF or broken pipe to ensure that this process dies when the Python driver dies:
while (System.in.read() != -1) {
// Do nothing
}
logDebug("Exiting due to broken pipe from Python driver")
System.exit(0)
}
}
Example 47
| Source File: LauncherBackend.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.launcher
import java.net.{InetAddress, Socket}
import org.apache.spark.SPARK_VERSION
import org.apache.spark.launcher.LauncherProtocol._
import org.apache.spark.util.{ThreadUtils, Utils}
protected def onDisconnected() : Unit = { }
private def fireStopRequest(): Unit = {
val thread = LauncherBackend.threadFactory.newThread(new Runnable() {
override def run(): Unit = Utils.tryLogNonFatalError {
onStopRequest()
}
})
thread.start()
}
private class BackendConnection(s: Socket) extends LauncherConnection(s) {
override protected def handle(m: Message): Unit = m match {
case _: Stop =>
fireStopRequest()
case _ =>
throw new IllegalArgumentException(s"Unexpected message type: ${m.getClass().getName()}")
}
override def close(): Unit = {
try {
super.close()
} finally {
onDisconnected()
_isConnected = false
}
}
}
}
private object LauncherBackend {
val threadFactory = ThreadUtils.namedThreadFactory("LauncherBackend")
}
Example 48
| Source File: GraphiteEndpointSink.scala From kafka-lag-exporter with Apache License 2.0 | 5 votes |
|
package com.lightbend.kafkalagexporter
import com.lightbend.kafkalagexporter.MetricsSink._
import com.lightbend.kafkalagexporter.EndpointSink.ClusterGlobalLabels
import java.net.Socket
import java.io.PrintWriter
import scala.util.{Try, Success, Failure}
import scala.util.Try
object GraphiteEndpointSink {
def apply(metricWhitelist: List[String], clusterGlobalLabels: ClusterGlobalLabels,
graphiteConfig: Option[GraphiteConfig]): MetricsSink = {
Try(new GraphiteEndpointSink(metricWhitelist, clusterGlobalLabels, graphiteConfig))
.fold(t => throw new Exception("Could not create Graphite Endpoint", t), sink => sink)
}
}
class GraphiteEndpointSink private(metricWhitelist: List[String], clusterGlobalLabels: ClusterGlobalLabels,
graphiteConfig: Option[GraphiteConfig]) extends EndpointSink(clusterGlobalLabels) {
def graphitePush(graphiteConfig: GraphiteConfig, metricName: String, metricValue: Double): Unit = {
Try(new Socket(graphiteConfig.host, graphiteConfig.port)) match {
case Success(socket) =>
Try(new PrintWriter(socket.getOutputStream)) match {
case Success(writer) =>
writer.print(s"${graphiteConfig.prefix.getOrElse("")}${metricName} ${metricValue} ${System.currentTimeMillis / 1000}\n")
writer.close
socket.close
case Failure(_) =>
socket.close
}
case Failure(_) => {
}
}
}
def metricNameToGraphiteMetricName(metricValue: MetricValue): String = {
(getGlobalLabelValuesOrDefault(metricValue.clusterName) ++ metricValue.labels
).map( x => x.replaceAll("\\.", "_")).mkString(".") + "." + metricValue.definition.name;
}
override def report(m: MetricValue): Unit = {
if (metricWhitelist.exists(m.definition.name.matches)) {
graphiteConfig.foreach { conf =>
graphitePush(conf, metricNameToGraphiteMetricName(m), m.value);
}
}
}
override def remove(m: RemoveMetric): Unit = {
}
}
Example 49
| Source File: ProxyServer.scala From devbox with Apache License 2.0 | 5 votes |
|
package cmdproxy
import java.io.BufferedReader
import java.io.InputStreamReader
import java.io.OutputStreamWriter
import java.io.PrintWriter
import java.net.InetAddress
import java.net.ServerSocket
import java.net.Socket
import scala.util.Using
import devbox.logger.FileLogger
import os.RelPath
import ujson.ParseException
import upickle.default.{macroRW, ReadWriter}
case class Request(workingDir: String, cmd: Seq[String])
object Request {
implicit val rw: ReadWriter[Request] = macroRW
}
val localDir: Map[os.RelPath, os.Path] = dirMapping.map(_.swap).toMap
def start(): Unit = {
logger.info(s"Starting command proxy server, listening at ${socket.getInetAddress}:${socket.getLocalPort}")
(new Thread("Git Proxy Thread") {
override def run(): Unit = {
while (!socket.isClosed) {
Using(socket.accept()) { handleConnection } recover {
case e: Exception =>
logger.error(s"Error handling request ${e.getMessage}")
case e: java.net.SocketException if e.getMessage == "Socket closed" =>
logger.error(s"Git proxy socket closed")
}
}
}
}).start()
}
def handleConnection(conn: Socket): Unit = try {
logger.info(s"Accepting connection from ${conn.getInetAddress}")
val in = new BufferedReader(new InputStreamReader(conn.getInputStream, ProxyServer.CHARSET_NAME))
val out = new PrintWriter(new OutputStreamWriter(conn.getOutputStream, ProxyServer.CHARSET_NAME))
upickle.default.read[Request](in.readLine()) match {
case Request(dir, args) =>
val workingDir = localDir
.collect{case (remote, local) if RelPath(dir).startsWith(remote) =>
local / RelPath(dir).relativeTo(remote)
}
.head
// being cautious here and only execute "git" commands
if (args.headOption.exists((_ == "git"))) {
logger.info(s"Executing `${args.mkString(" ")}` in $workingDir")
val proc = os.proc(args).call(
workingDir,
mergeErrIntoOut = true,
stdout = os.ProcessOutput.Readlines(str =>
out.println(upickle.default.write(Left[String, Int](str)))
),
check = false,
timeout = 10000
)
out.println(upickle.default.write(Right[String, Int](proc.exitCode)))
} else {
val msg = s"Not executing non-git commend: `${args.mkString(" ")}`."
logger.info(msg)
out.println(upickle.default.write(Right[String, Int](1)))
}
out.flush()
}
} catch {
case e: ParseException => logger.error(s"Error parsing incoming json request: ${e.getMessage}")
}
}
object ProxyServer {
val DEFAULT_PORT = 20280
val CHARSET_NAME = "UTF-8"
}
