org.apache.http.util.EntityUtils Scala Examples
The following examples show how to use org.apache.http.util.EntityUtils.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
Example 1
| Source File: HTTPClientStartMockDataFlow.scala From piflow with BSD 2-Clause "Simplified" License | 9 votes |
|
package cn.piflow.api
import org.apache.http.client.config.RequestConfig
import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClientBuilder
import org.apache.http.util.EntityUtils
object HTTPClientStartMockDataFlow {
def main(args: Array[String]): Unit = {
val json =
"""
|{
| "flow": {
| "name": "MockData",
| "executorMemory": "1g",
| "executorNumber": "1",
| "uuid": "8a80d63f720cdd2301723b7461d92600",
| "paths": [
| {
| "inport": "",
| "from": "MockData",
| "to": "ShowData",
| "outport": ""
| }
| ],
| "executorCores": "1",
| "driverMemory": "1g",
| "stops": [
| {
| "name": "MockData",
| "bundle": "cn.piflow.bundle.common.MockData",
| "uuid": "8a80d63f720cdd2301723b7461d92604",
| "properties": {
| "schema": "title:String, author:String, age:Int",
| "count": "10"
| },
| "customizedProperties": {
|
| }
| },
| {
| "name": "ShowData",
| "bundle": "cn.piflow.bundle.external.ShowData",
| "uuid": "8a80d63f720cdd2301723b7461d92602",
| "properties": {
| "showNumber": "5"
| },
| "customizedProperties": {
|
| }
| }
| ]
| }
|}
""".stripMargin
val url = "http://10.0.85.83:8001/flow/start"
val timeout = 1800
val requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout*1000)
.setConnectionRequestTimeout(timeout*1000)
.setSocketTimeout(timeout*1000).build()
val client = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build()
val post:HttpPost = new HttpPost(url)
post.addHeader("Content-Type", "application/json")
post.setEntity(new StringEntity(json))
val response:CloseableHttpResponse = client.execute(post)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("Code is " + str)
}
}
Example 2
| Source File: HTTPClientPutPlugin.scala From piflow with BSD 2-Clause "Simplified" License | 6 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientPutPlugin {
def main(args: Array[String]): Unit = {
val json = """{"plugin":"piflowexternal.jar"}"""
val url = "http://10.0.86.191:8002/plugin/add"
val client = HttpClients.createDefault()
val post:HttpPost = new HttpPost(url)
post.addHeader("Content-Type", "application/json")
post.setEntity(new StringEntity(json))
val response:CloseableHttpResponse = client.execute(post)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println(str)
}
}
Example 3
| Source File: HttpUtil.scala From sparta with Apache License 2.0 | 6 votes |
|
package com.stratio.benchmark.generator.utils
import org.apache.http.HttpStatus
import org.apache.http.client.methods.{HttpDelete, HttpGet, HttpPost, HttpPut}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClientBuilder
import org.apache.http.util.EntityUtils
import org.apache.log4j.Logger
import org.json4s.DefaultFormats
import org.json4s.native.JsonMethods._
import scala.io.Source
trait HttpUtil {
private val logger = Logger.getLogger(this.getClass)
def createPolicy(policyContent: String, endpoint: String)(implicit defaultFormats: DefaultFormats): String = {
val policyName = (parse(policyContent) \ "name").extract[String]
// If the policy exists when it launches the benchmark, it should stop and delete it.
getPolicyId(policyName, endpoint) match {
case Some(id) =>
stopPolicy(id, endpoint)
deletePolicy(id, endpoint)
case None => logger.debug(s"No policy with name $policyName exists in Sparta yet.")
}
val client = HttpClientBuilder.create().build()
val post = new HttpPost(s"$endpoint/policyContext")
post.setHeader("Content-type", "application/json")
post.setEntity(new StringEntity(policyContent))
val response = client.execute(post)
if(response.getStatusLine.getStatusCode != HttpStatus.SC_OK)
throw new IllegalStateException(s"Sparta status code is not OK: ${response.getStatusLine.getStatusCode}")
else {
val entity = response.getEntity
val policyId = (parse(EntityUtils.toString(entity)) \ "policyId").extract[String]
policyId
}
}
def getPolicyId(name: String, endpoint: String)(implicit defaultFormats: DefaultFormats): Option[String] = {
val client = HttpClientBuilder.create().build()
val get = new HttpGet(s"$endpoint/policy/findByName/$name")
val response = client.execute(get)
response.getStatusLine.getStatusCode match {
case HttpStatus.SC_OK =>
Option((parse(EntityUtils.toString(response.getEntity)) \ "id").extract[String])
case _ => None
}
}
def stopPolicy(id: String, endpoint: String): Unit = {
val client = HttpClientBuilder.create().build()
val put = new HttpPut(s"$endpoint/policyContext")
put.setHeader("Content-Type", "application/json")
val entity = new StringEntity(s"""{"id":"$id", "status":"Stopping"}""")
put.setEntity(entity)
val response = client.execute(put)
if(response.getStatusLine.getStatusCode != HttpStatus.SC_CREATED) {
logger.info(Source.fromInputStream(response.getEntity.getContent).mkString(""))
logger.info(s"Sparta status code is not OK: ${response.getStatusLine.getStatusCode}")
}
}
def deletePolicy(id: String, endpoint: String): Unit = {
val client = HttpClientBuilder.create().build()
val delete = new HttpDelete(s"$endpoint/policy/$id")
val response = client.execute(delete)
if(response.getStatusLine.getStatusCode != HttpStatus.SC_OK)
logger.info(s"Sparta status code is not OK: ${response.getStatusLine.getStatusCode}")
}
}
Example 4
| Source File: LdContextResolver.scala From NGSI-LD_Experimental with MIT License | 5 votes |
|
package utils
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClientBuilder
import org.apache.http.util.EntityUtils
import scala.collection.mutable
object LdContextResolver {
private val ldContexts = mutable.Map[String, Map[String, Any]]()
def resolveContext(ldContextLoc: String, ldContextAcc: mutable.Map[String, String]): Unit = synchronized {
var ldContext:Map[String, Any] = Map[String,Any]()
if (ldContexts.contains(ldContextLoc)) {
Console.println(s"loading @context from cache: ${ldContextLoc}")
ldContext = ldContexts(ldContextLoc)
}
else {
Console.println(s"Resolving JSON-LD @context: ${ldContextLoc}")
val getRequest = new HttpGet(ldContextLoc)
// send the GET request
val httpClient = HttpClientBuilder.create().build()
val result = httpClient.execute(getRequest)
if (result.getStatusLine.getStatusCode == 200) {
val ldContextStr = EntityUtils.toString(result.getEntity, "UTF-8")
ldContext = ParserUtil.parse(ldContextStr).asInstanceOf[Map[String, Any]]
ldContexts += (ldContextLoc -> ldContext)
}
}
val firstLevel = ldContext.getOrElse("@context", None)
if (firstLevel == None) {
Console.println(s"It seems ${ldContextLoc} does not contain a valid JSON-LD @context")
return
}
if (firstLevel.isInstanceOf[List[Any]]) {
val list = firstLevel.asInstanceOf[List[Any]]
list.foreach(l => {
if (l.isInstanceOf[String]) {
resolveContext(l.asInstanceOf[String], ldContextAcc)
}
else if (l.isInstanceOf[Map[String, String]]) {
ldContextAcc ++= l.asInstanceOf[Map[String, String]]
}
})
}
else if (firstLevel.isInstanceOf[Map[String,Any]]) {
val contextKeys = firstLevel.asInstanceOf[Map[String,Any]]
contextKeys.keySet.foreach(contextKey => {
val contextValue = contextKeys(contextKey)
if (contextValue.isInstanceOf[String]) {
ldContextAcc += (contextKey -> contextValue.asInstanceOf[String])
}
// Supporting { "@type": "http://example.org/MyType", "@id": "http://example.org/id234" }
else if (contextValue.isInstanceOf[Map[String,String]]) {
val contextValueMap = contextValue.asInstanceOf[Map[String,String]]
ldContextAcc += (contextKey -> contextValueMap("@id"))
}
else {
Console.println(s"Cannot resolve @context: ${ldContextLoc}")
return
}
})
}
else {
Console.println(s"Cannot resolve @context: ${ldContextLoc}")
return
}
}
}
Example 5
| Source File: HTTPClientGetFlowGroupInfo.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientGetFlowGroupInfo {
def main(args: Array[String]): Unit = {
val url = "http://10.0.85.83:8001/group/info?groupId=group_91198855-afbc-4726-856f-31326173a90f"
val client = HttpClients.createDefault()
val getFlowGroupInfoData:HttpGet = new HttpGet(url)
val response:CloseableHttpResponse = client.execute(getFlowGroupInfoData)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("Code is " + str)
}
}
Example 6
| Source File: PostUrl.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.bundle.http
import java.io.{BufferedReader, InputStreamReader}
import java.net.URI
import cn.piflow.conf.bean.PropertyDescriptor
import cn.piflow.conf.util.{ImageUtil, MapUtil}
import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
import org.apache.commons.httpclient.HttpClient
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FSDataInputStream, FileSystem, Path}
import org.apache.http.client.methods.HttpPost
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
import org.apache.spark.sql.SparkSession
class PostUrl extends ConfigurableStop{
override val authorEmail: String = "[email protected]"
override val inportList: List[String] = List(Port.DefaultPort)
override val outportList: List[String] = List(Port.DefaultPort)
override val description: String = "Send a post request to the specified http"
var url : String= _
var jsonPath : String = _
override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
val spark = pec.get[SparkSession]()
//read json from hdfs
val conf = new Configuration()
val fs = FileSystem.get(URI.create(jsonPath),conf)
val stream: FSDataInputStream = fs.open(new Path(jsonPath))
val bufferReader = new BufferedReader(new InputStreamReader(stream))
var lineTxt = bufferReader.readLine()
val buffer = new StringBuffer()
while (lineTxt != null ){
buffer.append(lineTxt.mkString)
lineTxt=bufferReader.readLine()
}
// post
val client = HttpClients.createDefault()
val httpClient = new HttpClient()
httpClient.getParams().setContentCharset("utf-8")
val post = new HttpPost(url)
post.addHeader("content-Type","application/json")
post.setEntity(new StringEntity(buffer.toString))
val response = client.execute(post)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("Code is " + str)
}
override def setProperties(map: Map[String, Any]): Unit = {
url = MapUtil.get(map,key="url").asInstanceOf[String]
jsonPath = MapUtil.get(map,key="jsonPath").asInstanceOf[String]
}
override def getPropertyDescriptor(): List[PropertyDescriptor] = {
var descriptor : List[PropertyDescriptor] = List()
val url = new PropertyDescriptor()
.name("url")
.displayName("Url")
.defaultValue("")
.description("http request address")
.required(true)
.example("http://master:8002/flow/start")
val jsonPath = new PropertyDescriptor()
.name("jsonPath")
.displayName("JsonPath")
.defaultValue("")
.description("json parameter path for post request")
.required(true)
.example("hdfs://master:9000/work/flow.json")
descriptor = url :: descriptor
descriptor = jsonPath :: descriptor
descriptor
}
override def getIcon(): Array[Byte] = {
ImageUtil.getImage("icon/http/PostUrl.png")
}
override def getGroup(): List[String] = {
List(StopGroup.HttpGroup.toString)
}
override def initialize(ctx: ProcessContext): Unit = {
}
}
Example 7
| Source File: GetUrlTest.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.bundle.http
import java.io.{BufferedReader, InputStreamReader, PrintWriter}
import java.net.{HttpURLConnection, InetAddress, URL, URLConnection}
import cn.piflow.Runner
import cn.piflow.conf.bean.FlowBean
import cn.piflow.conf.util.{FileUtil, OptionUtil}
import cn.piflow.util.{PropertyUtil, ServerIpUtil}
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
import org.apache.spark.sql.SparkSession
import org.h2.tools.Server
import org.junit.Test
import scala.util.parsing.json.JSON
class GetUrlTest {
@Test
def testFlow(): Unit ={
//parse flow json
val file = "src/main/resources/flow/http/getUrl.json"
val flowJsonStr = FileUtil.fileReader(file)
val map = OptionUtil.getAny(JSON.parseFull(flowJsonStr)).asInstanceOf[Map[String, Any]]
println(map)
//create flow
val flowBean = FlowBean(map)
val flow = flowBean.constructFlow()
val ip = InetAddress.getLocalHost.getHostAddress
cn.piflow.util.FileUtil.writeFile("server.ip=" + ip, ServerIpUtil.getServerIpFile())
val h2Server = Server.createTcpServer("-tcp", "-tcpAllowOthers", "-tcpPort","50001").start()
//execute flow
val spark = SparkSession.builder()
.master("local[12]")
.appName("hive")
.config("spark.driver.memory", "4g")
.config("spark.executor.memory", "8g")
.config("spark.cores.max", "8")
.config("hive.metastore.uris",PropertyUtil.getPropertyValue("hive.metastore.uris"))
.enableHiveSupport()
.getOrCreate()
val process = Runner.create()
.bind(classOf[SparkSession].getName, spark)
.bind("checkpoint.path", "")
.bind("debug.path","")
.start(flow);
process.awaitTermination();
val pid = process.pid();
println(pid + "!!!!!!!!!!!!!!!!!!!!!")
spark.close();
}
}
Example 8
| Source File: FlowLauncher.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.util
import java.io.File
import java.util.Date
import java.util.concurrent.CountDownLatch
import cn.piflow.Flow
import org.apache.hadoop.security.SecurityUtil
import org.apache.http.client.methods.{CloseableHttpResponse, HttpPut}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
import org.apache.spark.launcher.SparkLauncher
object FlowLauncher {
def launch(flow: Flow) : SparkLauncher = {
var flowJson = flow.getFlowJson()
println("FlowLauncher json:" + flowJson)
val flowJsonencryptAES = SecurityUtil.encryptAES(flowJson)
var appId : String = ""
val countDownLatch = new CountDownLatch(1)
val launcher = new SparkLauncher
val sparkLauncher =launcher
.setAppName(flow.getFlowName())
.setMaster(PropertyUtil.getPropertyValue("spark.master"))
//.setDeployMode(PropertyUtil.getPropertyValue("spark.deploy.mode"))
.setAppResource(ConfigureUtil.getPiFlowBundlePath())
.setVerbose(true)
.setConf("spark.driver.memory", flow.getDriverMemory())
.setConf("spark.executor.instances", flow.getExecutorNum())
.setConf("spark.executor.memory", flow.getExecutorMem())
.setConf("spark.executor.cores",flow.getExecutorCores())
.addFile(PropertyUtil.getConfigureFile())
.addFile(ServerIpUtil.getServerIpFile())
.setMainClass("cn.piflow.api.StartFlowMain")
.addAppArgs(flowJsonencryptAES)
val sparkMaster = PropertyUtil.getPropertyValue("spark.master")
if(sparkMaster.equals("yarn")){
sparkLauncher.setDeployMode(PropertyUtil.getPropertyValue("spark.deploy.mode"))
sparkLauncher.setConf("spark.hadoop.yarn.resourcemanager.hostname", PropertyUtil.getPropertyValue("yarn.resourcemanager.hostname"))
}
//add other jars for application
val classPath = PropertyUtil.getClassPath()
val classPathFile = new File(classPath)
if(classPathFile.exists()){
FileUtil.getJarFile(new File(classPath)).foreach(f => {
println(f.getPath)
sparkLauncher.addJar(f.getPath)
})
}
val scalaPath = PropertyUtil.getScalaPath()
val scalaPathFile = new File(scalaPath)
if(scalaPathFile.exists()){
FileUtil.getJarFile(new File(scalaPath)).foreach(f => {
println(f.getPath)
sparkLauncher.addJar(f.getPath)
})
}
sparkLauncher
}
def stop(appID: String) = {
println("Stop Flow !!!!!!!!!!!!!!!!!!!!!!!!!!")
//yarn application kill appId
val url = ConfigureUtil.getYarnResourceManagerWebAppAddress() + appID + "/state"
val client = HttpClients.createDefault()
val put:HttpPut = new HttpPut(url)
val body ="{\"state\":\"KILLED\"}"
put.addHeader("Content-Type", "application/json")
put.setEntity(new StringEntity(body))
val response:CloseableHttpResponse = client.execute(put)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
//update db
println("Update flow state after Stop Flow !!!!!!!!!!!!!!!!!!!!!!!!!!")
H2Util.updateFlowState(appID, FlowState.KILLED)
H2Util.updateFlowFinishedTime(appID, new Date().toString)
"ok"
}
}
Example 9
| Source File: DWSHeartbeatResult.scala From Linkis with Apache License 2.0 | 5 votes |
|
package com.webank.wedatasphere.linkis.httpclient.dws.response
import java.util
import com.webank.wedatasphere.linkis.httpclient.discovery.HeartbeatResult
import org.apache.http.HttpResponse
import org.apache.http.util.EntityUtils
class DWSHeartbeatResult(response: HttpResponse, serverUrl: String) extends HeartbeatResult with DWSResult {
setResponse()
private def setResponse(): Unit = {
val entity = response.getEntity
val responseBody: String = if (entity != null) {
EntityUtils.toString(entity, "UTF-8")
} else {
null
}
val statusCode: Int = response.getStatusLine.getStatusCode
val url: String = serverUrl
val contentType: String = entity.getContentType.getValue
set(responseBody, statusCode, url, contentType)
}
if (getStatus != 0) warn(s"heartbeat to gateway $serverUrl failed! message: $getMessage.")
override val isHealthy: Boolean = getData.get("isHealthy") match {
case b: java.lang.Boolean => b
case s if s != null => s.toString.toBoolean
case _ => false
}
def getGatewayList: Array[String] = getData.get("gatewayList") match {
case l: util.List[String] => l.toArray(new Array[String](l.size())).map("http://" + _)
case array: Array[String] => array.map("http://" + _)
case _ => Array.empty
}
}
Example 10
| Source File: ElasticSearchAccess.scala From osstracker with Apache License 2.0 | 5 votes |
|
package com.netflix.oss.tools.osstrackerscraper
import org.apache.http.client.methods._
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client._
import org.apache.http.util.EntityUtils
import org.slf4j.LoggerFactory
import play.api.libs.json.{Json, JsObject}
class ElasticSearchAccess(esHost: String, esPort: Int) {
val logger = LoggerFactory.getLogger(getClass)
def indexDocInES(url: String, jsonDoc: String): Boolean = {
val client = HttpClientBuilder.create().build()
val req = new HttpPost(getFullUrl(url))
req.addHeader("Content-Type", "application/json")
req.setEntity(new StringEntity(jsonDoc))
val resp = client.execute(req)
resp.getStatusLine.getStatusCode match {
case 201 => {
true
}
case _ => {
logger.error(s"error creating es document for url = $url")
val respS = EntityUtils.toString(resp.getEntity)
logger.error(s"return code = ${resp.getStatusLine} and doc = ${respS}")
false
}
}
}
def getESDocForRepo(simpleDate: String, repoName: String): Option[JsObject] = {
val client = HttpClientBuilder.create().build()
val req = new HttpPost(getFullUrl("/osstracker/repo_stats/_search"))
req.addHeader("Content-Type", "application/json")
val jsonDoc = raw"""{"query":{"bool":{"must":[{"match":{"repo_name":"$repoName"}},{"match":{"asOfYYYYMMDD":"$simpleDate"}}]}}}"""
req.setEntity(new StringEntity(jsonDoc))
val resp = client.execute(req)
val resC = resp.getStatusLine.getStatusCode
resp.getStatusLine.getStatusCode match {
case 404 => None: Option[JsObject]
case _ =>
val respS = EntityUtils.toString(resp.getEntity)
val jsVal = Json.parse(respS)
val hits = (jsVal \ "hits" \ "total").as[Int]
hits match {
case 0 => None: Option[JsObject]
case _ => Some(((jsVal \ "hits" \ "hits")(0) \ "_source").get.asInstanceOf[JsObject])
}
}
}
def getESDocForRepos(simpleDate: String): Option[JsObject] = {
val client = HttpClientBuilder.create().build()
val req = new HttpPost(getFullUrl("/osstracker/allrepos_stats/_search"))
req.addHeader("Content-Type", "application/json")
val jsonDoc = raw"""{"query":{"match":{"asOfYYYYMMDD":"$simpleDate"}}}"""
req.setEntity(new StringEntity(jsonDoc))
val resp = client.execute(req)
val resC = resp.getStatusLine.getStatusCode
resp.getStatusLine.getStatusCode match {
case 404 => None: Option[JsObject]
case _ =>
val respS = EntityUtils.toString(resp.getEntity)
val jsVal = Json.parse(respS)
val hits = (jsVal \ "hits" \ "total").as[Int]
hits match {
case 0 => None: Option[JsObject]
case _ => Some(((jsVal \ "hits" \ "hits")(0) \ "_source").get.asInstanceOf[JsObject])
}
}
}
def getFullUrl(uri: String): String = {
s"http://${esHost}:${esPort}${uri}"
}
}
Example 11
| Source File: HttpInputDStream.scala From prosparkstreaming with Apache License 2.0 | 5 votes |
|
package org.apress.prospark
import java.util.Timer
import java.util.TimerTask
import scala.reflect.ClassTag
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
import org.apache.spark.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.api.java.JavaDStream
import org.apache.spark.streaming.api.java.JavaDStream.fromDStream
import org.apache.spark.streaming.api.java.JavaStreamingContext
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.receiver.Receiver
class HttpInputDStream(
@transient ssc_ : StreamingContext,
storageLevel: StorageLevel,
url: String,
interval: Long) extends ReceiverInputDStream[String](ssc_) with Logging {
def getReceiver(): Receiver[String] = {
new HttpReceiver(storageLevel, url, interval)
}
}
class HttpReceiver(
storageLevel: StorageLevel,
url: String,
interval: Long) extends Receiver[String](storageLevel) with Logging {
var httpClient: CloseableHttpClient = _
var trigger: Timer = _
def onStop() {
httpClient.close()
logInfo("Disconnected from Http Server")
}
def onStart() {
httpClient = HttpClients.createDefault()
trigger = new Timer()
trigger.scheduleAtFixedRate(new TimerTask {
def run() = doGet()
}, 0, interval * 1000)
logInfo("Http Receiver initiated")
}
def doGet() {
logInfo("Fetching data from Http source")
val response = httpClient.execute(new HttpGet(url))
try {
val content = EntityUtils.toString(response.getEntity())
store(content)
} catch {
case e: Exception => restart("Error! Problems while connecting", e)
} finally {
response.close()
}
}
}
object HttpUtils {
def createStream(
ssc: StreamingContext,
storageLevel: StorageLevel = StorageLevel.MEMORY_AND_DISK_SER_2,
url: String,
interval: Long): DStream[String] = {
new HttpInputDStream(ssc, storageLevel, url, interval)
}
def createStream(
jssc: JavaStreamingContext,
storageLevel: StorageLevel,
url: String,
interval: Long): JavaDStream[String] = {
implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[String]]
createStream(jssc.ssc, storageLevel, url, interval)
}
}
Example 12
| Source File: HttpInputDStream.scala From prosparkstreaming with Apache License 2.0 | 5 votes |
|
package org.apress.prospark
import java.util.Timer
import java.util.TimerTask
import scala.reflect.ClassTag
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
import org.apache.spark.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.api.java.JavaDStream
import org.apache.spark.streaming.api.java.JavaDStream.fromDStream
import org.apache.spark.streaming.api.java.JavaStreamingContext
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.receiver.Receiver
class HttpInputDStream(
@transient ssc_ : StreamingContext,
storageLevel: StorageLevel,
url: String,
interval: Long) extends ReceiverInputDStream[String](ssc_) with Logging {
def getReceiver(): Receiver[String] = {
new HttpReceiver(storageLevel, url, interval)
}
}
class HttpReceiver(
storageLevel: StorageLevel,
url: String,
interval: Long) extends Receiver[String](storageLevel) with Logging {
var httpClient: CloseableHttpClient = _
var trigger: Timer = _
def onStop() {
httpClient.close()
logInfo("Disconnected from Http Server")
}
def onStart() {
httpClient = HttpClients.createDefault()
trigger = new Timer()
trigger.scheduleAtFixedRate(new TimerTask {
def run() = doGet()
}, 0, interval * 1000)
logInfo("Http Receiver initiated")
}
def doGet() {
logInfo("Fetching data from Http source")
val response = httpClient.execute(new HttpGet(url))
try {
val content = EntityUtils.toString(response.getEntity())
store(content)
} catch {
case e: Exception => restart("Error! Problems while connecting", e)
} finally {
response.close()
}
}
}
object HttpUtils {
def createStream(
ssc: StreamingContext,
storageLevel: StorageLevel = StorageLevel.MEMORY_AND_DISK_SER_2,
url: String,
interval: Long): DStream[String] = {
new HttpInputDStream(ssc, storageLevel, url, interval)
}
def createStream(
jssc: JavaStreamingContext,
storageLevel: StorageLevel,
url: String,
interval: Long): JavaDStream[String] = {
implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[String]]
createStream(jssc.ssc, storageLevel, url, interval)
}
}
Example 13
| Source File: HTTPClientGetFlowInfo.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.commons.httpclient.URI
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet, HttpPost}
import org.apache.http.impl.client.HttpClients
import org.apache.http.message.BasicNameValuePair
import org.apache.http.util.EntityUtils
import org.omg.CORBA.NameValuePair
object HTTPClientGetFlowInfo {
def main(args: Array[String]): Unit = {
val url = "http://10.0.86.98:8001/flow/info?appID=application_1562293222869_0420"
val client = HttpClients.createDefault()
val getFlowDebugData:HttpGet = new HttpGet(url)
val response:CloseableHttpResponse = client.execute(getFlowDebugData)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("Code is " + str)
}
}
Example 14
| Source File: CmWellConsumeHandler.scala From CM-Well with Apache License 2.0 | 5 votes |
|
package cmwell.tools.neptune.export
import java.net.{URL, URLDecoder, URLEncoder}
import java.time.Instant
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.DefaultHttpClient
import org.apache.http.util.EntityUtils
import org.slf4j.LoggerFactory
object CmWellConsumeHandler {
protected lazy val logger = LoggerFactory.getLogger("cm_well_consumer")
val maxRetry = 5
private val sleepTimeout = 10000
def bulkConsume(cluster: String, position: String, format: String, updateMode:Boolean, retryCount:Int= 0): CloseableHttpResponse = {
val withMeta = if(updateMode) "&with-meta" else ""
val url = "http://" + cluster + "/_bulk-consume?position=" + position + "&format=" + format + withMeta
val client = new DefaultHttpClient
client.setHttpRequestRetryHandler(new CustomHttpClientRetryHandler())
val get = new HttpGet(url)
logger.info("Going to bulk consume,url= " + url)
val response = client.execute(get)
val statusCode = response.getStatusLine.getStatusCode
if (statusCode != 200 && statusCode != 204) {
if(statusCode == 503) {
logger.error("Failed to bulk consume, error status code=" + statusCode + "response entity=" + EntityUtils.toString(response.getEntity) + ".Going to retry...")
Thread.sleep(sleepTimeout)
bulkConsume(cluster, position, format, updateMode)
}
else{
if (retryCount < maxRetry) {
logger.error("Failed to bulk consume, error status code=" + statusCode + "response entity=" + EntityUtils.toString(response.getEntity) + ".Going to retry...,retry count=" + retryCount)
Thread.sleep(sleepTimeout)
bulkConsume(cluster, position, format, updateMode, retryCount + 1)
} else {
throw new Throwable("Failed to consume from cm-well, error code status=" + statusCode + ", response entity=" + EntityUtils.toString(response.getEntity))
}
}
}
response
}
def retrivePositionFromCreateConsumer(cluster: String, lengthHint: Int, qp: Option[String], updateMode:Boolean, automaticUpdateMode:Boolean, toolStartTime:Instant, retryCount:Int = 0): String = {
val withDeletedParam = if(updateMode || automaticUpdateMode) "&with-deleted" else ""
//initial mode
val qpTillStartTime = if(!updateMode && !automaticUpdateMode) URLEncoder.encode(",system.lastModified<") + toolStartTime.toString else ""
//automatic update mode
val qpAfterStartTime = if(!updateMode && automaticUpdateMode) URLEncoder.encode(",system.lastModified>>" )+ toolStartTime.toString else ""
val createConsumerUrl = "http://" + cluster + "/?op=create-consumer&qp=-system.parent.parent_hierarchy:/meta/" + qp.getOrElse("") + qpTillStartTime + qpAfterStartTime + "&recursive&length-hint=" + lengthHint + withDeletedParam
logger.info("create-consumer-url=" + createConsumerUrl)
val get = new HttpGet(createConsumerUrl)
val client = new DefaultHttpClient
client.setHttpRequestRetryHandler(new CustomHttpClientRetryHandler())
val response = client.execute(get)
val res = response.getAllHeaders.find(_.getName == "X-CM-WELL-POSITION").map(_.getValue).getOrElse("")
logger.info("create-Consumer http status=" + response.getStatusLine.getStatusCode)
val statusCode = response.getStatusLine.getStatusCode
if (statusCode != 200) {
if(statusCode == 503){
logger.error("Failed to retrieve position via create-consumer api,error status code=" + statusCode + ", response entity=" + EntityUtils.toString(response.getEntity) + ".Going to retry...")
Thread.sleep(sleepTimeout)
retrivePositionFromCreateConsumer(cluster, lengthHint, qp, updateMode, automaticUpdateMode, toolStartTime)
}else {
if (retryCount < maxRetry) {
logger.error("Failed to retrieve position via create-consumer api,error status code=" + statusCode + ", response entity=" + EntityUtils.toString(response.getEntity) + ".Going to retry..., retry count=" + retryCount)
Thread.sleep(sleepTimeout)
retrivePositionFromCreateConsumer(cluster, lengthHint, qp, updateMode, automaticUpdateMode, toolStartTime, retryCount+1)
}
else {
throw new Throwable("Failed to consume from cm-well, error code status=" + statusCode + ", response entity=" + EntityUtils.toString(response.getEntity))
}
}
}
res
}
}
Example 15
| Source File: HttpUtil.scala From CM-Well with Apache License 2.0 | 5 votes |
|
package cmwell.analytics.util
import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
import org.apache.http.HttpResponse
import org.apache.http.client.methods.{HttpGet, HttpPost}
import org.apache.http.client.{ClientProtocolException, ResponseHandler}
import org.apache.http.entity.{ContentType, StringEntity}
import org.apache.http.impl.client.{CloseableHttpClient, HttpClients}
import org.apache.http.util.EntityUtils
object HttpUtil {
private val mapper = new ObjectMapper()
private val responseHandler = new ResponseHandler[String]() {
override def handleResponse(response: HttpResponse): String = {
val status = response.getStatusLine.getStatusCode
if (status >= 200 && status < 300) {
val entity = response.getEntity
if (entity != null)
EntityUtils.toString(entity)
else
throw new ClientProtocolException("Unexpected null response")
}
else {
throw new ClientProtocolException("Unexpected response status: " + status)
}
}
}
def get(url: String, client: CloseableHttpClient): String = {
val httpGet = new HttpGet(url)
client.execute(httpGet, responseHandler)
}
def get(url: String): String = {
val client = HttpClients.createDefault
try {
get(url, client)
}
finally {
client.close()
}
}
def getJson(url: String): JsonNode = {
val client = HttpClients.createDefault
try {
getJson(url, client)
}
finally {
client.close()
}
}
def getJson(url: String, client: CloseableHttpClient): JsonNode = {
mapper.readTree(get(url, client))
}
def getJson(url: String, content: String, contentType: String): JsonNode = {
val httpPost = new HttpPost(url)
val entity = new StringEntity(content, ContentType.create(contentType, "UTF-8"))
httpPost.setEntity(entity)
val client = HttpClients.createDefault
try {
mapper.readTree(client.execute(httpPost, responseHandler))
}
finally {
client.close()
}
}
}
Example 16
| Source File: ProxyCrawler.scala From ProxyCrawler with Apache License 2.0 | 5 votes |
|
package org.crowdcrawler.proxycrawler
import java.io.IOException
import java.net.URI
import java.security.cert.X509Certificate
import com.typesafe.scalalogging.Logger
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClients
import org.apache.http.ssl.{TrustStrategy, SSLContexts}
import org.apache.http.conn.ssl.{NoopHostnameVerifier, SSLConnectionSocketFactory}
import org.apache.http.util.EntityUtils
import org.crowdcrawler.proxycrawler.crawler.plugins.AbstractPlugin
import org.apache.http.HttpHeaders
import org.slf4j.LoggerFactory
import scala.collection.immutable
import scala.collection.mutable
class ProxyCrawler(plugins: List[AbstractPlugin]) {
*;q=0.8"),
(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate, sdch"),
(HttpHeaders.ACCEPT_LANGUAGE, "en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4"),
(HttpHeaders.CONNECTION, "keep-alive")
)
private val CLIENT = {
// trust all certificates including self-signed certificates
val sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
def isTrusted(chain: Array[X509Certificate], authType: String) = true
}).build()
val connectionFactory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE)
HttpClients.custom().setSSLSocketFactory(connectionFactory).build()
}
def apply(classNames: String*): ProxyCrawler = {
val plugins = mutable.ListBuffer.empty[AbstractPlugin]
for (className <- classNames) {
val clazz = Class.forName("org.crowdcrawler.proxycrawler.crawler.plugins." + className)
plugins += clazz.newInstance().asInstanceOf[AbstractPlugin]
}
new ProxyCrawler(plugins.toList)
}
private def createRequest(uri: URI, headers: immutable.Map[String, String]): HttpGet = {
val request = new HttpGet(uri)
for (header <- headers) {
request.setHeader(header._1, header._2)
}
request
}
}
Example 17
| Source File: HttpProxyChecker.scala From ProxyCrawler with Apache License 2.0 | 5 votes |
|
package org.crowdcrawler.proxycrawler.checker
import java.net.URI
import java.nio.charset.StandardCharsets
import org.apache.http.annotation.ThreadSafe
import org.apache.http.HttpHost
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
@ThreadSafe
private[checker] object HttpProxyChecker extends AbstractProxyChecker {
private val CLIENT = HttpClients.custom().setMaxConnTotal(AbstractProxyChecker.MAX_CONN)
.disableRedirectHandling().build()
private val TARGET_URL = new URI("http://www.baidu.com")
def check(host: String, port: Int): (Int, Int) = {
val request = new HttpGet(TARGET_URL)
AbstractProxyChecker.configureRequest(request, Some(new HttpHost(host, port, "http")))
val response = CLIENT.execute(request)
val statusCode = response.getStatusLine.getStatusCode
val html = EntityUtils.toString(response.getEntity, StandardCharsets.UTF_8)
if (statusCode == 200 && html.contains("<title>百度一下")) (statusCode, html.getBytes.length) else (statusCode, -1)
}
}
Example 18
| Source File: SocksProxyChecker.scala From ProxyCrawler with Apache License 2.0 | 5 votes |
|
package org.crowdcrawler.proxycrawler.checker
import java.net
import java.net.{InetSocketAddress, Socket, URI}
import java.nio.charset.StandardCharsets
import javax.net.ssl.{HostnameVerifier, SSLContext}
import org.apache.http.annotation.ThreadSafe
import org.apache.http.client.methods.HttpGet
import org.apache.http.client.protocol.HttpClientContext
import org.apache.http.config.RegistryBuilder
import org.apache.http.conn.socket.{ConnectionSocketFactory, PlainConnectionSocketFactory}
import org.apache.http.conn.ssl.{NoopHostnameVerifier, SSLConnectionSocketFactory}
import org.apache.http.impl.client.HttpClients
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager
import org.apache.http.protocol.HttpContext
import org.apache.http.util.EntityUtils
@ThreadSafe
private[checker] object SocksProxyChecker extends AbstractProxyChecker {
private class MyHttpConnectionSocketFactory extends PlainConnectionSocketFactory {
override def createSocket(context: HttpContext): Socket = {
val socksAddress = context.getAttribute("socks.address").asInstanceOf[InetSocketAddress]
val proxy = new net.Proxy(net.Proxy.Type.SOCKS, socksAddress)
new Socket(proxy)
}
}
private class MyHttpsConnectionSocketFactory(sslContext: SSLContext, verifier: HostnameVerifier)
extends SSLConnectionSocketFactory(sslContext) {
override def createSocket(context: HttpContext): Socket = {
val socksAddress = context.getAttribute("socks.address").asInstanceOf[InetSocketAddress]
val proxy = new net.Proxy(net.Proxy.Type.SOCKS, socksAddress)
new Socket(proxy)
}
}
private val CLIENT = {
val reg = RegistryBuilder.create[ConnectionSocketFactory]()
.register("http", new MyHttpConnectionSocketFactory())
.register("https",
new MyHttpsConnectionSocketFactory(HttpsProxyChecker.SSL_CONTEXT, NoopHostnameVerifier.INSTANCE))
.build()
val cm = new PoolingHttpClientConnectionManager(reg)
cm.setMaxTotal(AbstractProxyChecker.MAX_CONN)
HttpClients.custom().setConnectionManager(cm).disableRedirectHandling().build()
}
private val TARGET_URL = new URI("http://www.baidu.com")
def check(host: String, port: Int): (Int, Int) = {
val request = new HttpGet(TARGET_URL)
AbstractProxyChecker.configureRequest(request)
val httpContext = {
val socksAddress = new InetSocketAddress(host, port)
val context = HttpClientContext.create()
context.setAttribute("socks.address", socksAddress)
context
}
val response = CLIENT.execute(request, httpContext)
val statusCode = response.getStatusLine.getStatusCode
val html = EntityUtils.toString(response.getEntity, StandardCharsets.UTF_8)
if (statusCode == 200 && html.contains("<title>百度一下")) (statusCode, html.getBytes.length) else (statusCode, -1)
}
}
Example 19
| Source File: HttpsProxyChecker.scala From ProxyCrawler with Apache License 2.0 | 5 votes |
|
package org.crowdcrawler.proxycrawler.checker
import java.net.URI
import java.nio.charset.StandardCharsets
import java.security.cert.X509Certificate
import org.apache.http.HttpHost
import org.apache.http.annotation.ThreadSafe
import org.apache.http.client.methods.HttpGet
import org.apache.http.conn.ssl.{NoopHostnameVerifier, SSLConnectionSocketFactory}
import org.apache.http.impl.client.HttpClients
import org.apache.http.ssl.{TrustStrategy, SSLContexts}
import org.apache.http.util.EntityUtils
@ThreadSafe
private[checker] object HttpsProxyChecker extends AbstractProxyChecker {
// trust all certificates including self-signed certificates
private[checker] val SSL_CONTEXT = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
def isTrusted(chain: Array[X509Certificate], authType: String) = true
}).build()
private val CLIENT = {
val connectionFactory = new SSLConnectionSocketFactory(SSL_CONTEXT, NoopHostnameVerifier.INSTANCE)
HttpClients.custom().setSSLSocketFactory(connectionFactory).setMaxConnTotal(AbstractProxyChecker.MAX_CONN)
.disableRedirectHandling().build()
}
private val TARGET_URL = new URI("https://www.google.com")
def check(host: String, port: Int): (Int, Int) = {
val request = new HttpGet(TARGET_URL)
AbstractProxyChecker.configureRequest(request, Some(new HttpHost(host, port, "http")))
val response = CLIENT.execute(request)
val statusCode = response.getStatusLine.getStatusCode
val html = EntityUtils.toString(response.getEntity, StandardCharsets.UTF_8)
if (statusCode == 200 && html.contains("<title>Google</title>")) (statusCode, html.getBytes.length)
else (statusCode, -1)
}
}
Example 20
| Source File: ChuckNorris.scala From sumobot with Apache License 2.0 | 5 votes |
|
package com.sumologic.sumobot.plugins.chuck
import java.net.URLEncoder
import com.sumologic.sumobot.core.model.{IncomingMessage, OutgoingMessage, UserSender}
import com.sumologic.sumobot.plugins.BotPlugin
import org.apache.http.HttpResponse
import org.apache.http.util.EntityUtils
import play.api.libs.json.Json
import slack.models.User
class ChuckNorris extends BotPlugin {
override protected def help: String =
s"""I can Chuck Norris someone.
|
|chuck norris - Generally Chuck Norris.
|chuck norris me - Chuck Norris yourself.
|chuck norris <person> - Chuck Norris them.
""".stripMargin
private val ChuckNorrisAtMention = matchText(s"chuck norris $UserId.*")
private val ChuckNorrisMe = matchText(s"chuck norris me")
private val ChuckNorris = matchText(s"chuck norris")
private val BaseUrl = "http://api.icndb.com/jokes/random"
override protected def receiveIncomingMessage: ReceiveIncomingMessage = {
case msg@IncomingMessage(ChuckNorris(), _, _, _, _, _, _) =>
msg.httpGet(BaseUrl)(convertResponse)
case msg@IncomingMessage(ChuckNorrisMe(), _, _, _, _, _, sentByUser: UserSender) =>
msg.httpGet(url(sentByUser.slackUser))(convertResponse)
case msg@IncomingMessage(ChuckNorrisAtMention(userId), _, _, _, _, _, _) =>
userById(userId).foreach {
user =>
msg.httpGet(url(user))(convertResponse)
}
}
private def convertResponse(message: IncomingMessage, response: HttpResponse): OutgoingMessage = {
if (response.getStatusLine.getStatusCode == 200) {
val json = Json.parse(EntityUtils.toString(response.getEntity))
val joke = json \ "value" \ "joke"
val cleanJoke = joke.as[String].
replaceAllLiterally(""", "\"")
message.message(cleanJoke)
} else {
message.message("Chuck Norris is busy right now.")
}
}
private def url(user: User): String =
user.profile match {
case Some(profile) =>
s"$BaseUrl?firstName=${URLEncoder.encode(profile.first_name.getOrElse(user.name), "utf-8")}&lastName=${URLEncoder.encode(profile.last_name.getOrElse(""), "utf-8")}"
case None =>
s"$BaseUrl?firstName=${URLEncoder.encode(user.name, "utf-8")}&lastName="
}
}
Example 21
| Source File: Advice.scala From sumobot with Apache License 2.0 | 5 votes |
|
package com.sumologic.sumobot.plugins.advice
import com.sumologic.sumobot.core.model.{IncomingMessage, OutgoingMessage}
import com.sumologic.sumobot.plugins.BotPlugin
import org.apache.http.HttpResponse
import org.apache.http.util.EntityUtils
import play.api.libs.json.Json
object Advice {
val AdviceAbout = BotPlugin.matchText("(what should I do about|what do you think about|how do you handle) (.*)")
val RandomAdvice = BotPlugin.matchText("I need some advice")
private[advice] def parseResponse(response: HttpResponse): Seq[String] = {
val json = Json.parse(EntityUtils.toString(response.getEntity))
(json \\ "advice").map(_.as[String])
}
}
class Advice extends BotPlugin {
override protected def help: String =
s"""
|what should I do about <something>
|what do you think about <something>
|how do you handle <something>
|I need some advice
""".stripMargin
import Advice._
override protected def receiveIncomingMessage: ReceiveIncomingMessage = {
case msg@IncomingMessage(RandomAdvice(), _, _, _, _, _, _) =>
msg.httpGet(s"http://api.adviceslip.com/advice")(respondWithAdvice)
case msg@IncomingMessage(AdviceAbout(_, something), true, _, _, _, _, _) =>
msg.httpGet(s"http://api.adviceslip.com/advice/search/${urlEncode(something.trim)}")(respondWithAdvice)
}
private def respondWithAdvice(msg: IncomingMessage, response: HttpResponse): OutgoingMessage = {
if (response.getStatusLine.getStatusCode == 200) {
val choices = parseResponse(response)
if (choices.nonEmpty) {
msg.message(chooseRandom(choices: _*))
} else {
msg.response("No advice for you, sorry.")
}
} else {
msg.response("No advice for you, sorry.")
}
}
}
Example 22
| Source File: HTTPClientStartFlowSocketTextStreaming.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientStartFlowSocketTextStreaming {
def main(args: Array[String]): Unit = {
val json =
"""
|{
| "flow":{
| "name":"socketStreaming",
| "uuid":"1234",
| "stops":[
| {
| "uuid":"1111",
| "name":"SocketTextStream",
| "bundle":"cn.piflow.bundle.streaming.SocketTextStream",
| "properties":{
| "hostname":"10.0.86.98",
| "port":"9999",
| "batchDuration":"5"
| }
|
| },
| {
| "uuid":"2222",
| "name":"ConvertSchema",
| "bundle":"cn.piflow.bundle.common.ConvertSchema",
| "properties":{
| "schema":"value->line"
| }
| },
| {
| "uuid":"3333",
| "name":"CsvSave",
| "bundle":"cn.piflow.bundle.csv.CsvSave",
| "properties":{
| "csvSavePath":"hdfs://10.0.86.89:9000/xjzhu/flowStreaming",
| "header":"true",
| "delimiter":","
| }
| }
| ],
| "paths":[
| {
| "from":"SocketTextStream",
| "outport":"",
| "inport":"",
| "to":"ConvertSchema"
| },
| {
| "from":"ConvertSchema",
| "outport":"",
| "inport":"",
| "to":"CsvSave"
| }
| ]
| }
|}
""".stripMargin
val url = "http://10.0.86.98:8001/flow/start"
val client = HttpClients.createDefault()
val post:HttpPost = new HttpPost(url)
post.addHeader("Content-Type", "application/json")
post.setEntity(new StringEntity(json))
val response:CloseableHttpResponse = client.execute(post)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("Code is " + str)
}
}
Example 23
| Source File: HTTPClientGetFlowDebugData.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientGetFlowDebugData {
def main(args: Array[String]): Unit = {
//
val url = "http://10.0.86.191:8002/flow/debugData?appID=application_1562293222869_0090&stopName=Fork&port=out3"
val client = HttpClients.createDefault()
val getFlowInfo:HttpGet = new HttpGet(url)
val response:CloseableHttpResponse = client.execute(getFlowInfo)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println( str)
}
}
Example 24
| Source File: HTTPClientGetAllPlugin.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientGetAllPlugin {
def main(args: Array[String]): Unit = {
val url = "http://10.0.85.83:8001/plugin/info"
val client = HttpClients.createDefault()
val getFlowInfo:HttpGet = new HttpGet(url)
val response:CloseableHttpResponse = client.execute(getFlowInfo)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("result : " + str)
}
}
Example 25
| Source File: HTTPClientGetScheduleInfo.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.config.RequestConfig
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet, HttpPost}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.{HttpClientBuilder, HttpClients}
import org.apache.http.util.EntityUtils
object HTTPClientGetScheduleInfo {
def main(args: Array[String]): Unit = {
val url = "http://10.0.85.83:8001/schedule/info?scheduleId=schedule_9339f584-4ec5-4e12-a51d-4214182ff63a"
val client = HttpClients.createDefault()
val getFlowDebugData:HttpGet = new HttpGet(url)
val response:CloseableHttpResponse = client.execute(getFlowDebugData)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("Code is " + str)
}
}
Example 26
| Source File: HTTPClientRemovePlugin.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientRemovePlugin {
def main(args: Array[String]): Unit = {
val json = """{"plugin":"piflowexternal.jar"}"""
val url = "http://10.0.85.83:8001/plugin/remove"
val client = HttpClients.createDefault()
val post:HttpPost = new HttpPost(url)
post.addHeader("Content-Type", "application/json")
post.setEntity(new StringEntity(json))
val response:CloseableHttpResponse = client.execute(post)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println(str)
}
}
Example 27
| Source File: HTTPClientStartFlowTextFileStreaming.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientStartFlowTextFileStreaming {
def main(args: Array[String]): Unit = {
//mybe you should change the directory when no data received
val json =
"""
|{
| "flow":{
| "name":"TextFileStream",
| "uuid":"1234",
| "stops":[
| {
| "uuid":"1111",
| "name":"TextFileStream",
| "bundle":"cn.piflow.bundle.streaming.TextFileStream",
| "properties":{
| "directory":"hdfs://10.0.86.191:9000/xjzhu/TextFileStream3",
| "batchDuration":"5"
| }
|
| },4.2bcefghw
| {
| "uuid":"2222",
| "name":"ConvertSchema",
| "bundle":"cn.piflow.bundle.common.ConvertSchema",
| "properties":{
| "schema":"value->line"
| }
| },
| {
| "uuid":"3333",
| "name":"CsvSave",
| "bundle":"cn.piflow.bundle.csv.CsvSave",
| "properties":{
| "csvSavePath":"hdfs://10.0.86.89:9000/xjzhu/flowStreaming",
| "header":"true",
| "delimiter":","
| }
| }
| ],
| "paths":[
| {
| "from":"TextFileStream",
| "outport":"",
| "inport":"",
| "to":"ConvertSchema"
| },
| {
| "from":"ConvertSchema",
| "outport":"",
| "inport":"",
| "to":"CsvSave"
| }
| ]
| }
|}
""".stripMargin
val url = "http://10.0.86.98:8001/flow/start"
val client = HttpClients.createDefault()
val post:HttpPost = new HttpPost(url)
post.addHeader("Content-Type", "application/json")
post.setEntity(new StringEntity(json))
val response:CloseableHttpResponse = client.execute(post)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("Code is " + str)
}
}
Example 28
| Source File: HTTPClientStartFlowIncremental.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.config.RequestConfig
import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClientBuilder
import org.apache.http.util.EntityUtils
object HTTPClientStartFlowIncremental {
def main(args: Array[String]): Unit = {
//val json:String = """{"flow":{"name":"Flow","uuid":"1234","stops":[{"uuid":"1111","name":"XmlParser","bundle":"cn.piflow.bundle.xml.XmlParser","properties":{"xmlpath":"hdfs://10.0.86.89:9000/xjzhu/dblp.mini.xml","rowTag":"phdthesis"}},{"uuid":"2222","name":"SelectField","bundle":"cn.piflow.bundle.common.SelectField","properties":{"schema":"title,author,pages"}},{"uuid":"3333","name":"PutHiveStreaming","bundle":"cn.piflow.bundle.hive.PutHiveStreaming","properties":{"database":"sparktest","table":"dblp_phdthesis"}},{"uuid":"4444","name":"CsvParser","bundle":"cn.piflow.bundle.csv.CsvParser","properties":{"csvPath":"hdfs://10.0.86.89:9000/xjzhu/phdthesis.csv","header":"false","delimiter":",","schema":"title,author,pages"}},{"uuid":"555","name":"Merge","bundle":"cn.piflow.bundle.common.Merge","properties":{}},{"uuid":"666","name":"Fork","bundle":"cn.piflow.bundle.common.Fork","properties":{"outports":["out1","out2","out3"]}},{"uuid":"777","name":"JsonSave","bundle":"cn.piflow.bundle.json.JsonSave","properties":{"jsonSavePath":"hdfs://10.0.86.89:9000/xjzhu/phdthesis.json"}},{"uuid":"888","name":"CsvSave","bundle":"cn.piflow.bundle.csv.CsvSave","properties":{"csvSavePath":"hdfs://10.0.86.89:9000/xjzhu/phdthesis_result.csv","header":"true","delimiter":","}}],"paths":[{"from":"XmlParser","outport":"","inport":"","to":"SelectField"},{"from":"SelectField","outport":"","inport":"data1","to":"Merge"},{"from":"CsvParser","outport":"","inport":"data2","to":"Merge"},{"from":"Merge","outport":"","inport":"","to":"Fork"},{"from":"Fork","outport":"out1","inport":"","to":"PutHiveStreaming"},{"from":"Fork","outport":"out2","inport":"","to":"JsonSave"},{"from":"Fork","outport":"out3","inport":"","to":"CsvSave"}]}}"""
val json ="""
|{
| "flow":{
| "name":"incremental",
| "uuid":"1234",
| "stops":[
| {
| "uuid":"1111",
| "name":"MysqlReadIncremental",
| "bundle":"cn.piflow.bundle.jdbc.MysqlReadIncremental",
| "properties":{
| "url":"jdbc:mysql://10.0.86.90:3306/sparktest",
| "sql":"select * from student where id > #~#",
| "user":"root",
| "password":"root",
| "incrementalField":"id"
| }
| },
| {
| "uuid":"888",
| "name":"CsvSave",
| "bundle":"cn.piflow.bundle.csv.CsvSave",
| "properties":{
| "csvSavePath":"hdfs://10.0.86.89:9000/xjzhu/phdthesis_result.csv",
| "header":"true",
| "delimiter":","
| }
| }
| ],
| "paths":[
| {
| "from":"MysqlReadIncremental",
| "outport":"",
| "inport":"",
| "to":"CsvSave"
| }
| ]
| }
|}
""".stripMargin
val url = "http://10.0.86.98:8001/flow/start"
val timeout = 1800
val requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout*1000)
.setConnectionRequestTimeout(timeout*1000)
.setSocketTimeout(timeout*1000).build()
val client = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build()
val post:HttpPost = new HttpPost(url)
post.addHeader("Content-Type", "application/json")
post.setEntity(new StringEntity(json))
val response:CloseableHttpResponse = client.execute(post)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("Code is " + str)
}
}
Example 29
| Source File: HTTPClientStartScalaFlow.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.config.RequestConfig
import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClientBuilder
import org.apache.http.util.EntityUtils
object HTTPClientStartScalaFlow {
def main(args: Array[String]): Unit = {
val json =
"""
|{
| "flow":{
| "name":"scalaTest",
| "uuid":"1234567890",
| "stops":[
| {
| "uuid":"1111",
| "name":"CsvParser",
| "bundle":"cn.piflow.bundle.csv.CsvParser",
| "properties":{
| "csvPath":"hdfs://10.0.86.191:9000/xjzhu/test.csv",
| "header":"false",
| "delimiter":",",
| "schema":"title,author"
| }
| },
| {
| "uuid":"2222",
| "name":"ExecuteScalaFile",
| "bundle":"cn.piflow.bundle.script.ExecuteScalaFile",
| "properties":{
| "plugin":"stop_scalaTest_ExecuteScalaFile_4444",
| "script":"val df = in.read() \n df.createOrReplaceTempView(\"people\") \n val df1 = spark.sql(\"select * from prople where author like 'xjzhu'\") \n out.write(df1);"
| }
| },
| {
| "uuid":"3333",
| "name":"CsvSave",
| "bundle":"cn.piflow.bundle.csv.CsvSave",
| "properties":{
| "csvSavePath":"hdfs://10.0.86.191:9000/xjzhu/CSVOverwrite",
| "header": "true",
| "delimiter":",",
| "partition":"1",
| "saveMode": "overwrite"
| }
|
| }
| ],
| "paths":[
| {
| "from":"CsvParser",
| "outport":"",
| "inport":"",
| "to":"ExecuteScalaFile"
| },
| {
| "from":"ExecuteScalaFile",
| "outport":"",
| "inport":"",
| "to":"CsvSave"
| }
| ]
| }
|}
""".stripMargin
val url = "http://10.0.85.83:8001/flow/start"
val timeout = 1800
val requestConfig = RequestConfig.custom()
.setConnectTimeout(timeout*1000)
.setConnectionRequestTimeout(timeout*1000)
.setSocketTimeout(timeout*1000).build()
val client = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build()
val post:HttpPost = new HttpPost(url)
post.addHeader("Content-Type", "application/json")
post.setEntity(new StringEntity(json))
val response:CloseableHttpResponse = client.execute(post)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("Code is " + str)
}
}
Example 30
| Source File: HTTPClientGetStopInfo.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientGetStopInfo {
def main(args: Array[String]): Unit = {
val url = "http://10.0.85.83:8001/stop/info?bundle=cn.piflow.bundle.csv.CsvParser"
val client = HttpClients.createDefault()
val getFlowInfo:HttpGet = new HttpGet(url)
val response:CloseableHttpResponse = client.execute(getFlowInfo)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("result : " + str)
}
}
Example 31
| Source File: HTTPClientStopSchedule.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientStopSchedule {
def main(args: Array[String]): Unit = {
val json = """{"scheduleId":"schedule_badbf32a-674d-42a9-8e13-e91ae1539e01"}"""
val url = "http://10.0.88.13:8002/schedule/stop"
val client = HttpClients.createDefault()
val post:HttpPost = new HttpPost(url)
post.addHeader("Content-Type", "application/json")
post.setEntity(new StringEntity(json))
val response:CloseableHttpResponse = client.execute(post)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println(str)
}
}
Example 32
| Source File: HTTPClientGetFlowGroupProcess.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientGetFlowGroupProcess {
def main(args: Array[String]): Unit = {
val url = "http://10.0.85.83:8001/group/progress?groupId=group_527ff279-a278-41b0-a658-7595a576fbb9"
val client = HttpClients.createDefault()
val getFlowGroupProgressData:HttpGet = new HttpGet(url)
val response:CloseableHttpResponse = client.execute(getFlowGroupProgressData)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("Code is " + str)
}
}
Example 33
| Source File: HTTPClientStopFlowGroup.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientStopFlowGroup {
def main(args: Array[String]): Unit = {
val json = """{"groupId":"group_91198855-afbc-4726-856f-31326173a90f"}"""
val url = "http://10.0.85.83:8001/group/stop"
val client = HttpClients.createDefault()
val post:HttpPost = new HttpPost(url)
post.addHeader("Content-Type", "application/json")
post.setEntity(new StringEntity(json))
val response:CloseableHttpResponse = client.execute(post)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println(str)
}
}
Example 34
| Source File: HTTPClientStopFlow.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientStopFlow {
def main(args: Array[String]): Unit = {
val json = """{"appID":"application_1562293222869_0099"}"""
val url = "http://10.0.86.191:8002/flow/stop"
val client = HttpClients.createDefault()
val post:HttpPost = new HttpPost(url)
post.addHeader("Content-Type", "application/json")
post.setEntity(new StringEntity(json))
val response:CloseableHttpResponse = client.execute(post)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println(str)
}
}
Example 35
| Source File: HTTPClientGetStops.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientGetStops {
def main(args: Array[String]): Unit = {
//val url = "http://10.0.86.98:8002/stop/listWithGroup"
val url = "http://10.0.86.98:8001/stop/list"
val client = HttpClients.createDefault()
val getGroups:HttpGet = new HttpGet(url)
val response:CloseableHttpResponse = client.execute(getGroups)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("Groups is: " + str)
}
}
Example 36
| Source File: HTTPClientStartFlowKafkaStreaming.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientStartFlowKafkaStreaming {
def main(args: Array[String]): Unit = {
val json =
"""
|{
| "flow":{
| "name":"kafkaStreaming",
| "uuid":"1234",
| "stops":[
| {
| "uuid":"1111",
| "name":"kafkaStream",
| "bundle":"cn.piflow.bundle.streaming.KafkaStream",
| "properties":{
| "brokers":"10.0.86.191:9092,10.0.86.203:9092,10.0.86.210:9092",
| "groupId":"piflow",
| "topics":"streaming",
| "batchDuration":"5"
| }
|
| },
| {
| "uuid":"2222",
| "name":"ConvertSchema",
| "bundle":"cn.piflow.bundle.common.ConvertSchema",
| "properties":{
| "schema":"value->line"
| }
| },
| {
| "uuid":"3333",
| "name":"CsvSave",
| "bundle":"cn.piflow.bundle.csv.CsvSave",
| "properties":{
| "csvSavePath":"hdfs://10.0.86.89:9000/xjzhu/flowStreaming",
| "header":"true",
| "delimiter":","
| }
| }
| ],
| "paths":[
| {
| "from":"kafkaStream",
| "outport":"",
| "inport":"",
| "to":"ConvertSchema"
| },
| {
| "from":"ConvertSchema",
| "outport":"",
| "inport":"",
| "to":"CsvSave"
| }
| ]
| }
|}
""".stripMargin
val url = "http://10.0.86.191:8002/flow/start"
val client = HttpClients.createDefault()
val post:HttpPost = new HttpPost(url)
post.addHeader("Content-Type", "application/json")
post.setEntity(new StringEntity(json))
val response:CloseableHttpResponse = client.execute(post)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("Code is " + str)
}
}
Example 37
| Source File: HTTPClientStartFlowFlumeStreaming.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost}
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientStartFlowFlumeStreaming {
def main(args: Array[String]): Unit = {
//flume hostname must be one of the cluster worker node, the port should be greater than 10000
val json=
"""
|{
| "flow":{
| "name":"flumeStreaming",
| "uuid":"1234",
| "stops":[
| {
| "uuid":"1111",
| "name":"FlumeStream",
| "bundle":"cn.piflow.bundle.streaming.FlumeStream",
| "properties":{
| "hostname":"slave2",
| "port":"10002",
| "batchDuration":"5"
| }
|
| },
| {
| "uuid":"2222",
| "name":"ConvertSchema",
| "bundle":"cn.piflow.bundle.common.ConvertSchema",
| "properties":{
| "schema":"value->line"
| }
| },
| {
| "uuid":"3333",
| "name":"CsvSave",
| "bundle":"cn.piflow.bundle.csv.CsvSave",
| "properties":{
| "csvSavePath":"hdfs://10.0.86.89:9000/xjzhu/flowStreaming",
| "header":"true",
| "delimiter":","
| }
| }
| ],
| "paths":[
| {
| "from":"FlumeStream",
| "outport":"",
| "inport":"",
| "to":"ConvertSchema"
| },
| {
| "from":"ConvertSchema",
| "outport":"",
| "inport":"",
| "to":"CsvSave"
| }
| ]
| }
|}
""".stripMargin
val url = "http://10.0.86.98:8001/flow/start"
val client = HttpClients.createDefault()
val post:HttpPost = new HttpPost(url)
post.addHeader("Content-Type", "application/json")
post.setEntity(new StringEntity(json))
val response:CloseableHttpResponse = client.execute(post)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("Code is " + str)
}
}
Example 38
| Source File: HTTPClientGetFlowCheckpoints.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientGetFlowCheckpoints {
def main(args: Array[String]): Unit = {
val url = "http://10.0.86.98:8001/flow/checkpoints?appID=process_9e291e46-fe25-4e7c-943d-0ff85dddb22d_1"
val client = HttpClients.createDefault()
val getFlowInfo:HttpGet = new HttpGet(url)
val response:CloseableHttpResponse = client.execute(getFlowInfo)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("Flow checkpoint is " + str)
}
}
Example 39
| Source File: HTTPClientGetGroups.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
|
package cn.piflow.api
import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet}
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
object HTTPClientGetGroups {
def main(args: Array[String]): Unit = {
val url = "http://10.0.86.98:8001/stop/groups"
val client = HttpClients.createDefault()
val getGroups:HttpGet = new HttpGet(url)
val response:CloseableHttpResponse = client.execute(getGroups)
val entity = response.getEntity
val str = EntityUtils.toString(entity,"UTF-8")
println("Groups is: " + str)
}
}
