org.apache.http.entity.StringEntity Scala Examples
The following examples show how to use org.apache.http.entity.StringEntity.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
Example 1
Source File: HTTPClientStartMockDataFlow.scala From piflow with BSD 2-Clause "Simplified" License | 9 votes |
package cn.piflow.api import org.apache.http.client.config.RequestConfig import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost} import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClientBuilder import org.apache.http.util.EntityUtils object HTTPClientStartMockDataFlow { def main(args: Array[String]): Unit = { val json = """ |{ | "flow": { | "name": "MockData", | "executorMemory": "1g", | "executorNumber": "1", | "uuid": "8a80d63f720cdd2301723b7461d92600", | "paths": [ | { | "inport": "", | "from": "MockData", | "to": "ShowData", | "outport": "" | } | ], | "executorCores": "1", | "driverMemory": "1g", | "stops": [ | { | "name": "MockData", | "bundle": "cn.piflow.bundle.common.MockData", | "uuid": "8a80d63f720cdd2301723b7461d92604", | "properties": { | "schema": "title:String, author:String, age:Int", | "count": "10" | }, | "customizedProperties": { | | } | }, | { | "name": "ShowData", | "bundle": "cn.piflow.bundle.external.ShowData", | "uuid": "8a80d63f720cdd2301723b7461d92602", | "properties": { | "showNumber": "5" | }, | "customizedProperties": { | | } | } | ] | } |} """.stripMargin val url = "http://10.0.85.83:8001/flow/start" val timeout = 1800 val requestConfig = RequestConfig.custom() .setConnectTimeout(timeout*1000) .setConnectionRequestTimeout(timeout*1000) .setSocketTimeout(timeout*1000).build() val client = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build() val post:HttpPost = new HttpPost(url) post.addHeader("Content-Type", "application/json") post.setEntity(new StringEntity(json)) val response:CloseableHttpResponse = client.execute(post) val entity = response.getEntity val str = EntityUtils.toString(entity,"UTF-8") println("Code is " + str) } }
Example 2
Source File: HttpUtil.scala From sparta with Apache License 2.0 | 6 votes |
package com.stratio.benchmark.generator.utils import org.apache.http.HttpStatus import org.apache.http.client.methods.{HttpDelete, HttpGet, HttpPost, HttpPut} import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClientBuilder import org.apache.http.util.EntityUtils import org.apache.log4j.Logger import org.json4s.DefaultFormats import org.json4s.native.JsonMethods._ import scala.io.Source trait HttpUtil { private val logger = Logger.getLogger(this.getClass) def createPolicy(policyContent: String, endpoint: String)(implicit defaultFormats: DefaultFormats): String = { val policyName = (parse(policyContent) \ "name").extract[String] // If the policy exists when it launches the benchmark, it should stop and delete it. getPolicyId(policyName, endpoint) match { case Some(id) => stopPolicy(id, endpoint) deletePolicy(id, endpoint) case None => logger.debug(s"No policy with name $policyName exists in Sparta yet.") } val client = HttpClientBuilder.create().build() val post = new HttpPost(s"$endpoint/policyContext") post.setHeader("Content-type", "application/json") post.setEntity(new StringEntity(policyContent)) val response = client.execute(post) if(response.getStatusLine.getStatusCode != HttpStatus.SC_OK) throw new IllegalStateException(s"Sparta status code is not OK: ${response.getStatusLine.getStatusCode}") else { val entity = response.getEntity val policyId = (parse(EntityUtils.toString(entity)) \ "policyId").extract[String] policyId } } def getPolicyId(name: String, endpoint: String)(implicit defaultFormats: DefaultFormats): Option[String] = { val client = HttpClientBuilder.create().build() val get = new HttpGet(s"$endpoint/policy/findByName/$name") val response = client.execute(get) response.getStatusLine.getStatusCode match { case HttpStatus.SC_OK => Option((parse(EntityUtils.toString(response.getEntity)) \ "id").extract[String]) case _ => None } } def stopPolicy(id: String, endpoint: String): Unit = { val client = HttpClientBuilder.create().build() val put = new HttpPut(s"$endpoint/policyContext") put.setHeader("Content-Type", "application/json") val entity = new StringEntity(s"""{"id":"$id", "status":"Stopping"}""") put.setEntity(entity) val response = client.execute(put) if(response.getStatusLine.getStatusCode != HttpStatus.SC_CREATED) { logger.info(Source.fromInputStream(response.getEntity.getContent).mkString("")) logger.info(s"Sparta status code is not OK: ${response.getStatusLine.getStatusCode}") } } def deletePolicy(id: String, endpoint: String): Unit = { val client = HttpClientBuilder.create().build() val delete = new HttpDelete(s"$endpoint/policy/$id") val response = client.execute(delete) if(response.getStatusLine.getStatusCode != HttpStatus.SC_OK) logger.info(s"Sparta status code is not OK: ${response.getStatusLine.getStatusCode}") } }
Example 3
Source File: HTTPClientPutPlugin.scala From piflow with BSD 2-Clause "Simplified" License | 6 votes |
package cn.piflow.api import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost} import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClients import org.apache.http.util.EntityUtils object HTTPClientPutPlugin { def main(args: Array[String]): Unit = { val json = """{"plugin":"piflowexternal.jar"}""" val url = "http://10.0.86.191:8002/plugin/add" val client = HttpClients.createDefault() val post:HttpPost = new HttpPost(url) post.addHeader("Content-Type", "application/json") post.setEntity(new StringEntity(json)) val response:CloseableHttpResponse = client.execute(post) val entity = response.getEntity val str = EntityUtils.toString(entity,"UTF-8") println(str) } }
Example 4
Source File: HTTPClientStopSchedule.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
package cn.piflow.api import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost} import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClients import org.apache.http.util.EntityUtils object HTTPClientStopSchedule { def main(args: Array[String]): Unit = { val json = """{"scheduleId":"schedule_badbf32a-674d-42a9-8e13-e91ae1539e01"}""" val url = "http://10.0.88.13:8002/schedule/stop" val client = HttpClients.createDefault() val post:HttpPost = new HttpPost(url) post.addHeader("Content-Type", "application/json") post.setEntity(new StringEntity(json)) val response:CloseableHttpResponse = client.execute(post) val entity = response.getEntity val str = EntityUtils.toString(entity,"UTF-8") println(str) } }
Example 5
Source File: PagerDutyEventApi.scala From sumobot with Apache License 2.0 | 5 votes |
package com.sumologic.sumobot.plugins.pagerduty import com.sumologic.sumobot.plugins.HttpClientWithTimeOut import org.apache.http.client.methods.HttpPost import org.apache.http.entity.{ContentType, StringEntity} import play.api.libs.json.Json class PagerDutyEventApi { case class PagerDutyEvent(service_key: String, event_type: String, incident_key: String, description: String) implicit val pagerDutyEventFormat = Json.format[PagerDutyEvent] private val SubmitUrl = "https://events.pagerduty.com/generic/2010-04-15/create_event.json" private val client = HttpClientWithTimeOut.client() def page(channel: String, serviceKey: String, description: String) { val event = Json.toJson(PagerDutyEvent(serviceKey, "trigger", channel, description)) val entity = new StringEntity(event.toString(), ContentType.APPLICATION_JSON) val post = new HttpPost(SubmitUrl) post.setEntity(entity) client.execute(post) } }
Example 6
Source File: HttpExecutionTest.scala From maze with Apache License 2.0 | 5 votes |
package fr.vsct.dt.maze.helpers import com.typesafe.scalalogging.StrictLogging import fr.vsct.dt.maze.core.Commands.expectThat import fr.vsct.dt.maze.core.Predef._ import fr.vsct.dt.maze.core.{Predicate, Result} import org.apache.http._ import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet} import org.apache.http.entity.{ContentType, StringEntity} import org.apache.http.impl.client.CloseableHttpClient import org.apache.http.message.{BasicHttpResponse, BasicStatusLine} import org.apache.http.protocol.HttpContext import org.scalatest.FlatSpec import scala.beans.BeanProperty class HttpExecutionTest extends FlatSpec { class MockHttpClient(val response: String) extends CloseableHttpClient with StrictLogging { var init = false override def doExecute(target: HttpHost, request: HttpRequest, context: HttpContext): CloseableHttpResponse = { if (!init) throw new IllegalStateException("Client is not initialized") logger.info("Doing actual http call") val r = if(request.getRequestLine.getUri == "http://some-url.com") { val t = new BasicCloseableHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK")) t.setEntity(new StringEntity(response, "UTF-8")) t } else { val t = new BasicCloseableHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, HttpStatus.SC_BAD_REQUEST, "KO")) t.setEntity(new StringEntity("""{"status": "ko"}""", ContentType.APPLICATION_JSON)) t } r } @Deprecated override def getConnectionManager = null @Deprecated override def getParams = null override def close(): Unit = {} class BasicCloseableHttpResponse(statusLine: StatusLine) extends BasicHttpResponse(statusLine) with CloseableHttpResponse { override def close(): Unit = {} } } "a http check" should "not do an effective call until apply is effectively called" in { Http.client = new MockHttpClient("Youppy !") val requestOk = new HttpGet("http://some-url.com") val check1: Predicate = Http.execute(requestOk).status is 200 val check2: Predicate = Http.execute(requestOk).response is "Youppy !" val check3 = check1 || check2 val check4 = !check3 Http.client.asInstanceOf[MockHttpClient].init = true assert(check1.get() == Result.success) assert(check2.get() == Result.success) assert(check3.get() == Result.success) assert(check4.get() == Result.failure(s"Expected ${check3.label} to be false")) expectThat(Http.get("http://some-error-url.com").status is 400) expectThat(Http.get("http://some-url.com").isOk) expectThat(!Http.get("http://some-error-url.com").isOk) expectThat(Http.get("http://some-error-url.com").responseAs(classOf[Stupid]) is Stupid(status = "ko")) } } case class Stupid(@BeanProperty status: String)
Example 7
Source File: HttpUtil.scala From CM-Well with Apache License 2.0 | 5 votes |
package cmwell.analytics.util import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} import org.apache.http.HttpResponse import org.apache.http.client.methods.{HttpGet, HttpPost} import org.apache.http.client.{ClientProtocolException, ResponseHandler} import org.apache.http.entity.{ContentType, StringEntity} import org.apache.http.impl.client.{CloseableHttpClient, HttpClients} import org.apache.http.util.EntityUtils object HttpUtil { private val mapper = new ObjectMapper() private val responseHandler = new ResponseHandler[String]() { override def handleResponse(response: HttpResponse): String = { val status = response.getStatusLine.getStatusCode if (status >= 200 && status < 300) { val entity = response.getEntity if (entity != null) EntityUtils.toString(entity) else throw new ClientProtocolException("Unexpected null response") } else { throw new ClientProtocolException("Unexpected response status: " + status) } } } def get(url: String, client: CloseableHttpClient): String = { val httpGet = new HttpGet(url) client.execute(httpGet, responseHandler) } def get(url: String): String = { val client = HttpClients.createDefault try { get(url, client) } finally { client.close() } } def getJson(url: String): JsonNode = { val client = HttpClients.createDefault try { getJson(url, client) } finally { client.close() } } def getJson(url: String, client: CloseableHttpClient): JsonNode = { mapper.readTree(get(url, client)) } def getJson(url: String, content: String, contentType: String): JsonNode = { val httpPost = new HttpPost(url) val entity = new StringEntity(content, ContentType.create(contentType, "UTF-8")) httpPost.setEntity(entity) val client = HttpClients.createDefault try { mapper.readTree(client.execute(httpPost, responseHandler)) } finally { client.close() } } }
Example 8
Source File: VaultHelper.scala From sparta with Apache License 2.0 | 5 votes |
package com.stratio.sparta.plugin.helper import java.io.{BufferedReader, InputStreamReader} import akka.event.slf4j.SLF4JLogging import org.apache.http.client.HttpClient import org.apache.http.client.methods.{HttpPost, HttpUriRequest} import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClientBuilder import scala.util.parsing.json.JSON object VaultHelper extends SLF4JLogging { lazy val client: HttpClient = HttpClientBuilder.create().build() lazy val jsonTemplate: String = "{ \"token\" : \"_replace_\" }" def getTemporalToken(vaultHost: String, token: String): String = { val requestUrl = s"$vaultHost/v1/sys/wrapping/wrap" log.debug(s"Requesting temporal token: $requestUrl") val post = new HttpPost(requestUrl) post.addHeader("X-Vault-Token", token) post.addHeader("X-Vault-Wrap-TTL", "2000s") post.setEntity(new StringEntity(jsonTemplate.replace("_replace_", token))) getContentFromResponse(post, "wrap_info")("token").asInstanceOf[String] } private def getContentFromResponse(uriRequest: HttpUriRequest, parentField: String): Map[String, Any] = { val response = client.execute(uriRequest) val rd = new BufferedReader(new InputStreamReader(response.getEntity.getContent)) val json = JSON.parseFull( Stream.continually(rd.readLine()).takeWhile(_ != null).mkString).get.asInstanceOf[Map[String, Any]] log.debug(s"getFrom Vault ${json.mkString("\n")}") if (response.getStatusLine.getStatusCode != 200) { val errors = json("errors").asInstanceOf[List[String]].mkString("\n") throw new RuntimeException(errors) } else json(parentField).asInstanceOf[Map[String, Any]] } }
Example 9
Source File: TSDBUpdater.scala From sprue with Apache License 2.0 | 5 votes |
package com.cloudera.sprue import java.io._ import org.apache.commons._ import org.apache.http._ import org.apache.http.client._ import org.apache.http.client.methods.HttpPost import java.util.ArrayList import org.apache.http.client.entity.UrlEncodedFormEntity import com.google.gson.Gson import java.util.HashMap import java.lang.reflect.Type import com.google.gson.reflect.TypeToken import org.apache.http.entity.StringEntity import org.apache.http.impl.client.DefaultHttpClient import org.apache.spark.sql.Row case class MetricsTags(state: String) case class OpenTSDBMessageElement(metric: String, timestamp: Long, value: Long, tags: MetricsTags) object TSDBUpdater { val client = new DefaultHttpClient() // val client = HttpClientBuilder.create.build } class TSDBUpdater (url : String) extends Serializable { def loadPatientStats (row : Row) { val metricList = new ArrayList[OpenTSDBMessageElement]() val jmap = new MetricsTags(row.getString(0)) val evalTimestamp = row.getLong(1) val sirsMetric = new OpenTSDBMessageElement("sirs", evalTimestamp, row.getLong(2), jmap) metricList.add(sirsMetric) val sepsisMetric = new OpenTSDBMessageElement("sepsis", evalTimestamp, row.getLong(3), jmap) metricList.add(sepsisMetric) val severeSepsisMetric = new OpenTSDBMessageElement("severeSepsis", evalTimestamp, row.getLong(4), jmap) metricList.add(severeSepsisMetric) val septicShockMetric = new OpenTSDBMessageElement("septicShock", evalTimestamp, row.getLong(5), jmap) metricList.add(septicShockMetric) val organMetric = new OpenTSDBMessageElement("organDysfunctionSyndrome", evalTimestamp, row.getLong(6), jmap) metricList.add(organMetric) val metricsAsJson = new Gson().toJson(metricList) val post = new HttpPost(url) post.setHeader("Content-type", "application/json"); post.setEntity(new StringEntity(metricsAsJson)); val response = TSDBUpdater.client.execute(post) // println("response =====" + response.toString()) } }
Example 10
Source File: ElasticSearchAccess.scala From osstracker with Apache License 2.0 | 5 votes |
package com.netflix.oss.tools.osstrackerscraper import org.apache.http.client.methods._ import org.apache.http.entity.StringEntity import org.apache.http.impl.client._ import org.apache.http.util.EntityUtils import org.slf4j.LoggerFactory import play.api.libs.json.{Json, JsObject} class ElasticSearchAccess(esHost: String, esPort: Int) { val logger = LoggerFactory.getLogger(getClass) def indexDocInES(url: String, jsonDoc: String): Boolean = { val client = HttpClientBuilder.create().build() val req = new HttpPost(getFullUrl(url)) req.addHeader("Content-Type", "application/json") req.setEntity(new StringEntity(jsonDoc)) val resp = client.execute(req) resp.getStatusLine.getStatusCode match { case 201 => { true } case _ => { logger.error(s"error creating es document for url = $url") val respS = EntityUtils.toString(resp.getEntity) logger.error(s"return code = ${resp.getStatusLine} and doc = ${respS}") false } } } def getESDocForRepo(simpleDate: String, repoName: String): Option[JsObject] = { val client = HttpClientBuilder.create().build() val req = new HttpPost(getFullUrl("/osstracker/repo_stats/_search")) req.addHeader("Content-Type", "application/json") val jsonDoc = raw"""{"query":{"bool":{"must":[{"match":{"repo_name":"$repoName"}},{"match":{"asOfYYYYMMDD":"$simpleDate"}}]}}}""" req.setEntity(new StringEntity(jsonDoc)) val resp = client.execute(req) val resC = resp.getStatusLine.getStatusCode resp.getStatusLine.getStatusCode match { case 404 => None: Option[JsObject] case _ => val respS = EntityUtils.toString(resp.getEntity) val jsVal = Json.parse(respS) val hits = (jsVal \ "hits" \ "total").as[Int] hits match { case 0 => None: Option[JsObject] case _ => Some(((jsVal \ "hits" \ "hits")(0) \ "_source").get.asInstanceOf[JsObject]) } } } def getESDocForRepos(simpleDate: String): Option[JsObject] = { val client = HttpClientBuilder.create().build() val req = new HttpPost(getFullUrl("/osstracker/allrepos_stats/_search")) req.addHeader("Content-Type", "application/json") val jsonDoc = raw"""{"query":{"match":{"asOfYYYYMMDD":"$simpleDate"}}}""" req.setEntity(new StringEntity(jsonDoc)) val resp = client.execute(req) val resC = resp.getStatusLine.getStatusCode resp.getStatusLine.getStatusCode match { case 404 => None: Option[JsObject] case _ => val respS = EntityUtils.toString(resp.getEntity) val jsVal = Json.parse(respS) val hits = (jsVal \ "hits" \ "total").as[Int] hits match { case 0 => None: Option[JsObject] case _ => Some(((jsVal \ "hits" \ "hits")(0) \ "_source").get.asInstanceOf[JsObject]) } } } def getFullUrl(uri: String): String = { s"http://${esHost}:${esPort}${uri}" } }
Example 11
Source File: Slack.scala From amadou with Apache License 2.0 | 5 votes |
package com.mediative.amadou import org.apache.http.client.methods.HttpPost import org.apache.http.entity.{ContentType, StringEntity} import org.apache.http.impl.client.HttpClients import org.json4s.NoTypeHints import org.json4s.jackson.Serialization import org.json4s.jackson.Serialization.{write} object Slack { case class PostException(msg: String) extends RuntimeException(msg) case class Payload( channel: String, text: String, username: String, icon_emoji: String, link_names: Boolean) } def post(message: String, icon: String = this.icon): Unit = { val payload = Payload(channel, message, user, icon, true) logger.info(s"Posting $payload to $url") val client = HttpClients.createDefault() val requestEntity = new StringEntity(write(payload), ContentType.APPLICATION_JSON) val postMethod = new HttpPost(url) postMethod.setEntity(requestEntity) val response = client.execute(postMethod) client.close() val status = response.getStatusLine if (status.getStatusCode != 200) throw PostException( s"$url replied with status ${status.getStatusCode}: ${status.getReasonPhrase}") } }
Example 12
Source File: ValidateTweetCompliance.scala From redrock with Apache License 2.0 | 5 votes |
package com.restapi import org.apache.http.client.methods.HttpPost import org.apache.http.entity.StringEntity import org.apache.http.impl.client.DefaultHttpClient import org.slf4j.LoggerFactory import play.api.libs.json._ import org.apache.http.params.BasicHttpParams import org.apache.http.params.HttpConnectionParams import org.apache.http.params.HttpParams import scala.io.Source._ object ValidateTweetCompliance { val logger = LoggerFactory.getLogger(this.getClass) def getNonCompliantTweets(jsonMessageRequest: String): List[String] = { val response = performBluemixCheck(jsonMessageRequest) try{ val jsonResponse = Json.parse(response).as[JsObject] if (jsonResponse.keys.contains("nonCompliant")) { return (jsonResponse \ "nonCompliant").as[List[String]] } else { return List[String]() } } catch { case e: Exception => { logger.error("Could not validate tweets list: Error on transforming response", e) return List[String]() } } } def performBluemixCheck(jsonMessageRequest: String): String = { try { val startTime = System.nanoTime() val httpClient = new DefaultHttpClient() val params = httpClient.getParams(); HttpConnectionParams.setConnectionTimeout(params, 2000); HttpConnectionParams.setSoTimeout(params, 2000); val request = new HttpPost(LoadConf.restConf.getString("bluemixProduction.requestURLforMessagesCheck")) // scalastyle:ignore val JSONentity: StringEntity = new StringEntity(jsonMessageRequest, "UTF-8") JSONentity.setContentType("application/json") request.setEntity(JSONentity) val httpResponse = httpClient.execute(request) val entity = httpResponse.getEntity() var jsonResponse = "{}" if (entity != null) { val inputStream = entity.getContent() jsonResponse = fromInputStream(inputStream).getLines.mkString inputStream.close } httpClient.getConnectionManager().shutdown() val elapsed = (System.nanoTime() - startTime) / 1e9 logger.info(s"Response for bluemix check tweets (sec): $elapsed") return jsonResponse } catch { case e: Exception => { logger.error("Could not validate tweets list", e) return "{}" } } } }
Example 13
Source File: SlackNotificationService.scala From pulse with Apache License 2.0 | 5 votes |
package io.phdata.pulse.alertengine.notification import com.typesafe.scalalogging.LazyLogging import io.phdata.pulse.alertengine.{ SlackAlertProfile, TriggeredAlert } import org.apache.http.client.methods.HttpPost import org.apache.http.entity.StringEntity import org.apache.http.impl.client.DefaultHttpClient class SlackNotificationService() extends LazyLogging { def notify(alerts: Iterable[TriggeredAlert], profile: SlackAlertProfile): Unit = for (alert <- alerts) { val formattedBody = NotificationFormatter.formatMessage(alert) val formattedSubject = NotificationFormatter.formatSubject(alert) sendSlackMsg(profile.url, formattedSubject + formattedBody) } def sendSlackMsg(url: String, message: String): Unit = { val httpClient = new DefaultHttpClient() try { val httpPost = new HttpPost(url) val jsonStr = s""" |{"text": "$message |---------------------------------------------"} |""".stripMargin val entity = new StringEntity(jsonStr) httpPost.setEntity(entity) httpPost.setHeader("Content-type", "application/json") httpClient.execute(httpPost) } catch { case e: Exception => logger.error(s"Error sending slack message $message", e) } finally { httpClient.getConnectionManager.shutdown() } } }
Example 14
Source File: FlowLauncher.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
package cn.piflow.util import java.io.File import java.util.Date import java.util.concurrent.CountDownLatch import cn.piflow.Flow import org.apache.hadoop.security.SecurityUtil import org.apache.http.client.methods.{CloseableHttpResponse, HttpPut} import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClients import org.apache.http.util.EntityUtils import org.apache.spark.launcher.SparkLauncher object FlowLauncher { def launch(flow: Flow) : SparkLauncher = { var flowJson = flow.getFlowJson() println("FlowLauncher json:" + flowJson) val flowJsonencryptAES = SecurityUtil.encryptAES(flowJson) var appId : String = "" val countDownLatch = new CountDownLatch(1) val launcher = new SparkLauncher val sparkLauncher =launcher .setAppName(flow.getFlowName()) .setMaster(PropertyUtil.getPropertyValue("spark.master")) //.setDeployMode(PropertyUtil.getPropertyValue("spark.deploy.mode")) .setAppResource(ConfigureUtil.getPiFlowBundlePath()) .setVerbose(true) .setConf("spark.driver.memory", flow.getDriverMemory()) .setConf("spark.executor.instances", flow.getExecutorNum()) .setConf("spark.executor.memory", flow.getExecutorMem()) .setConf("spark.executor.cores",flow.getExecutorCores()) .addFile(PropertyUtil.getConfigureFile()) .addFile(ServerIpUtil.getServerIpFile()) .setMainClass("cn.piflow.api.StartFlowMain") .addAppArgs(flowJsonencryptAES) val sparkMaster = PropertyUtil.getPropertyValue("spark.master") if(sparkMaster.equals("yarn")){ sparkLauncher.setDeployMode(PropertyUtil.getPropertyValue("spark.deploy.mode")) sparkLauncher.setConf("spark.hadoop.yarn.resourcemanager.hostname", PropertyUtil.getPropertyValue("yarn.resourcemanager.hostname")) } //add other jars for application val classPath = PropertyUtil.getClassPath() val classPathFile = new File(classPath) if(classPathFile.exists()){ FileUtil.getJarFile(new File(classPath)).foreach(f => { println(f.getPath) sparkLauncher.addJar(f.getPath) }) } val scalaPath = PropertyUtil.getScalaPath() val scalaPathFile = new File(scalaPath) if(scalaPathFile.exists()){ FileUtil.getJarFile(new File(scalaPath)).foreach(f => { println(f.getPath) sparkLauncher.addJar(f.getPath) }) } sparkLauncher } def stop(appID: String) = { println("Stop Flow !!!!!!!!!!!!!!!!!!!!!!!!!!") //yarn application kill appId val url = ConfigureUtil.getYarnResourceManagerWebAppAddress() + appID + "/state" val client = HttpClients.createDefault() val put:HttpPut = new HttpPut(url) val body ="{\"state\":\"KILLED\"}" put.addHeader("Content-Type", "application/json") put.setEntity(new StringEntity(body)) val response:CloseableHttpResponse = client.execute(put) val entity = response.getEntity val str = EntityUtils.toString(entity,"UTF-8") //update db println("Update flow state after Stop Flow !!!!!!!!!!!!!!!!!!!!!!!!!!") H2Util.updateFlowState(appID, FlowState.KILLED) H2Util.updateFlowFinishedTime(appID, new Date().toString) "ok" } }
Example 15
Source File: PostUrl.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
package cn.piflow.bundle.http import java.io.{BufferedReader, InputStreamReader} import java.net.URI import cn.piflow.conf.bean.PropertyDescriptor import cn.piflow.conf.util.{ImageUtil, MapUtil} import cn.piflow.conf.{ConfigurableStop, Port, StopGroup} import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext} import org.apache.commons.httpclient.HttpClient import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FSDataInputStream, FileSystem, Path} import org.apache.http.client.methods.HttpPost import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClients import org.apache.http.util.EntityUtils import org.apache.spark.sql.SparkSession class PostUrl extends ConfigurableStop{ override val authorEmail: String = "[email protected]" override val inportList: List[String] = List(Port.DefaultPort) override val outportList: List[String] = List(Port.DefaultPort) override val description: String = "Send a post request to the specified http" var url : String= _ var jsonPath : String = _ override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = { val spark = pec.get[SparkSession]() //read json from hdfs val conf = new Configuration() val fs = FileSystem.get(URI.create(jsonPath),conf) val stream: FSDataInputStream = fs.open(new Path(jsonPath)) val bufferReader = new BufferedReader(new InputStreamReader(stream)) var lineTxt = bufferReader.readLine() val buffer = new StringBuffer() while (lineTxt != null ){ buffer.append(lineTxt.mkString) lineTxt=bufferReader.readLine() } // post val client = HttpClients.createDefault() val httpClient = new HttpClient() httpClient.getParams().setContentCharset("utf-8") val post = new HttpPost(url) post.addHeader("content-Type","application/json") post.setEntity(new StringEntity(buffer.toString)) val response = client.execute(post) val entity = response.getEntity val str = EntityUtils.toString(entity,"UTF-8") println("Code is " + str) } override def setProperties(map: Map[String, Any]): Unit = { url = MapUtil.get(map,key="url").asInstanceOf[String] jsonPath = MapUtil.get(map,key="jsonPath").asInstanceOf[String] } override def getPropertyDescriptor(): List[PropertyDescriptor] = { var descriptor : List[PropertyDescriptor] = List() val url = new PropertyDescriptor() .name("url") .displayName("Url") .defaultValue("") .description("http request address") .required(true) .example("http://master:8002/flow/start") val jsonPath = new PropertyDescriptor() .name("jsonPath") .displayName("JsonPath") .defaultValue("") .description("json parameter path for post request") .required(true) .example("hdfs://master:9000/work/flow.json") descriptor = url :: descriptor descriptor = jsonPath :: descriptor descriptor } override def getIcon(): Array[Byte] = { ImageUtil.getImage("icon/http/PostUrl.png") } override def getGroup(): List[String] = { List(StopGroup.HttpGroup.toString) } override def initialize(ctx: ProcessContext): Unit = { } }
Example 16
Source File: HTTPClientStartFlowFlumeStreaming.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
package cn.piflow.api import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost} import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClients import org.apache.http.util.EntityUtils object HTTPClientStartFlowFlumeStreaming { def main(args: Array[String]): Unit = { //flume hostname must be one of the cluster worker node, the port should be greater than 10000 val json= """ |{ | "flow":{ | "name":"flumeStreaming", | "uuid":"1234", | "stops":[ | { | "uuid":"1111", | "name":"FlumeStream", | "bundle":"cn.piflow.bundle.streaming.FlumeStream", | "properties":{ | "hostname":"slave2", | "port":"10002", | "batchDuration":"5" | } | | }, | { | "uuid":"2222", | "name":"ConvertSchema", | "bundle":"cn.piflow.bundle.common.ConvertSchema", | "properties":{ | "schema":"value->line" | } | }, | { | "uuid":"3333", | "name":"CsvSave", | "bundle":"cn.piflow.bundle.csv.CsvSave", | "properties":{ | "csvSavePath":"hdfs://10.0.86.89:9000/xjzhu/flowStreaming", | "header":"true", | "delimiter":"," | } | } | ], | "paths":[ | { | "from":"FlumeStream", | "outport":"", | "inport":"", | "to":"ConvertSchema" | }, | { | "from":"ConvertSchema", | "outport":"", | "inport":"", | "to":"CsvSave" | } | ] | } |} """.stripMargin val url = "http://10.0.86.98:8001/flow/start" val client = HttpClients.createDefault() val post:HttpPost = new HttpPost(url) post.addHeader("Content-Type", "application/json") post.setEntity(new StringEntity(json)) val response:CloseableHttpResponse = client.execute(post) val entity = response.getEntity val str = EntityUtils.toString(entity,"UTF-8") println("Code is " + str) } }
Example 17
Source File: HTTPClientStartFlowKafkaStreaming.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
package cn.piflow.api import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost} import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClients import org.apache.http.util.EntityUtils object HTTPClientStartFlowKafkaStreaming { def main(args: Array[String]): Unit = { val json = """ |{ | "flow":{ | "name":"kafkaStreaming", | "uuid":"1234", | "stops":[ | { | "uuid":"1111", | "name":"kafkaStream", | "bundle":"cn.piflow.bundle.streaming.KafkaStream", | "properties":{ | "brokers":"10.0.86.191:9092,10.0.86.203:9092,10.0.86.210:9092", | "groupId":"piflow", | "topics":"streaming", | "batchDuration":"5" | } | | }, | { | "uuid":"2222", | "name":"ConvertSchema", | "bundle":"cn.piflow.bundle.common.ConvertSchema", | "properties":{ | "schema":"value->line" | } | }, | { | "uuid":"3333", | "name":"CsvSave", | "bundle":"cn.piflow.bundle.csv.CsvSave", | "properties":{ | "csvSavePath":"hdfs://10.0.86.89:9000/xjzhu/flowStreaming", | "header":"true", | "delimiter":"," | } | } | ], | "paths":[ | { | "from":"kafkaStream", | "outport":"", | "inport":"", | "to":"ConvertSchema" | }, | { | "from":"ConvertSchema", | "outport":"", | "inport":"", | "to":"CsvSave" | } | ] | } |} """.stripMargin val url = "http://10.0.86.191:8002/flow/start" val client = HttpClients.createDefault() val post:HttpPost = new HttpPost(url) post.addHeader("Content-Type", "application/json") post.setEntity(new StringEntity(json)) val response:CloseableHttpResponse = client.execute(post) val entity = response.getEntity val str = EntityUtils.toString(entity,"UTF-8") println("Code is " + str) } }
Example 18
Source File: HTTPClientStopFlow.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
package cn.piflow.api import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost} import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClients import org.apache.http.util.EntityUtils object HTTPClientStopFlow { def main(args: Array[String]): Unit = { val json = """{"appID":"application_1562293222869_0099"}""" val url = "http://10.0.86.191:8002/flow/stop" val client = HttpClients.createDefault() val post:HttpPost = new HttpPost(url) post.addHeader("Content-Type", "application/json") post.setEntity(new StringEntity(json)) val response:CloseableHttpResponse = client.execute(post) val entity = response.getEntity val str = EntityUtils.toString(entity,"UTF-8") println(str) } }
Example 19
Source File: HTTPClientStopFlowGroup.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
package cn.piflow.api import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost} import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClients import org.apache.http.util.EntityUtils object HTTPClientStopFlowGroup { def main(args: Array[String]): Unit = { val json = """{"groupId":"group_91198855-afbc-4726-856f-31326173a90f"}""" val url = "http://10.0.85.83:8001/group/stop" val client = HttpClients.createDefault() val post:HttpPost = new HttpPost(url) post.addHeader("Content-Type", "application/json") post.setEntity(new StringEntity(json)) val response:CloseableHttpResponse = client.execute(post) val entity = response.getEntity val str = EntityUtils.toString(entity,"UTF-8") println(str) } }
Example 20
Source File: HTTPClientStartFlowSocketTextStreaming.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
package cn.piflow.api import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost} import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClients import org.apache.http.util.EntityUtils object HTTPClientStartFlowSocketTextStreaming { def main(args: Array[String]): Unit = { val json = """ |{ | "flow":{ | "name":"socketStreaming", | "uuid":"1234", | "stops":[ | { | "uuid":"1111", | "name":"SocketTextStream", | "bundle":"cn.piflow.bundle.streaming.SocketTextStream", | "properties":{ | "hostname":"10.0.86.98", | "port":"9999", | "batchDuration":"5" | } | | }, | { | "uuid":"2222", | "name":"ConvertSchema", | "bundle":"cn.piflow.bundle.common.ConvertSchema", | "properties":{ | "schema":"value->line" | } | }, | { | "uuid":"3333", | "name":"CsvSave", | "bundle":"cn.piflow.bundle.csv.CsvSave", | "properties":{ | "csvSavePath":"hdfs://10.0.86.89:9000/xjzhu/flowStreaming", | "header":"true", | "delimiter":"," | } | } | ], | "paths":[ | { | "from":"SocketTextStream", | "outport":"", | "inport":"", | "to":"ConvertSchema" | }, | { | "from":"ConvertSchema", | "outport":"", | "inport":"", | "to":"CsvSave" | } | ] | } |} """.stripMargin val url = "http://10.0.86.98:8001/flow/start" val client = HttpClients.createDefault() val post:HttpPost = new HttpPost(url) post.addHeader("Content-Type", "application/json") post.setEntity(new StringEntity(json)) val response:CloseableHttpResponse = client.execute(post) val entity = response.getEntity val str = EntityUtils.toString(entity,"UTF-8") println("Code is " + str) } }
Example 21
Source File: HTTPClientStartScalaFlow.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
package cn.piflow.api import org.apache.http.client.config.RequestConfig import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost} import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClientBuilder import org.apache.http.util.EntityUtils object HTTPClientStartScalaFlow { def main(args: Array[String]): Unit = { val json = """ |{ | "flow":{ | "name":"scalaTest", | "uuid":"1234567890", | "stops":[ | { | "uuid":"1111", | "name":"CsvParser", | "bundle":"cn.piflow.bundle.csv.CsvParser", | "properties":{ | "csvPath":"hdfs://10.0.86.191:9000/xjzhu/test.csv", | "header":"false", | "delimiter":",", | "schema":"title,author" | } | }, | { | "uuid":"2222", | "name":"ExecuteScalaFile", | "bundle":"cn.piflow.bundle.script.ExecuteScalaFile", | "properties":{ | "plugin":"stop_scalaTest_ExecuteScalaFile_4444", | "script":"val df = in.read() \n df.createOrReplaceTempView(\"people\") \n val df1 = spark.sql(\"select * from prople where author like 'xjzhu'\") \n out.write(df1);" | } | }, | { | "uuid":"3333", | "name":"CsvSave", | "bundle":"cn.piflow.bundle.csv.CsvSave", | "properties":{ | "csvSavePath":"hdfs://10.0.86.191:9000/xjzhu/CSVOverwrite", | "header": "true", | "delimiter":",", | "partition":"1", | "saveMode": "overwrite" | } | | } | ], | "paths":[ | { | "from":"CsvParser", | "outport":"", | "inport":"", | "to":"ExecuteScalaFile" | }, | { | "from":"ExecuteScalaFile", | "outport":"", | "inport":"", | "to":"CsvSave" | } | ] | } |} """.stripMargin val url = "http://10.0.85.83:8001/flow/start" val timeout = 1800 val requestConfig = RequestConfig.custom() .setConnectTimeout(timeout*1000) .setConnectionRequestTimeout(timeout*1000) .setSocketTimeout(timeout*1000).build() val client = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build() val post:HttpPost = new HttpPost(url) post.addHeader("Content-Type", "application/json") post.setEntity(new StringEntity(json)) val response:CloseableHttpResponse = client.execute(post) val entity = response.getEntity val str = EntityUtils.toString(entity,"UTF-8") println("Code is " + str) } }
Example 22
Source File: HTTPClientStartFlowIncremental.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
package cn.piflow.api import org.apache.http.client.config.RequestConfig import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost} import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClientBuilder import org.apache.http.util.EntityUtils object HTTPClientStartFlowIncremental { def main(args: Array[String]): Unit = { //val json:String = """{"flow":{"name":"Flow","uuid":"1234","stops":[{"uuid":"1111","name":"XmlParser","bundle":"cn.piflow.bundle.xml.XmlParser","properties":{"xmlpath":"hdfs://10.0.86.89:9000/xjzhu/dblp.mini.xml","rowTag":"phdthesis"}},{"uuid":"2222","name":"SelectField","bundle":"cn.piflow.bundle.common.SelectField","properties":{"schema":"title,author,pages"}},{"uuid":"3333","name":"PutHiveStreaming","bundle":"cn.piflow.bundle.hive.PutHiveStreaming","properties":{"database":"sparktest","table":"dblp_phdthesis"}},{"uuid":"4444","name":"CsvParser","bundle":"cn.piflow.bundle.csv.CsvParser","properties":{"csvPath":"hdfs://10.0.86.89:9000/xjzhu/phdthesis.csv","header":"false","delimiter":",","schema":"title,author,pages"}},{"uuid":"555","name":"Merge","bundle":"cn.piflow.bundle.common.Merge","properties":{}},{"uuid":"666","name":"Fork","bundle":"cn.piflow.bundle.common.Fork","properties":{"outports":["out1","out2","out3"]}},{"uuid":"777","name":"JsonSave","bundle":"cn.piflow.bundle.json.JsonSave","properties":{"jsonSavePath":"hdfs://10.0.86.89:9000/xjzhu/phdthesis.json"}},{"uuid":"888","name":"CsvSave","bundle":"cn.piflow.bundle.csv.CsvSave","properties":{"csvSavePath":"hdfs://10.0.86.89:9000/xjzhu/phdthesis_result.csv","header":"true","delimiter":","}}],"paths":[{"from":"XmlParser","outport":"","inport":"","to":"SelectField"},{"from":"SelectField","outport":"","inport":"data1","to":"Merge"},{"from":"CsvParser","outport":"","inport":"data2","to":"Merge"},{"from":"Merge","outport":"","inport":"","to":"Fork"},{"from":"Fork","outport":"out1","inport":"","to":"PutHiveStreaming"},{"from":"Fork","outport":"out2","inport":"","to":"JsonSave"},{"from":"Fork","outport":"out3","inport":"","to":"CsvSave"}]}}""" val json =""" |{ | "flow":{ | "name":"incremental", | "uuid":"1234", | "stops":[ | { | "uuid":"1111", | "name":"MysqlReadIncremental", | "bundle":"cn.piflow.bundle.jdbc.MysqlReadIncremental", | "properties":{ | "url":"jdbc:mysql://10.0.86.90:3306/sparktest", | "sql":"select * from student where id > #~#", | "user":"root", | "password":"root", | "incrementalField":"id" | } | }, | { | "uuid":"888", | "name":"CsvSave", | "bundle":"cn.piflow.bundle.csv.CsvSave", | "properties":{ | "csvSavePath":"hdfs://10.0.86.89:9000/xjzhu/phdthesis_result.csv", | "header":"true", | "delimiter":"," | } | } | ], | "paths":[ | { | "from":"MysqlReadIncremental", | "outport":"", | "inport":"", | "to":"CsvSave" | } | ] | } |} """.stripMargin val url = "http://10.0.86.98:8001/flow/start" val timeout = 1800 val requestConfig = RequestConfig.custom() .setConnectTimeout(timeout*1000) .setConnectionRequestTimeout(timeout*1000) .setSocketTimeout(timeout*1000).build() val client = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build() val post:HttpPost = new HttpPost(url) post.addHeader("Content-Type", "application/json") post.setEntity(new StringEntity(json)) val response:CloseableHttpResponse = client.execute(post) val entity = response.getEntity val str = EntityUtils.toString(entity,"UTF-8") println("Code is " + str) } }
Example 23
Source File: HTTPClientStartFlowTextFileStreaming.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
package cn.piflow.api import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost} import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClients import org.apache.http.util.EntityUtils object HTTPClientStartFlowTextFileStreaming { def main(args: Array[String]): Unit = { //mybe you should change the directory when no data received val json = """ |{ | "flow":{ | "name":"TextFileStream", | "uuid":"1234", | "stops":[ | { | "uuid":"1111", | "name":"TextFileStream", | "bundle":"cn.piflow.bundle.streaming.TextFileStream", | "properties":{ | "directory":"hdfs://10.0.86.191:9000/xjzhu/TextFileStream3", | "batchDuration":"5" | } | | },4.2bcefghw | { | "uuid":"2222", | "name":"ConvertSchema", | "bundle":"cn.piflow.bundle.common.ConvertSchema", | "properties":{ | "schema":"value->line" | } | }, | { | "uuid":"3333", | "name":"CsvSave", | "bundle":"cn.piflow.bundle.csv.CsvSave", | "properties":{ | "csvSavePath":"hdfs://10.0.86.89:9000/xjzhu/flowStreaming", | "header":"true", | "delimiter":"," | } | } | ], | "paths":[ | { | "from":"TextFileStream", | "outport":"", | "inport":"", | "to":"ConvertSchema" | }, | { | "from":"ConvertSchema", | "outport":"", | "inport":"", | "to":"CsvSave" | } | ] | } |} """.stripMargin val url = "http://10.0.86.98:8001/flow/start" val client = HttpClients.createDefault() val post:HttpPost = new HttpPost(url) post.addHeader("Content-Type", "application/json") post.setEntity(new StringEntity(json)) val response:CloseableHttpResponse = client.execute(post) val entity = response.getEntity val str = EntityUtils.toString(entity,"UTF-8") println("Code is " + str) } }
Example 24
Source File: HTTPClientRemovePlugin.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
package cn.piflow.api import org.apache.http.client.methods.{CloseableHttpResponse, HttpPost} import org.apache.http.entity.StringEntity import org.apache.http.impl.client.HttpClients import org.apache.http.util.EntityUtils object HTTPClientRemovePlugin { def main(args: Array[String]): Unit = { val json = """{"plugin":"piflowexternal.jar"}""" val url = "http://10.0.85.83:8001/plugin/remove" val client = HttpClients.createDefault() val post:HttpPost = new HttpPost(url) post.addHeader("Content-Type", "application/json") post.setEntity(new StringEntity(json)) val response:CloseableHttpResponse = client.execute(post) val entity = response.getEntity val str = EntityUtils.toString(entity,"UTF-8") println(str) } }
Example 25
Source File: HTTPClientGetScheduleInfo.scala From piflow with BSD 2-Clause "Simplified" License | 5 votes |
package cn.piflow.api import org.apache.http.client.config.RequestConfig import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet, HttpPost} import org.apache.http.entity.StringEntity import org.apache.http.impl.client.{HttpClientBuilder, HttpClients} import org.apache.http.util.EntityUtils object HTTPClientGetScheduleInfo { def main(args: Array[String]): Unit = { val url = "http://10.0.85.83:8001/schedule/info?scheduleId=schedule_9339f584-4ec5-4e12-a51d-4214182ff63a" val client = HttpClients.createDefault() val getFlowDebugData:HttpGet = new HttpGet(url) val response:CloseableHttpResponse = client.execute(getFlowDebugData) val entity = response.getEntity val str = EntityUtils.toString(entity,"UTF-8") println("Code is " + str) } }