org.apache.commons.lang.StringUtils Scala Examples

package com.webank.wedatasphere.linkis.server.conf

import org.apache.commons.lang.StringUtils
import org.springframework.core.`type`.classreading.{MetadataReader, MetadataReaderFactory}
import org.springframework.core.`type`.filter.TypeFilter

class DataWorkCloudCustomExcludeFilter extends TypeFilter {
  private val excludePackages = ServerConfiguration.BDP_SERVER_EXCLUDE_PACKAGES.getValue.split(",").filter(StringUtils.isNotBlank)
  private val excludeClasses = ServerConfiguration.BDP_SERVER_EXCLUDE_CLASSES.getValue.split(",").filter(StringUtils.isNotBlank)
  private val excludeAnnotation = ServerConfiguration.BDP_SERVER_EXCLUDE_ANNOTATION.getValue.split(",").filter(StringUtils.isNotBlank)

  override def `match`(metadataReader: MetadataReader, metadataReaderFactory: MetadataReaderFactory): Boolean = {
    val className = metadataReader.getClassMetadata.getClassName
    excludeClasses.contains(className) ||
     excludePackages.exists(className.startsWith) ||
      excludeAnnotation.exists(metadataReader.getAnnotationMetadata.hasAnnotation)
  }
} 

package com.webank.wedatasphere.linkis.gateway.security.token

import java.io.File
import java.util.Properties
import java.util.concurrent.TimeUnit

import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.gateway.config.GatewayConfiguration._
import com.webank.wedatasphere.linkis.gateway.http.GatewayContext
import com.webank.wedatasphere.linkis.gateway.security.{GatewaySSOUtils, SecurityFilter}
import com.webank.wedatasphere.linkis.server.Message
import org.apache.commons.io.{FileUtils, IOUtils}
import org.apache.commons.lang.StringUtils


object TokenAuthentication extends Logging {

  private val (props, file) = if(ENABLE_TOKEN_AUTHENTICATION.getValue)
    (new Properties, new File(this.getClass.getClassLoader.getResource(TOKEN_AUTHENTICATION_CONFIG.getValue).toURI.getPath))
  else (null, null)
  private var lastModified = 0l

  if(ENABLE_TOKEN_AUTHENTICATION.getValue) {
    Utils.defaultScheduler.scheduleAtFixedRate(new Runnable {
      override def run(): Unit = Utils.tryAndError(init())
    }, TOKEN_AUTHENTICATION_SCAN_INTERVAL.getValue, TOKEN_AUTHENTICATION_SCAN_INTERVAL.getValue, TimeUnit.MILLISECONDS)
    init()
  }

  private def init(): Unit = if(file.lastModified() > lastModified) {
    lastModified = file.lastModified()
    info(s"loading token authentication file $file.")
    val newProps = new Properties
    val input = FileUtils.openInputStream(file)
    Utils.tryFinally(newProps.load(input))(IOUtils.closeQuietly(input))
    props.putAll(newProps)
  }

  private def validateTokenUser(token: String, tokenUser: String): Boolean = {
    val tokenUsers = props.getProperty(token)
    if(tokenUsers == "*" || (StringUtils.isNotBlank(tokenUsers) && tokenUsers.contains(tokenUser))) true
    else false
  }

  def isTokenRequest(gatewayContext: GatewayContext) : Boolean = {
    (gatewayContext.getRequest.getHeaders.containsKey(TOKEN_KEY) &&
      gatewayContext.getRequest.getHeaders.containsKey(TOKEN_USER_KEY)) || (
      gatewayContext.getRequest.getCookies.containsKey(TOKEN_KEY) &&
        gatewayContext.getRequest.getCookies.containsKey(TOKEN_USER_KEY))
  }

  def tokenAuth(gatewayContext: GatewayContext): Boolean = {
    if(!ENABLE_TOKEN_AUTHENTICATION.getValue) {
      val message = Message.noLogin(s"Gateway未启用token认证，请采用其他认证方式!") << gatewayContext.getRequest.getRequestURI
      SecurityFilter.filterResponse(gatewayContext, message)
      return false
    }
    var token = gatewayContext.getRequest.getHeaders.get(TOKEN_KEY)(0)
    var tokenUser = gatewayContext.getRequest.getHeaders.get(TOKEN_USER_KEY)(0)
    if(StringUtils.isBlank(token) || StringUtils.isBlank(tokenUser)) {
      token = gatewayContext.getRequest.getCookies.get(TOKEN_KEY)(0).getValue
      tokenUser = gatewayContext.getRequest.getCookies.get(TOKEN_USER_KEY)(0).getValue
      if(StringUtils.isBlank(token) || StringUtils.isBlank(tokenUser)) {
        val message = Message.noLogin(s"请在Header或Cookie中同时指定$TOKEN_KEY 和 $TOKEN_USER_KEY，以便完成token认证！") << gatewayContext.getRequest.getRequestURI
        SecurityFilter.filterResponse(gatewayContext, message)
        return false
      }
    }
    if(validateTokenUser(token, tokenUser)){
      info(s"Token authentication succeed, uri: ${gatewayContext.getRequest.getRequestURI}, token: $token, tokenUser: $tokenUser.")
      GatewaySSOUtils.setLoginUser(gatewayContext.getRequest, tokenUser)
      true
    } else {
      val message = Message.noLogin(s"未授权的token$token，无法将请求绑定给tokenUser$tokenUser!") << gatewayContext.getRequest.getRequestURI
      SecurityFilter.filterResponse(gatewayContext, message)
      false
    }
  }

} 

package com.webank.wedatasphere.linkis.gateway.parser

import com.webank.wedatasphere.linkis.DataWorkCloudApplication
import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.gateway.http.{GatewayContext, GatewayRoute}
import com.webank.wedatasphere.linkis.rpc.conf.RPCConfiguration
import com.webank.wedatasphere.linkis.rpc.interceptor.ServiceInstanceUtils
import com.webank.wedatasphere.linkis.server.Message
import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration
import org.apache.commons.lang.StringUtils


  protected def responseHeartbeat(gatewayContext: GatewayContext): Unit = {
    val gatewayServiceInstances = ServiceInstanceUtils.getRPCServerLoader.getServiceInstances(DataWorkCloudApplication.getApplicationName)
    val msg = Message.ok("Gateway heartbeat ok!").data("gatewayList", gatewayServiceInstances.map(_.getInstance)).data("isHealthy", true)
    sendMessageResponse(msg, gatewayContext)
  }
}
object AbstractGatewayParser {
  val GATEWAY_HEART_BEAT_URL = Array("gateway", "heartbeat")
}
class DefaultGatewayParser(gatewayParsers: Array[GatewayParser]) extends AbstractGatewayParser {

  private val COMMON_REGEX = "/api/rest_[a-zA-Z]+/(v\\d+)/([^/]+)/.+".r
  private val CLIENT_HEARTBEAT_REGEX = s"/api/rest_[a-zA-Z]+/(v\\d+)/${AbstractGatewayParser.GATEWAY_HEART_BEAT_URL.mkString("/")}".r

  override def shouldContainRequestBody(gatewayContext: GatewayContext): Boolean = gatewayContext.getRequest.getMethod.toUpperCase != "GET" &&
    (gatewayContext.getRequest.getRequestURI match {
    case uri if uri.startsWith(ServerConfiguration.BDP_SERVER_USER_URI.getValue) => true
    case _ => gatewayParsers.exists(_.shouldContainRequestBody(gatewayContext))
  })

  override def parse(gatewayContext: GatewayContext): Unit = {
    val path = gatewayContext.getRequest.getRequestURI
    if(gatewayContext.getGatewayRoute == null) {
      gatewayContext.setGatewayRoute(new GatewayRoute)
      gatewayContext.getGatewayRoute.setRequestURI(path)
    }
    gatewayParsers.foreach(_.parse(gatewayContext))
    if(gatewayContext.getGatewayRoute.getServiceInstance == null) path match {
      case CLIENT_HEARTBEAT_REGEX(version) =>
        if(sendResponseWhenNotMatchVersion(gatewayContext, version)) return
        info(gatewayContext.getRequest.getRemoteAddress + " try to heartbeat.")
        responseHeartbeat(gatewayContext)
      case COMMON_REGEX(version, serviceId) =>
        if(sendResponseWhenNotMatchVersion(gatewayContext, version)) return
        val applicationName = if(RPCConfiguration.ENABLE_PUBLIC_SERVICE.getValue && RPCConfiguration.PUBLIC_SERVICE_LIST.contains(serviceId))
          RPCConfiguration.PUBLIC_SERVICE_APPLICATION_NAME.getValue else serviceId
        gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(applicationName, null))
      case p if p.startsWith("/dws/") =>
        //TODO add version support
        val params = gatewayContext.getGatewayRoute.getParams
        params.put("proxyId", "dws")
        val secondaryProxyId = StringUtils.substringBetween(p, "/dws/", "/")
        if(StringUtils.isNotBlank(secondaryProxyId)){
          params.put("proxyId", "dws/" + secondaryProxyId)
        }
        gatewayContext.getGatewayRoute.setParams(params)
      case _ =>
        sendErrorResponse(s"Cannot find a service to deal $path, ignore it.", gatewayContext)
    }
  }
} 

package com.webank.wedatasphere.linkis.gateway.route

import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.gateway.exception.TooManyServiceException
import com.webank.wedatasphere.linkis.gateway.http.GatewayContext
import com.webank.wedatasphere.linkis.rpc.interceptor.ServiceInstanceUtils
import com.webank.wedatasphere.linkis.rpc.sender.SpringCloudFeignConfigurationCache
import com.webank.wedatasphere.linkis.server.Message
import com.webank.wedatasphere.linkis.server.exception.NoApplicationExistsException
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.exception.ExceptionUtils


trait GatewayRouter {

  def route(gatewayContext: GatewayContext): ServiceInstance

}
abstract class AbstractGatewayRouter extends GatewayRouter with Logging {
  import scala.collection.JavaConversions._
  protected def findAndRefreshIfNotExists(serviceId: String, findService: => Option[String]): Option[String] = {
    var service = findService
    if(service.isEmpty) {
      val applicationNotExists = new NoApplicationExistsException(10050, "Application " +
        serviceId + " is not exists any instances.")
      Utils.tryThrow(Utils.waitUntil(() =>{
        ServiceInstanceUtils.refreshServiceInstances()
        service = findService
        service.nonEmpty
      }, ServiceInstanceUtils.serviceRefreshMaxWaitTime(), 500, 2000)){ t =>
        warn(s"Need a random $serviceId instance, but no one can find in Discovery refresh.", t)
        applicationNotExists.initCause(t)
        applicationNotExists
      }
    }
    service
  }
  protected def findService(parsedServiceId: String, tooManyDeal: List[String] => Option[String]): Option[String] = {
    val services = SpringCloudFeignConfigurationCache.getDiscoveryClient
      .getServices.filter(_.toLowerCase.contains(parsedServiceId.toLowerCase)).toList
    if(services.length == 1) Some(services.head)
    else if(services.length > 1) tooManyDeal(services)
    else None
  }
}

class DefaultGatewayRouter(gatewayRouters: Array[GatewayRouter]) extends AbstractGatewayRouter{

  private def findCommonService(parsedServiceId: String) = findService(parsedServiceId, services => {
    val errorMsg = new TooManyServiceException(s"Cannot find a correct serviceId for parsedServiceId $parsedServiceId, service list is: " + services)
    warn("", errorMsg)
    throw errorMsg
  })

  protected def findReallyService(gatewayContext: GatewayContext): ServiceInstance = {
    var serviceInstance: ServiceInstance = null
    for (router <- gatewayRouters if serviceInstance == null) {
      serviceInstance = router.route(gatewayContext)
    }
    if(serviceInstance == null) serviceInstance = gatewayContext.getGatewayRoute.getServiceInstance
    val service = findAndRefreshIfNotExists(serviceInstance.getApplicationName,
      findCommonService(serviceInstance.getApplicationName))
    service.map { applicationName =>
      if(StringUtils.isNotBlank(serviceInstance.getInstance)) {
        val _serviceInstance = ServiceInstance(applicationName, serviceInstance.getInstance)
        ServiceInstanceUtils.getRPCServerLoader.getOrRefreshServiceInstance(_serviceInstance)
        _serviceInstance
      } else ServiceInstance(applicationName, null)
    }.get
  }
  override def route(gatewayContext: GatewayContext): ServiceInstance = if(gatewayContext.getGatewayRoute.getServiceInstance != null) {
    val parsedService = gatewayContext.getGatewayRoute.getServiceInstance.getApplicationName
    val serviceInstance = Utils.tryCatch(findReallyService(gatewayContext)){ t =>
      val message = Message.error(ExceptionUtils.getRootCauseMessage(t)) << gatewayContext.getRequest.getRequestURI
      message.data("data", gatewayContext.getRequest.getRequestBody)
      warn("", t)
      if(gatewayContext.isWebSocketRequest) gatewayContext.getResponse.writeWebSocket(message) else gatewayContext.getResponse.write(message)
      gatewayContext.getResponse.sendResponse()
      return null
    }
    info("GatewayRouter route requestUri " + gatewayContext.getGatewayRoute.getRequestURI + " with parsedService " + parsedService + " to " + serviceInstance)
    serviceInstance
  } else null
} 

package com.webank.wedatasphere.linkis.metadata.ddl

import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.metadata.conf.MdqConfiguration
import com.webank.wedatasphere.linkis.metadata.domain.mdq.bo.{MdqTableBO, MdqTableFieldsInfoBO}
import com.webank.wedatasphere.linkis.metadata.exception.MdqIllegalParamException
import org.apache.commons.lang.StringUtils

import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer

object ScalaDDLCreator extends DDLCreator with SQLConst with Logging{



  override def createDDL(tableInfo:MdqTableBO, user:String): String = {
    logger.info(s"begin to generate ddl for user $user using ScalaDDLCreator")
    val dbName = tableInfo.getTableBaseInfo.getBase.getDatabase
    val tableName = tableInfo.getTableBaseInfo.getBase.getName
    val fields = tableInfo.getTableFieldsInfo
    val createTableCode = new StringBuilder
    createTableCode.append(SPARK_SQL).append(LEFT_PARENTHESES).append(MARKS).append(CREATE_TABLE)
    createTableCode.append(dbName).append(".").append(tableName)
    createTableCode.append(LEFT_PARENTHESES)
    val partitions = new ArrayBuffer[MdqTableFieldsInfoBO]()
    val fieldsArray = new ArrayBuffer[String]()
    fields foreach {
      field =>
        if (field.getPartitionField != null && field.getPartitionField == true) partitions += field else{
          val name = field.getName
          val _type = field.getType
          val desc = field.getComment
          if (StringUtils.isNotEmpty(desc)){
            fieldsArray += (name + SPACE + _type + SPACE + COMMENT + SPACE + SINGLE_MARK + desc + SINGLE_MARK)
          }else{
            fieldsArray += (name + SPACE + _type)
          }
        }
    }
    createTableCode.append(fieldsArray.mkString(COMMA)).append(RIGHT_PARENTHESES).append(SPACE)
    if (partitions.nonEmpty){
      val partitionArr = new ArrayBuffer[String]()
      partitions foreach {
        p => val name = p.getName
          val _type = p.getType
          if (StringUtils.isEmpty(name) || StringUtils.isEmpty(_type)) throw MdqIllegalParamException("partition name or type is null")
          partitionArr += (name + SPACE + _type)
      }
      createTableCode.append(PARTITIONED_BY).append(LEFT_PARENTHESES).append(partitionArr.mkString(COMMA)).
        append(RIGHT_PARENTHESES).append(SPACE)
    }
    //如果是分区表，但是没有分区字段，默认是用ds做分区
    if(partitions.isEmpty && tableInfo.getTableBaseInfo.getBase.getPartitionTable){
      val partition = MdqConfiguration.DEFAULT_PARTITION_NAME.getValue
      val _type = "string"
      createTableCode.append(PARTITIONED_BY).append(LEFT_PARENTHESES).append(partition).append(SPACE).append(_type).
        append(RIGHT_PARENTHESES).append(SPACE)
    }
    createTableCode.append(STORED_AS).append(SPACE).append(MdqConfiguration.DEFAULT_STORED_TYPE.getValue).append(SPACE)
    createTableCode.append(MARKS)
    createTableCode.append(RIGHT_PARENTHESES)
    val finalCode = createTableCode.toString()
    logger.info(s"End to create ddl code, code is $finalCode")
    finalCode
  }

  def main(args: Array[String]): Unit = {
    val filePath = "E:\\data\\json\\data.json"
    val json = scala.io.Source.fromFile(filePath).mkString
    println(json)

   // val obj = new Gson().fromJson(json, classOf[MdqTableVO])
    //val sql = createDDL(obj, "hadoop")
    //println(System.currentTimeMillis())
    //println(sql)
  }


} 

package com.webank.wedatasphere.linkis.cs.storage

import java.util.Date

import com.webank.wedatasphere.linkis.common.io.FsPath
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.cs.client.service.CSTableService
import com.webank.wedatasphere.linkis.cs.common.entity.metadata.{CSColumn, CSTable}
import com.webank.wedatasphere.linkis.storage.resultset.StorageResultSetWriter
import com.webank.wedatasphere.linkis.storage.resultset.table.{TableMetaData, TableRecord, TableResultSet}
import com.webank.wedatasphere.linkis.storage.utils.StorageUtils
import org.apache.commons.lang.StringUtils


class CSTableResultSetWriter(tableResult: TableResultSet, maxCacheSize: Long,
                             storePath: FsPath, contextIDStr: String, nodeName: String, alias: String) extends StorageResultSetWriter[TableMetaData, TableRecord](tableResult, maxCacheSize, storePath) with Logging {

  override def toString: String = {
    try {
      registerToCS
    } catch {
      case t: Throwable =>
        info("Failed to register tmp table", t)
    }
    super.toString
  }

  private def registerToCS: Unit = {

    if (StringUtils.isNotBlank(contextIDStr) && StringUtils.isNotBlank(nodeName) && !isEmpty) {
      info("Start to register resultSet to cs")
      flush()
      val csTable = new CSTable
      csTable.setAlias(alias)
      csTable.setAvailable(true)
      csTable.setComment("cs temp table")
      csTable.setCreateTime(new Date())
      csTable.setCreator(StorageUtils.getJvmUser)
      csTable.setExternalUse(true)
      csTable.setImport(false)
      csTable.setLocation(toFSPath.getSchemaPath)
      csTable.setPartitionTable(false)
      csTable.setView(true)
      val csColumns = getMetaData.asInstanceOf[TableMetaData].columns.map { column =>
        val csColumn = new CSColumn
        csColumn.setName(column.columnName)
        csColumn.setType(column.dataType.typeName)
        csColumn.setComment(column.comment)
        csColumn
      }
      csTable.setColumns(csColumns)
      CSTableService.getInstance().registerCSTable(contextIDStr, nodeName, alias, csTable)
      info("Finished to register resultSet to cs")
    }
  }
} 

package com.webank.wedatasphere.linkis.storage.resultset

import java.util

import com.webank.wedatasphere.linkis.common.io.resultset.ResultSet
import com.webank.wedatasphere.linkis.common.io.{FsPath, MetaData, Record}
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.storage.FSFactory
import com.webank.wedatasphere.linkis.storage.domain.Dolphin
import com.webank.wedatasphere.linkis.storage.exception.{StorageErrorException, StorageWarnException}
import com.webank.wedatasphere.linkis.storage.utils.{StorageConfiguration, StorageUtils}
import org.apache.commons.lang.StringUtils


class DefaultResultSetFactory extends ResultSetFactory with Logging{

  private val resultClasses: Map[String, Class[ResultSet[ResultMetaData, ResultRecord]]] = StorageUtils.loadClasses(
    StorageConfiguration.STORAGE_RESULT_SET_CLASSES.getValue,
    StorageConfiguration.STORAGE_RESULT_SET_PACKAGE.getValue, t => t.newInstance().resultSetType().toLowerCase)

  val resultTypes = ResultSetFactory.resultSetType.keys.toArray

  override def getResultSetByType(resultSetType: String): ResultSet[_ <: MetaData, _ <: Record] = {
    if(! resultClasses.contains(resultSetType)) throw new StorageErrorException(50000, s"Unsupported result type(不支持的结果类型)：$resultSetType" )
    resultClasses(resultSetType).newInstance()
  }

  override def getResultSetByPath(fsPath: FsPath): ResultSet[_ <: MetaData, _ <: Record] = {
    getResultSetByPath(fsPath,StorageUtils.getJvmUser)
  }

  override def getResultSetByContent(content: String): ResultSet[_ <: MetaData, _ <: Record] = {
    getResultSetByType(Dolphin.getType(content))
  }

  override def exists(resultSetType: String): Boolean = resultClasses.contains(resultSetType)

  override def isResultSetPath(path: String): Boolean = {
    path.endsWith(Dolphin.DOLPHIN_FILE_SUFFIX)
  }

  override def isResultSet(content: String): Boolean =  Utils.tryCatch(resultClasses.contains(Dolphin.getType(content))){ t =>
    info("Wrong result Set: " + t.getMessage)
    false
  }

  override def getResultSet(output: String): ResultSet[_ <: MetaData, _ <: Record] = getResultSet(output,StorageUtils.getJvmUser)

  override def getResultSetType:Array[String] = resultTypes

  override def getResultSetByPath(fsPath: FsPath, proxyUser: String): ResultSet[_ <: MetaData, _ <: Record] = {
    if(fsPath == null ) return null
    info("Get Result Set By Path:" + fsPath.getPath)
    val fs = FSFactory.getFsByProxyUser(fsPath,proxyUser)
    fs.init(new util.HashMap[String,String]())
    val inputStream = fs.read(fsPath)
    val resultSetType = Dolphin.getType(inputStream)
    if(StringUtils.isEmpty(resultSetType)) throw new StorageWarnException(51000, s"The file (${fsPath.getPath}) is empty(文件(${fsPath.getPath}) 为空)")
    Utils.tryQuietly(inputStream.close())
    Utils.tryQuietly(fs.close())
    getResultSetByType(resultSetType)
  }

  override def getResultSet(output: String, proxyUser: String): ResultSet[_ <: MetaData, _ <: Record] = {
    if(isResultSetPath(output)) {
      getResultSetByPath(new FsPath(output), proxyUser)
    } else if (isResultSet(output)) {
      getResultSetByContent(output)
    }
    else null
  }
} 

package com.webank.wedatasphere.linkis.httpclient.authentication

import java.util.concurrent.ConcurrentHashMap

import com.webank.wedatasphere.linkis.httpclient.Client
import com.webank.wedatasphere.linkis.httpclient.config.ClientConfig
import com.webank.wedatasphere.linkis.httpclient.request.{Action, UserAction}
import org.apache.commons.lang.StringUtils
import org.apache.http.HttpResponse


abstract class AbstractAuthenticationStrategy extends AuthenticationStrategy {
  private var client: Client = _
  private val userNameToAuthentications = new ConcurrentHashMap[String, Authentication]()
  private var clientConfig: ClientConfig = _
  protected val sessionMaxAliveTime: Long

  def setClient(client: Client): Unit = this.client = client
  def getClient: Client = client

  protected def getKeyByUserAndURL(user: String, serverUrl: String): String = user + "@" + serverUrl

  protected def getKey(requestAction: Action, serverUrl: String): String = requestAction match {
    case _: AuthenticationAction => null
    case authAction: UserAction =>
      getKeyByUserAndURL(authAction.getUser, serverUrl)
    case _ if StringUtils.isNotBlank(clientConfig.getAuthTokenKey) =>
      getKeyByUserAndURL(clientConfig.getAuthTokenKey, serverUrl)
    case _ => null
  }

  def setClientConfig(clientConfig: ClientConfig): Unit = this.clientConfig = clientConfig
  def getClientConfig: ClientConfig = clientConfig

  def login(requestAction: Action, serverUrl: String): Authentication = {
    val key = getKey(requestAction, serverUrl)
    if(key == null) return null
    if(userNameToAuthentications.containsKey(key) && !isTimeout(userNameToAuthentications.get(key))) {
      val authenticationAction = userNameToAuthentications.get(key)
      authenticationAction.updateLastAccessTime()
      authenticationAction
    } else key.intern() synchronized {
      var authentication = userNameToAuthentications.get(key)
      if(authentication == null || isTimeout(authentication)) {
        authentication = tryLogin(requestAction, serverUrl)
        userNameToAuthentications.put(key, authentication)
      }
      authentication
    }
  }

  def tryLogin(requestAction: Action, serverUrl: String): Authentication = {
    val action = getAuthenticationAction(requestAction, serverUrl)
    client.execute(action, 5000) match {
      case r: AuthenticationResult => r.getAuthentication
    }
  }

  protected def getAuthenticationAction(requestAction: Action, serverUrl: String): AuthenticationAction

  def getAuthenticationResult(response: HttpResponse, requestAction: AuthenticationAction): AuthenticationResult

  def isTimeout(authentication: Authentication): Boolean = System.currentTimeMillis() - authentication.getLastAccessTime >= sessionMaxAliveTime
} 

package com.webank.wedatasphere.linkis.rpc.instancealias.impl

import java.util.Base64
import java.util.regex.Pattern

import com.webank.wedatasphere.linkis.rpc.instancealias.InstanceAliasConverter
import org.apache.commons.lang.StringUtils
import org.springframework.stereotype.Component


@Component
class DefaultInstanceAliasConverter extends InstanceAliasConverter  {

  val pattern = Pattern.compile("[a-zA-Z\\d=\\+/]+")

  // todo use base64 for the moment
  override def instanceToAlias(instance: String): String = {
    new String(Base64.getEncoder.encode(instance.getBytes()))
  }

  override def aliasToInstance(alias: String): String = {
    new String(Base64.getDecoder.decode(alias))
  }

  override def checkAliasFormatValid(alias: String): Boolean = {
    if (StringUtils.isBlank(alias)) {
      return false
    }
    val matcher = pattern.matcher(alias)
    matcher.find()
  }
} 

package com.webank.wedatasphere.linkis.rpc.interceptor.common

import com.netflix.loadbalancer.ILoadBalancer
import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.protocol.{Protocol, SingleInstanceProtocol}
import com.webank.wedatasphere.linkis.rpc.interceptor.RPCLoadBalancer
import org.apache.commons.lang.StringUtils
import org.springframework.stereotype.Component


@Component
class SingleInstanceRPCLoadBalancer extends RPCLoadBalancer with Logging {
  override val order: Int = 20

  override def choose(protocol: Protocol, originService: ServiceInstance, lb: ILoadBalancer): Option[ServiceInstance] = protocol match {
    case _: SingleInstanceProtocol =>
      if(StringUtils.isEmpty(originService.getInstance)) synchronized {
        if(StringUtils.isEmpty(originService.getInstance)) {
          val servers = lb.getAllServers
          val server = servers.get((math.random * servers.size()).toInt)
          originService.setInstance(server.getHostPort)
          warn(originService.getApplicationName + " choose " + server.getHostPort + " to build a single instance connection.")
        }
      }
      Some(originService)
    case _ => None
  }
} 

package com.webank.wedatasphere.linkis.rpc.interceptor.common

import java.net.ConnectException

import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.common.exception.DWCRetryException
import com.webank.wedatasphere.linkis.common.utils.RetryHandler
import com.webank.wedatasphere.linkis.protocol.RetryableProtocol
import com.webank.wedatasphere.linkis.rpc.exception.{DWCRPCRetryException, NoInstanceExistsException}
import com.webank.wedatasphere.linkis.rpc.interceptor.{RPCInterceptor, RPCInterceptorChain, RPCInterceptorExchange, ServiceInstanceRPCInterceptorChain}
import com.webank.wedatasphere.linkis.rpc.utils.RPCUtils
import feign.RetryableException
import org.apache.commons.lang.StringUtils
import org.springframework.stereotype.Component


@Component
class RetryableRPCInterceptor extends RPCInterceptor {
  override val order: Int = 20

//  private val commonRetryHandler = new RPCRetryHandler
//  commonRetryHandler.setRetryInfo(new RetryableProtocol{})
//
//  private def isCommonRetryHandler(retry: RetryableProtocol): Boolean = retry.maxPeriod == commonRetryHandler.getRetryMaxPeriod &&
//    retry.period == commonRetryHandler.getRetryPeriod && retry.retryNum == commonRetryHandler.getRetryNum &&
//    (retry.retryExceptions.isEmpty || commonRetryHandler.getRetryExceptions.containsSlice(retry.retryExceptions))

  override def intercept(interceptorExchange: RPCInterceptorExchange, chain: RPCInterceptorChain): Any = interceptorExchange.getProtocol match {
    case retry: RetryableProtocol =>
      val retryName = retry.getClass.getSimpleName
//      if(isCommonRetryHandler(retry)) commonRetryHandler.retry(chain.handle(interceptorExchange), retryName)
//      else {
        val retryHandler = new RPCRetryHandler
        retryHandler.setRetryInfo(retry, chain)
        retryHandler.retry(chain.handle(interceptorExchange), retryName)
//      }
    case _ => chain.handle(interceptorExchange)
  }

  class RPCRetryHandler extends RetryHandler {
    addRetryException(classOf[ConnectException])
    addRetryException(classOf[RetryableException])
    private var serviceInstance: Option[ServiceInstance] = None
    def setRetryInfo(retry: RetryableProtocol, chain: RPCInterceptorChain): Unit ={
      setRetryNum(retry.retryNum)
      setRetryPeriod(retry.period)
      setRetryMaxPeriod(retry.maxPeriod)
      retry.retryExceptions.foreach(addRetryException)
      chain match {
        case s: ServiceInstanceRPCInterceptorChain => serviceInstance = Option(s.getServiceInstance)
        case _ =>
      }
    }

    private def isNoServiceException(t: Throwable): Boolean = RPCUtils.isReceiverNotExists(t)

    override def exceptionCanRetry(t: Throwable): Boolean = t match {
      case _: DWCRPCRetryException => true
      case r: DWCRetryException => r.getErrCode == DWCRPCRetryException.RPC_RETRY_ERROR_CODE
      case _ => (serviceInstance.exists(s => StringUtils.isBlank(s.getInstance)) && isNoServiceException(t)) || super.exceptionCanRetry(t)
    }
  }
}
object RetryableRPCInterceptor {
  def isRetryableProtocol(message: Any): Boolean = message match {
    case protocol: RetryableProtocol => true
    case _ => false
  }
} 

package com.webank.wedatasphere.linkis.rpc.utils

import java.lang.reflect.UndeclaredThrowableException
import java.net.ConnectException

import com.netflix.client.ClientException
import com.webank.wedatasphere.linkis.rpc.exception.NoInstanceExistsException
import com.webank.wedatasphere.linkis.rpc.sender.SpringCloudFeignConfigurationCache
import feign.RetryableException
import org.apache.commons.lang.StringUtils
import scala.collection.JavaConversions._


object RPCUtils {

  def isReceiverNotExists(t: Throwable): Boolean = t match {
    case connect: ConnectException => connect.getMessage != null && connect.getMessage.contains("Connection refused")
    case _: NoInstanceExistsException => true
    case t: UndeclaredThrowableException =>
      t.getCause match {
        case _: NoInstanceExistsException => true
        case _ => false
      }
    case t: RetryableException => t.getCause match {
      case connect: ConnectException =>
        connect.getMessage != null && connect.getMessage.contains("Connection refused")
      case _ => false
    }
    case t: RuntimeException => t.getCause match {
      case client: ClientException => StringUtils.isNotBlank(client.getErrorMessage) &&
        client.getErrorMessage.contains("Load balancer does not have available server for client")
      case _ => false
    }
    case _ => false
  }

  def findService(parsedServiceId: String, tooManyDeal: List[String] => Option[String]): Option[String] = {
    val services = SpringCloudFeignConfigurationCache.getDiscoveryClient
      .getServices.filter(_.toLowerCase.contains(parsedServiceId.toLowerCase)).toList
    if(services.length == 1) Some(services.head)
    else if(services.length > 1) tooManyDeal(services)
    else None
  }
} 

package com.webank.wedatasphere.linkis.rpc.sender

import java.lang.reflect.Field

import com.netflix.client.ClientRequest
import com.netflix.client.config.IClientConfig
import com.netflix.loadbalancer.reactive.LoadBalancerCommand
import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.common.conf.{Configuration => DWCConfiguration}
import com.webank.wedatasphere.linkis.protocol.Protocol
import com.webank.wedatasphere.linkis.rpc.interceptor.{BaseRPCInterceptorChain, RPCInterceptor, RPCLoadBalancer, ServiceInstanceRPCInterceptorChain}
import com.webank.wedatasphere.linkis.rpc.transform.RPCConsumer
import com.webank.wedatasphere.linkis.rpc.{BaseRPCSender, RPCMessageEvent, RPCSpringBeanCache}
import com.webank.wedatasphere.linkis.server.{BDPJettyServerHelper, Message}
import feign._
import org.apache.commons.lang.StringUtils
import org.springframework.cloud.netflix.ribbon.ServerIntrospector
import org.springframework.cloud.openfeign.ribbon.{CachingSpringLoadBalancerFactory, FeignLoadBalancer, LoadBalancerFeignClient}


  override def deliver(message: Any): Unit = getRPCSenderListenerBus.post(RPCMessageEvent(message, serviceInstance))

  override def equals(obj: Any): Boolean = if(obj == null) false
    else obj match {
      case sender: SpringMVCRPCSender => sender.serviceInstance == serviceInstance
      case _ => false
    }

  override def hashCode(): Int = serviceInstance.hashCode()

  override val toString: String = if(StringUtils.isBlank(serviceInstance.getInstance)) s"RPCSender(${serviceInstance.getApplicationName})"
    else s"RPCSender($getApplicationName, ${serviceInstance.getInstance})"
}
private object SpringMVCRPCSender {
  private var requestField: Field = _
  def getRequest(req: ClientRequest): Request = {
    if(requestField == null) synchronized {
      if(requestField == null) {
        requestField = req.getClass.getDeclaredField("request")
        requestField.setAccessible(true)
      }
    }
    requestField.get(req).asInstanceOf[Request]
  }
} 

package com.webank.wedatasphere.linkis.rpc.conf

import com.netflix.discovery.EurekaClient
import com.webank.wedatasphere.linkis.DataWorkCloudApplication
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.rpc.RPCReceiveRestful
import com.webank.wedatasphere.linkis.rpc.interceptor.RPCServerLoader
import com.webank.wedatasphere.linkis.rpc.sender.eureka.EurekaRPCServerLoader
import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration
import org.apache.commons.lang.StringUtils
import org.springframework.boot.autoconfigure.condition.{ConditionalOnClass, ConditionalOnMissingBean}
import org.springframework.boot.context.event.ApplicationPreparedEvent
import org.springframework.cloud.openfeign.EnableFeignClients
import org.springframework.context.annotation.{Bean, Configuration}
import org.springframework.context.event.EventListener


@Configuration
@EnableFeignClients
class RPCSpringConfiguration extends Logging {

  @Bean(Array("rpcServerLoader"))
  @ConditionalOnClass(Array(classOf[EurekaClient]))
  @ConditionalOnMissingBean
  def createRPCServerLoader(): RPCServerLoader = new EurekaRPCServerLoader

  @EventListener
  def completeInitialize(applicationPreparedEvent: ApplicationPreparedEvent): Unit = {
    val restfulClasses = ServerConfiguration.BDP_SERVER_RESTFUL_REGISTER_CLASSES.getValue
    if(StringUtils.isEmpty(restfulClasses))
      DataWorkCloudApplication.setProperty(ServerConfiguration.BDP_SERVER_RESTFUL_REGISTER_CLASSES.key, classOf[RPCReceiveRestful].getName)
    else
      DataWorkCloudApplication.setProperty(ServerConfiguration.BDP_SERVER_RESTFUL_REGISTER_CLASSES.key, restfulClasses +
        "," + classOf[RPCReceiveRestful].getName)
    info("DataWorkCloud RPC need register RPCReceiveRestful, now add it to configuration.")
  }

} 

package com.webank.wedatasphere.linkis.server.socket.controller

import com.webank.wedatasphere.linkis.common.listener.ListenerEventBus
import com.webank.wedatasphere.linkis.server.exception.BDPServerErrorException
import org.apache.commons.lang.StringUtils


  override protected def doPostEvent(listener: ServerEventService, event: SocketServerEvent): Unit = {
    val serverEvent = event.serverEvent
    if(StringUtils.isEmpty(serverEvent.getMethod)) info("ignore empty method with " + serverEvent.getData)
    else if(serverEvent.getMethod.startsWith(listener.serviceName)) {
      val response = listener.onEvent(serverEvent)
      if(response != null) {
        response.setMethod(serverEvent.getMethod)
        event.socket.sendMessage(response)
      }
    }
  }

  override protected val dropEvent: DropEvent = new DropEvent {
    override def onDropEvent(event: SocketServerEvent): Unit = throw new BDPServerErrorException(11035, "WebSocket consumer has stopped, please contact the administrator to handle!(WebSocket的消费器已停止，请联系管理员处理！)")
    override def onBusStopped(event: SocketServerEvent): Unit = throw new BDPServerErrorException(11005, "The receive queue for WebSocket is full, please try again later!(WebSocket的接收队列已满，请稍后重试！)")
  }
} 

package com.webank.wedatasphere.linkis.server.socket

import java.util
import java.util.concurrent.atomic.AtomicInteger

import com.webank.wedatasphere.linkis.common.conf.Configuration.DEFAULT_DATE_PATTERN
import com.webank.wedatasphere.linkis.common.listener.Event
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.server.Message
import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration._
import com.webank.wedatasphere.linkis.server.exception.BDPServerErrorException
import com.webank.wedatasphere.linkis.server.socket.controller.{ServerListenerEventBus, SocketServerEvent}
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.exception.ExceptionUtils
import org.apache.commons.lang.time.DateFormatUtils
import org.eclipse.jetty.websocket.servlet._

import scala.collection.JavaConversions._


private[server] class ControllerServer(serverListenerEventBus: ServerListenerEventBus)
  extends WebSocketServlet with SocketListener
   with Event with Logging {

  private val socketList = new util.HashMap[Int, ServerSocket](BDP_SERVER_SOCKET_QUEUE_SIZE.getValue)
  private val idGenerator = new AtomicInteger(0)

  override def configure(webSocketServletFactory: WebSocketServletFactory): Unit = {
    webSocketServletFactory.setCreator(new WebSocketCreator {
      override def createWebSocket(servletUpgradeRequest: ServletUpgradeRequest,
                                   servletUpgradeResponse: ServletUpgradeResponse): AnyRef =
        ServerSocket(servletUpgradeRequest.getHttpServletRequest, ControllerServer.this)
    })
  }

  def sendMessage(id: Int, message: Message): Unit = {
    val socket = socketList.get(id)
    if(socket == null) throw new BDPServerErrorException(11004, s"ServerSocket($id) does not exist!(ServerSocket($id)不存在！)")
    socket.sendMessage(message)
  }

  def sendMessageToAll(message: Message): Unit =
    socketList.values().foreach(_.sendMessage(message))

  def sendMessageToUser(user: String, message: Message): Unit =
    socketList.values().filter(s => s != null && s.user.contains(user)).foreach(_.sendMessage(message))

  override def onClose(socket: ServerSocket, code: Int, message: String): Unit = {
    val date = DateFormatUtils.format(socket.createTime, DEFAULT_DATE_PATTERN.getValue)
    if(!socketList.containsKey(socket.id))
      warn(s"$socket created at $date has expired, ignore the close function!")
    else {
      info(s"$socket closed at $date with code $code and message: " + message)
      socketList synchronized {
        if(socketList.containsKey(socket.id)) socketList.remove(socket.id)
      }
    }
  }

  override def onOpen(socket: ServerSocket): Unit = socketList synchronized {
    val index = idGenerator.getAndIncrement()
    socket.id = index
    socketList.put(index, socket)
    info(s"open a new $socket with id $index for user ${socket.user.orNull}!")
  }

  override def onMessage(socket: ServerSocket, message: String): Unit = {
    if(StringUtils.isBlank(message)) {
      socket.sendMessage(Message.error("Empty message!"))
      return
    }
    val socketServerEvent = Utils.tryCatch(new SocketServerEvent(socket, message)){ t =>
      warn("parse message failed!", t)
      socket.sendMessage(Message.error(ExceptionUtils.getRootCauseMessage(t), t))
      return
    }
    if(socket.user.isEmpty && socketServerEvent.serverEvent.getMethod != BDP_SERVER_SOCKET_LOGIN_URI.getValue) {
      socket.sendMessage(Message.noLogin("You are not logged in, please login first!(您尚未登录，请先登录!)").data("websocketTag", socketServerEvent.serverEvent.getWebsocketTag) << socketServerEvent.serverEvent.getMethod)
    } else Utils.tryCatch(serverListenerEventBus.post(socketServerEvent)){
      case t: BDPServerErrorException => Message.error(t.getMessage, t).data("websocketTag", socketServerEvent.serverEvent.getWebsocketTag) << socketServerEvent.serverEvent.getMethod
    }
  }
} 

package com.webank.wedatasphere.linkis.gateway.security

import java.util.Properties
import java.util.concurrent.TimeUnit

import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.gateway.config.GatewayConfiguration._
import org.apache.commons.lang.StringUtils

object ProxyUserUtils extends Logging {

  private val props = new Properties
  if(ENABLE_PROXY_USER.getValue){
    Utils.defaultScheduler.scheduleAtFixedRate(new Runnable {
      override def run(): Unit = {
        info("loading proxy users.")
        val newProps = new Properties
        newProps.load(this.getClass.getResourceAsStream(PROXY_USER_CONFIG.getValue))
        props.clear()
        props.putAll(newProps)
      }
    }, 0, PROXY_USER_SCAN_INTERVAL.getValue, TimeUnit.MILLISECONDS)
  }

  def getProxyUser(umUser: String): String = if(ENABLE_PROXY_USER.getValue) {
    val proxyUser = props.getProperty(umUser)
    if(StringUtils.isBlank(proxyUser)) umUser else {
      info(s"switched to proxy user $proxyUser for umUser $umUser.")
      proxyUser
    }
  } else umUser

} 

package com.webank.wedatasphere.linkis

import java.util

import javax.servlet.http.HttpServletRequest
import com.webank.wedatasphere.linkis.common.exception.{ErrorException, ExceptionManager, FatalException, WarnException}
import com.webank.wedatasphere.linkis.common.utils.Utils
import com.webank.wedatasphere.linkis.server.exception.{BDPServerErrorException, NonLoginException}
import com.webank.wedatasphere.linkis.server.security.SecurityFilter
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.exception.ExceptionUtils
import org.slf4j.Logger

import scala.collection.{JavaConversions, mutable}


package object server {

  val EXCEPTION_MSG = "errorMsg"
  type JMap[K, V] = java.util.HashMap[K, V]

  implicit def getUser(req: HttpServletRequest): String = SecurityFilter.getLoginUsername(req)

  def validateFailed(message: String): Message = Message(status = 2).setMessage(message)
  def validate[T](json: util.Map[String, T], keys: String*): Unit = {
    keys.foreach(k => if(!json.contains(k) || json.get(k) == null || StringUtils.isEmpty(json.get(k).toString))
      throw new BDPServerErrorException(11001, s"Verification failed, $k cannot be empty!(验证失败，$k 不能为空！)"))
  }
  def error(message: String): Message = Message.error(message)
  implicit def ok(msg: String): Message = Message.ok(msg)
  implicit def error(t: Throwable): Message = Message.error(t)
  implicit def error(e: (String, Throwable)): Message = Message.error(e)
  implicit def error(msg: String, t: Throwable): Message = Message.error(msg -> t)
  //  def tryCatch[T](tryOp: => T)(catchOp: Throwable => T): T = Utils.tryCatch(tryOp)(catchOp)
//  def tryCatch(tryOp: => Message)(catchOp: Throwable => Message): Message = Utils.tryCatch(tryOp){
//    case nonLogin: NonLoginException => Message.noLogin(msg = nonLogin.getMessage)
//    case t => catchOp(t)
//  }
  def catchMsg(tryOp: => Message)(msg: String)(implicit log: Logger): Message = Utils.tryCatch(tryOp){
    case fatal: FatalException =>
      log.error("Fatal Error, system exit...", fatal)
      System.exit(fatal.getErrCode)
      Message.error("Fatal Error, system exit...")
    case nonLogin: NonLoginException =>
      val message = Message.noLogin(nonLogin.getMessage)
      message.data(EXCEPTION_MSG, nonLogin.toMap)
      message
    case error: ErrorException =>
      val cause = error.getCause
      val errorMsg = cause match {
        case t: ErrorException => s"error code(错误码): ${t.getErrCode}, error message(错误信息): ${t.getDesc}."
        case _ => s"error code(错误码): ${error.getErrCode}, error message(错误信息): ${error.getDesc}."
      }
      log.error(errorMsg, error)
      val message = Message.error(errorMsg)
      message.data(EXCEPTION_MSG, error.toMap)
      message
    case warn: WarnException =>
      val warnMsg = s"Warning code(警告码): ${warn.getErrCode}, Warning message(警告信息): ${warn.getDesc}."
      log.warn(warnMsg, warn)
      val message = Message.warn(warnMsg)
      message.data(EXCEPTION_MSG, warn.toMap)
      message
    case t =>
      log.error(msg, t)
      val errorMsg = ExceptionUtils.getRootCauseMessage(t)
      val message = if(StringUtils.isNotEmpty(errorMsg) && "operation failed(操作失败)" != msg) error(msg + "！the reason(原因)：" + errorMsg)
      else if(StringUtils.isNotEmpty(errorMsg)) error(errorMsg) else error(msg)
      message.data(EXCEPTION_MSG, ExceptionManager.unknownException(message.getMessage))
  }
  def catchIt(tryOp: => Message)(implicit log: Logger): Message = catchMsg(tryOp)("operation failed(操作失败)s")
  implicit def toScalaBuffer[T](list: util.List[T]): mutable.Buffer[T] = JavaConversions.asScalaBuffer(list)
  implicit def toScalaMap[K, V](map: util.Map[K, V]): mutable.Map[K, V] = JavaConversions.mapAsScalaMap(map)
  implicit def toJavaList[T](list: mutable.Buffer[T]): util.List[T] = {
    val arrayList = new util.ArrayList[T]
    list.foreach(arrayList.add)
    arrayList
  }
  implicit def toJavaMap[K, V](map: mutable.Map[K, V]): JMap[K, V] = {
    val hashMap = new util.HashMap[K, V]()
    map.foreach(m => hashMap.put(m._1, m._2))
    hashMap
  }
  implicit def toJavaMap[K, V](map: Map[K, V]): JMap[K, V] = {
    val hashMap = new util.HashMap[K, V]()
    map.foreach(m => hashMap.put(m._1, m._2))
    hashMap
  }
  implicit def asString(mapWithKey: (util.Map[String, Object], String)): String = mapWithKey._1.get(mapWithKey._2).asInstanceOf[String]
  implicit def getString(mapWithKey: (util.Map[String, String], String)): String = mapWithKey._1.get(mapWithKey._2)
  implicit def asInt(map: util.Map[String, Object], key: String): Int = map.get(key).asInstanceOf[Int]
  implicit def asBoolean(mapWithKey: (util.Map[String, Object], String)): Boolean = mapWithKey._1.get(mapWithKey._2).asInstanceOf[Boolean]

} 

package com.webank.wedatasphere.linkis.scheduler

import com.webank.wedatasphere.linkis.common.utils.Utils
import com.webank.wedatasphere.linkis.scheduler.exception.SchedulerErrorException
import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEvent
import org.apache.commons.lang.StringUtils


abstract class AbstractScheduler extends Scheduler {
  override def init(): Unit = {}

  override def start(): Unit = {}

  private def getEventId(index: Int, groupName: String): String = groupName + "_" + index
  private def getIndexAndGroupName(eventId: String): (Int, String) = {
    if(StringUtils.isBlank(eventId) || !eventId.contains("_"))
      throw new SchedulerErrorException(12011, s"Unrecognized execId $eventId.（不能识别的execId $eventId.)")
    val index = eventId.lastIndexOf("_")
    if(index < 1) throw new SchedulerErrorException(12011, s"Unrecognized execId $eventId.（不能识别的execId $eventId.)")
    (eventId.substring(index + 1).toInt, eventId.substring(0, index))
  }
  override def submit(event: SchedulerEvent): Unit = {
    val groupName = getSchedulerContext.getOrCreateGroupFactory.getGroupNameByEvent(event)
    val consumer = getSchedulerContext.getOrCreateConsumerManager.getOrCreateConsumer(groupName)
    val index = consumer.getConsumeQueue.offer(event)
    index.map(getEventId(_, groupName)).foreach(event.setId)
    if(index.isEmpty) throw  new SchedulerErrorException(12001,"The submission job failed and the queue is full!(提交作业失败，队列已满！)")
  }

  override def get(event: SchedulerEvent): Option[SchedulerEvent] = get(event.getId)

  override def get(eventId: String): Option[SchedulerEvent] = {
    val (index, groupName) = getIndexAndGroupName(eventId)
    val consumer = getSchedulerContext.getOrCreateConsumerManager.getOrCreateConsumer(groupName)
    consumer.getRunningEvents.find(_.getId == eventId).orElse(consumer.getConsumeQueue.get(index))
  }

  override def shutdown(): Unit = if(getSchedulerContext != null) {
    if(getSchedulerContext.getOrCreateConsumerManager != null)
      Utils.tryQuietly(getSchedulerContext.getOrCreateConsumerManager.shutdown())
    if(getSchedulerContext.getOrCreateExecutorManager != null)
      Utils.tryQuietly(getSchedulerContext.getOrCreateExecutorManager.shutdown())
    if(getSchedulerContext.getOrCreateSchedulerListenerBus != null)
      Utils.tryQuietly(getSchedulerContext.getOrCreateSchedulerListenerBus.stop())
  }
} 

package com.webank.wedatasphere.linkis.common.conf

import org.apache.commons.lang.StringUtils

import scala.collection.{JavaConversions, mutable}
import scala.collection.mutable.ArrayBuffer


object DWCArgumentsParser {
  protected val DWC_CONF = "--dwc-conf"
  protected val SPRING_CONF = "--spring-conf"
  private var dwcOptionMap = Map.empty[String, String]

  private[linkis] def setDWCOptionMap(dwcOptionMap: Map[String, String]) = this.dwcOptionMap = dwcOptionMap
  def getDWCOptionMap = dwcOptionMap

  def parse(args: Array[String]): DWCArgumentsParser = {
    val keyValueRegex = "([^=]+)=(.+)".r
    var i = 0
    val optionParser = new DWCArgumentsParser
    while(i < args.length) {
      args(i) match {
        case DWC_CONF | SPRING_CONF =>
          args(i + 1) match {
            case keyValueRegex(key, value) =>
              optionParser.setConf(args(i), key, value)
              i += 1
            case _ => throw new IllegalArgumentException("illegal commond line, format: --conf key=value.")
          }
        case _ => throw new IllegalArgumentException(s"illegal commond line, ${args(i)} cannot recognize.")
      }
      i += 1
    }
    optionParser.validate()
    optionParser
  }

  def formatToArray(optionParser: DWCArgumentsParser): Array[String] = {
    val options = ArrayBuffer[String]()
    def write(confMap: Map[String, String], optionType: String): Unit = confMap.foreach { case (key, value) =>
      if (StringUtils.isNotEmpty(key) && StringUtils.isNotEmpty(value)) {
        options += optionType
        options += (key + "=" + value)
      }
    }
    write(optionParser.getDWCConfMap, DWC_CONF)
    write(optionParser.getSpringConfMap, SPRING_CONF)
    options.toArray
  }
  def formatToArray(springOptionMap: Map[String, String], dwcOptionMap: Map[String, String]): Array[String] =
    formatToArray(new DWCArgumentsParser().setSpringConf(springOptionMap).setDWCConf(dwcOptionMap))

  def format(optionParser: DWCArgumentsParser): String = formatToArray(optionParser).mkString(" ")
  def format(springOptionMap: Map[String, String], dwcOptionMap: Map[String, String]): String =
    formatToArray(springOptionMap, dwcOptionMap).mkString(" ")

  def formatSpringOptions(springOptionMap: Map[String, String]): Array[String] = {
    val options = ArrayBuffer[String]()
    springOptionMap.foreach { case (key, value) =>
      if (StringUtils.isNotEmpty(key) && StringUtils.isNotEmpty(value)) {
        options += ("--" + key + "=" + value)
      }
    }
    options.toArray
  }
}
class DWCArgumentsParser {
  import DWCArgumentsParser._
  private val dwcOptionMap = new mutable.HashMap[String, String]()
  private val springOptionMap = new mutable.HashMap[String, String]()
  def getSpringConfMap = springOptionMap.toMap
  def getSpringConfs = JavaConversions.mapAsJavaMap(springOptionMap)
  def getDWCConfMap = dwcOptionMap.toMap
  def setConf(optionType: String, key: String, value: String) = {
    optionType match {
      case DWC_CONF =>
        dwcOptionMap += key -> value
      case SPRING_CONF =>
        springOptionMap += key -> value
    }
    this
  }
  def setSpringConf(optionMap: Map[String, String]): DWCArgumentsParser = {
    if(optionMap != null) this.springOptionMap ++= optionMap
    this
  }
  def setDWCConf(optionMap: Map[String, String]): DWCArgumentsParser = {
    if(optionMap != null) this.dwcOptionMap ++= optionMap
    this
  }
  def validate() = {}
} 

package com.hadooparchitecturebook.taxi360.setup.hbase

import java.io.File

import org.apache.commons.lang.StringUtils
import org.apache.hadoop.hbase.{HBaseConfiguration, HColumnDescriptor, HTableDescriptor, TableName}
import org.apache.hadoop.hbase.client.ConnectionFactory
import org.apache.hadoop.hbase.io.compress.Compression
import org.apache.hadoop.hbase.regionserver.{BloomType, ConstantSizeRegionSplitPolicy}
import org.apache.hadoop.hbase.util.Bytes

import scala.collection.mutable


object CreateSaltedTable {
  def main(args:Array[String]): Unit = {

    if (args.length == 0) {
      println("<tableName> <columnFamily> <regionCount> <numOfSalts> <hbaseConfigFolder>")
    }
    val tableName = args(0)
    val columnFamilyName = args(1)
    val regionCount = args(2).toInt
    val numOfSalts = args(3).toInt
    val hbaseConfigFolder = args(4)

    val conf = HBaseConfiguration.create()

    conf.addResource(new File(hbaseConfigFolder + "hbase-site.xml").toURI.toURL)

    val connection = ConnectionFactory.createConnection(conf)

    val admin = connection.getAdmin

    val tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName))

    val columnDescriptor = new HColumnDescriptor(columnFamilyName)

    columnDescriptor.setCompressionType(Compression.Algorithm.SNAPPY)
    columnDescriptor.setBlocksize(64 * 1024)
    columnDescriptor.setBloomFilterType(BloomType.ROW)

    tableDescriptor.addFamily(columnDescriptor)

    tableDescriptor.setMaxFileSize(Long.MaxValue)
    tableDescriptor.setRegionSplitPolicyClassName(classOf[ConstantSizeRegionSplitPolicy].getName)

    val splitKeys = new mutable.MutableList[Array[Byte]]
    for (i <- 0 to regionCount) {
      val regionSplitStr = StringUtils.leftPad((i*(numOfSalts/regionCount)).toString, 4, "0")
      splitKeys += Bytes.toBytes(regionSplitStr)
    }
    admin.createTable(tableDescriptor, splitKeys.toArray)
  }
} 

package com.ing.wbaa.rokku.proxy.provider

import java.io.File

import com.ing.wbaa.rokku.proxy.config.KerberosSettings
import com.typesafe.scalalogging.LazyLogging
import org.apache.commons.lang.StringUtils
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.security.UserGroupInformation

import scala.util.{ Failure, Success, Try }

trait KerberosLoginProvider extends LazyLogging {

  protected[this] def kerberosSettings: KerberosSettings

  loginUserFromKeytab(kerberosSettings.keytab, kerberosSettings.principal)

  private def loginUserFromKeytab(keytab: String, principal: String): Unit = {

    if (StringUtils.isNotBlank(keytab) && StringUtils.isNotBlank(principal)) {
      if (!new File(keytab).exists()) {
        logger.info("keytab file does not exist {}", keytab)
      } else {
        Try {
          UserGroupInformation.setConfiguration(new Configuration())
          UserGroupInformation.loginUserFromKeytab(principal, keytab)
        } match {
          case Success(_)         => logger.info("kerberos credentials provided {}", UserGroupInformation.getLoginUser)
          case Failure(exception) => logger.error("kerberos login error {}", exception)
        }
      }
    } else {
      logger.info("kerberos credentials are not provided")
    }
  }

} 

// Copyright 2017, Yahoo Holdings Inc.
// Licensed under the terms of the Apache License 2.0. Please see LICENSE file in project root for terms.
package com.yahoo.maha.report


  def close() {
    csvWriter.close()
  }

}

trait RowCSVWriterProvider {
  def newRowCSVWriter: RowCSVWriter
}

case class FileRowCSVWriterProvider(file: File) extends RowCSVWriterProvider {
  def newRowCSVWriter: RowCSVWriter = {
    if(file.exists() && file.length() > 0) {
      Files.write(file.toPath, Array[Byte](), StandardOpenOption.TRUNCATE_EXISTING) // Clear file
    }
    val fos = new FileOutputStream(file.getAbsoluteFile, true)
    val writerTry = safeCloseable(fos)(new OutputStreamWriter(_, StandardCharsets.UTF_8))
      .flatMap(safeCloseable(_)(new BufferedWriter(_)))
      .flatMap(safeCloseable(_)(new RowCSVWriter(_, RowCSVWriter.DEFAULT_SEPARATOR)))
    require(writerTry.isSuccess, s"Failed to create RowCSVWriter safely : $writerTry")
    writerTry.get
  }
} 

package com.enjoyyin.hive.proxy.jdbc.server

import java.net.InetSocketAddress

import org.apache.hive.service.auth.PlainSaslHelper
import org.apache.hive.service.auth.TSetIpAddressProcessor
import org.apache.thrift.TProcessor
import org.apache.thrift.TProcessorFactory
import org.apache.thrift.protocol.TBinaryProtocol
import org.apache.thrift.server.TServer
import org.apache.thrift.server.TThreadPoolServer
import org.apache.thrift.transport.TServerSocket
import org.apache.thrift.transport.TTransport
import com.enjoyyin.hive.proxy.jdbc.thrift.ThriftProxyService
import com.enjoyyin.hive.proxy.jdbc.util.Logging
import com.enjoyyin.hive.proxy.jdbc.util.ProxyConf.maxWorkerThreads
import com.enjoyyin.hive.proxy.jdbc.util.ProxyConf.minWorkerThreads
import com.enjoyyin.hive.proxy.jdbc.util.ProxyConf.portNum
import com.enjoyyin.hive.proxy.jdbc.util.ProxyConf.proxyHost
import com.enjoyyin.hive.proxy.jdbc.util.ProxyConf.authTypeStr
import com.enjoyyin.hive.proxy.jdbc.util.ProxyConf.AUTHENTICATION_OF_CUSTOM
import com.enjoyyin.hive.proxy.jdbc.util.ProxyConf.AUTHENTICATION_OF_NONE
import com.enjoyyin.hive.proxy.jdbc.util.Utils
import com.enjoyyin.hive.proxy.jdbc.rule.basic.DefaultLoginValidateRule
import org.apache.commons.lang.StringUtils
import org.apache.hadoop.hive.conf.HiveConf


class JDBCProxyServer extends Runnable with Logging {

  var server: TServer = _

  override def run(): Unit = {
    val serverAddress = getServerSocket(proxyHost, portNum)
    //hive.server2.authentication
    //hive.server2.custom.authentication.class
    authTypeStr.toUpperCase match {
      case AUTHENTICATION_OF_NONE =>
      case AUTHENTICATION_OF_CUSTOM =>
        val customClass = HiveConf.ConfVars.HIVE_SERVER2_CUSTOM_AUTHENTICATION_CLASS
        System.setProperty(customClass.varname, DefaultLoginValidateRule.getClass.getName)
      case _ => throw new IllegalArgumentException(s"Illegal hive.server2.authentication of value $authTypeStr.")
    }
    logInfo("Is authentication enable? " + authTypeStr)
    val thriftProxyService = new ThriftProxyService
    thriftProxyService.init
    Utils.addShutdownHook(thriftProxyService.close)
    val sargs = new TThreadPoolServer.Args(serverAddress)
      .processorFactory(new SQLPlainProcessorFactory(thriftProxyService))
      .transportFactory(PlainSaslHelper.getPlainTransportFactory(authTypeStr))
      .protocolFactory(new TBinaryProtocol.Factory())
      .minWorkerThreads(minWorkerThreads)
      .maxWorkerThreads(maxWorkerThreads)

    server = new TThreadPoolServer(sargs)
    
    logInfo("JDBC Proxy listening on " + serverAddress.getServerSocket)

    server.serve()
  }

  def getServerSocket(proxyHost: String, portNum: Int): TServerSocket = {
    var serverAddress: InetSocketAddress = null
    if (StringUtils.isNotBlank(proxyHost)) {
      serverAddress = new InetSocketAddress(proxyHost, portNum)
    } else {
      serverAddress = new InetSocketAddress(portNum)
    }
    new TServerSocket(serverAddress)
  }

}

object JDBCProxyServer {
  def main(args: Array[String]): Unit = {
    val mainThread = new Thread(new JDBCProxyServer)
    mainThread.setName("hive-jdbc-proxy-main-thread")
    mainThread.start()
  }
}

class SQLPlainProcessorFactory(val service: ThriftProxyService)
    extends TProcessorFactory(null) {
  override def getProcessor(trans: TTransport): TProcessor = {
    new TSetIpAddressProcessor(service)
  }
} 

package com.amazon.milan.application.sources

import com.amazon.milan.dataformats.DataInputFormat
import com.amazon.milan.typeutil.TypeDescriptor
import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize}
import org.apache.commons.lang.StringUtils



@JsonSerialize
@JsonDeserialize
class S3DataSource[T: TypeDescriptor](bucket: String,
                                      key: String,
                                      dataFormat: DataInputFormat[T],
                                      configuration: FileDataSource.Configuration)
  extends FileDataSource[T](S3DataSource.getS3Url(bucket, key), dataFormat, configuration) {

  def this(bucket: String, key: String, dataFormat: DataInputFormat[T]) {
    this(bucket, key, dataFormat, FileDataSource.Configuration.default)
  }
}


object S3DataSource {
  def getS3Url(bucket: String, key: String): String =
    s"s3://${StringUtils.strip(bucket, "/")}/${StringUtils.strip(key, "/")}"
} 

package org.schedoscope.dsl.transformations

import java.io.{FileInputStream, InputStream}

import org.apache.commons.lang.StringUtils
import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege
import org.apache.hadoop.hive.ql.udf.UDFLength
import org.apache.hive.hcatalog.data.schema.HCatSchema
import org.apache.hive.hcatalog.pig.HCatLoader
import org.apache.pig.builtin.ParquetStorer
import org.schedoscope.scheduler.service.ViewTransformationStatus



case class PigTransformation(latin: String, dirsToDelete: List[String] = List()) extends Transformation {

  def name = "pig"

  override def stringsToChecksum = List(latin)

  description = "[..]" + StringUtils.abbreviate(latin.replaceAll("\n", "").replaceAll("\t", "").replaceAll("\\s+", " "), 60)

  def defaultLibraries = {
    // FIXME: declare jars instead of any random class included in this jar
    val classes = List(
      // needed for usage of HCatalog table management
      classOf[HCatLoader], classOf[HCatSchema], classOf[HiveObjectPrivilege], classOf[UDFLength],
      // needed for usage of storage format Parquet with pig
      classOf[ParquetStorer])
    classes.map(cl => try {
      cl.getProtectionDomain().getCodeSource().getLocation().getFile
    } catch {
      case t: Throwable => null
    })
      .filter(cl => cl != null && !"".equals(cl.trim))
  }

  override def viewTransformationStatus = ViewTransformationStatus(
    name,
    Some(Map("latin" -> latin)))
}

object PigTransformation {

  def scriptFrom(inputStream: InputStream): String = scala.io.Source.fromInputStream(inputStream, "UTF-8").mkString

  def scriptFromResource(resourcePath: String): String = scriptFrom(getClass().getClassLoader().getResourceAsStream(resourcePath))

  def scriptFrom(filePath: String): String = scriptFrom(new FileInputStream(filePath))
} 

package org.schedoscope.dsl.transformations

import java.io.{FileInputStream, InputStream}
import java.util.Properties

import org.apache.commons.lang.StringUtils
import org.schedoscope.Settings
import org.schedoscope.scheduler.service.ViewTransformationStatus

import scala.collection.JavaConversions._


case class OozieTransformation(bundle: String, workflow: String, var workflowAppPath: String) extends Transformation {
  def name = "oozie"

  override def fileResourcesToChecksum = List(workflowAppPath)

  description = StringUtils.abbreviate(s"${bundle}/${workflow}", 100)

  override def viewTransformationStatus = ViewTransformationStatus(
    name,
    Some(Map(
      "bundle" -> bundle,
      "workflow" -> workflow)))
}

object OozieTransformation {
  def oozieWFPath(bundle: String, workflow: String) = s"${Settings().getDriverSettings("oozie").location}/workflows/${bundle}/${workflow}/"

  def configurationFrom(inputStream: InputStream): Map[String, String] = {
    val props = new Properties()

    try {
      props.load(inputStream)
    } catch {
      case t: Throwable =>
    }

    Map() ++ props
  }

  def configurationFrom(filePath: String): Map[String, String] = try
    configurationFrom(new FileInputStream(filePath))
  catch {
    case t: Throwable => Map()
  }

  def configurationFromResource(resourcePath: String): Map[String, String] =
    try
      configurationFrom(getClass().getClassLoader().getResourceAsStream(resourcePath))
    catch {
      case t: Throwable => Map()
    }
} 

package org.apache.livy.thriftserver.auth

import javax.security.sasl.AuthenticationException

import org.apache.commons.lang.StringUtils
import org.apache.hive.service.auth.PasswdAuthenticationProvider

import org.apache.livy.thriftserver.auth.ldap._
import org.apache.livy.LivyConf
import org.apache.livy.server.auth.LdapUtils

class LdapAuthenticationProviderImpl(val conf: LivyConf) extends PasswdAuthenticationProvider {
  private val filter: Filter = new ChainFilter(List(new UserFilter(conf)))
  private val searchFactory: DirSearchFactory = new LdapSearchFactory()

  @throws[AuthenticationException]
  def Authenticate(user: String, password: String): Unit = {
    createDirSearch(user, password)
    applyFilter(user)
  }

  @throws[AuthenticationException]
  private def createDirSearch(user: String, password: String): Unit = {
    if (StringUtils.isBlank(user) || StringUtils.isEmpty(user)) {
      throw new AuthenticationException(
        "Error validating LDAP: a null or blank user name has been provided")
    }
    if (StringUtils.isBlank(password) || StringUtils.isEmpty(password)) {
      throw new AuthenticationException(
        "Error validating LDAP: a null or blank password has been provided")
    }
    val principal = LdapUtils.createCandidatePrincipal(conf, user)
    try {
      searchFactory.getInstance(conf, principal, password)
    } catch {
      case e: AuthenticationException =>
        throw new AuthenticationException(s"Error validating LDAP user: $user", e)
    }
  }

  @throws[AuthenticationException]
  private def applyFilter(user: String): Unit = {
    if (LdapUtils.hasDomain(user)) {
      filter(LdapUtils.extractUserName(user))
    } else {
      filter(user)
    }
  }
} 

package scalabpe

import java.io._
import scala.collection.mutable.HashMap
import scala.collection.mutable.ArrayBuffer
import scala.io.Source
import org.apache.commons.io.FileUtils
import scala.xml._
import scala.collection.mutable._
import scalabpe.core._
import org.apache.commons.lang.StringUtils
import Tools._



object FormatFlowTool {

    def help() {
        println(
"""
usage: scalabpe.FormatFlowTool [options] dirname
options:
    -h|--help               帮助信息
""")
    }

    def parseArgs(args:Array[String]):HashMapStringAny = {
        val map = HashMapStringAny()
        var i = 0
        val files = ArrayBufferString()
        while(i < args.size) {
            args(i) match {
                case "-h" | "--help" => 
                    return null
                case s if s.startsWith("-") => 
                    println("invalid option "+s)
                    return null
                case _ => 
                    files += args(i)
                    i += 1
            }
        }
        map.put("files",files)
        map
    }

    def main(args:Array[String]) {

        var params = parseArgs(args)
        if( params == null ) {
            help()
            return
        }
        var files = params.nls("files")
        if( files.size == 0 ) {
            help()
            return
        }

        var dir = files(0)
        if( !new File(dir).exists() ) {
            val p1 = "compose_conf"+File.separator+dir
            if( new File(p1).exists ) {
                dir = p1
            } else {
                println("not a valid dir, dir="+dir)
                return
            }
        }

        processDir(dir,params)
    }

    def processDir(dir:String,params:HashMapStringAny) {
        val files = new File(dir).listFiles.filter(_.getName.endsWith(".flow"))
        for(f <- files ) {
            processFile(dir,f.getName,params)
        }
    }

    def processFile(dir:String,f:String,params:HashMapStringAny) {
        val lines = readAllLines(dir+File.separator+f)
        // TODO
    }

} 

package io.gzet

import java.text.SimpleDateFormat

import com.gravity.goose.{Configuration, Goose}
import io.gzet.HtmlFetcher.Content
import org.apache.commons.lang.StringUtils
import org.apache.spark.rdd.RDD

class HtmlFetcher(
                   connectionTimeout: Int = 10000,
                   socketTimeout: Int = 10000
                 ) extends Serializable {

  final val USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36"
  final val ISO_SDF = "yyyy-MM-dd'T'HH:mm:ssZ"

  def fetchWithContext(urlRdd: RDD[(Long, String)]): RDD[(Long, Content)] = {
    urlRdd mapPartitions { urls =>
      val sdf = new SimpleDateFormat(ISO_SDF)
      val goose = getGooseScraper
      urls map { case (id, url) =>
        (id, fetchUrl(goose, url, sdf))
      }
    }
  }

  def fetch(urlRdd: RDD[String]): RDD[Content] = {
    urlRdd mapPartitions { urls =>
      val sdf = new SimpleDateFormat(ISO_SDF)
      val goose = getGooseScraper
      urls map(url => fetchUrl(goose, url, sdf))
    }
  }

  private def getGooseScraper: Goose = {
    val conf: Configuration = new Configuration
    conf.setEnableImageFetching(false)
    conf.setBrowserUserAgent(USER_AGENT)
    conf.setConnectionTimeout(connectionTimeout)
    conf.setSocketTimeout(socketTimeout)
    new Goose(conf)
  }

  private def fetchUrl(goose: Goose, url: String, sdf: SimpleDateFormat) : Content = {

    try {

      val article = goose.extractContent(url)
      var body = None: Option[String]
      var title = None: Option[String]
      var description = None: Option[String]
      var publishDate = None: Option[String]

      if (StringUtils.isNotEmpty(article.cleanedArticleText))
        body = Some(article.cleanedArticleText)

      if (StringUtils.isNotEmpty(article.title))
        title = Some(article.title)

      if (StringUtils.isNotEmpty(article.metaDescription))
        description = Some(article.metaDescription)

      if (article.publishDate != null)
        publishDate = Some(sdf.format(article.publishDate))

      Content(url, title, description, body, publishDate)

    } catch {
      case e: Throwable =>
        Content(url, None, None, None, None)
    }
  }
}


object HtmlFetcher {

  case class Content(
                      url: String,
                      title: Option[String],
                      description: Option[String],
                      body: Option[String],
                      publishedDate: Option[String]
                    )

} 

package com.cloudera.sa.taxi360.setup.hbase

import java.io.File

import org.apache.commons.lang.StringUtils
import org.apache.hadoop.hbase.{HBaseConfiguration, HColumnDescriptor, HTableDescriptor, TableName}
import org.apache.hadoop.hbase.client.ConnectionFactory
import org.apache.hadoop.hbase.io.compress.Compression
import org.apache.hadoop.hbase.regionserver.{BloomType, ConstantSizeRegionSplitPolicy}
import org.apache.hadoop.hbase.util.Bytes

import scala.collection.mutable


object CreateSaltedTable {
  def main(args:Array[String]): Unit = {

    if (args.length == 0) {
      println("<tableName> <columnFamily> <regionCount> <numOfSalts> <hbaseConfigFolder>")
    }
    val tableName = args(0)
    val columnFamilyName = args(1)
    val regionCount = args(2).toInt
    val numOfSalts = args(3).toInt
    val hbaseConfigFolder = args(4)

    val conf = HBaseConfiguration.create()

    conf.addResource(new File(hbaseConfigFolder + "hbase-site.xml").toURI.toURL)

    val connection = ConnectionFactory.createConnection(conf)

    val admin = connection.getAdmin

    val tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName))

    val columnDescriptor = new HColumnDescriptor(columnFamilyName)

    columnDescriptor.setCompressionType(Compression.Algorithm.SNAPPY)
    columnDescriptor.setBlocksize(64 * 1024)
    columnDescriptor.setBloomFilterType(BloomType.ROW)

    tableDescriptor.addFamily(columnDescriptor)

    tableDescriptor.setMaxFileSize(Long.MaxValue)
    tableDescriptor.setRegionSplitPolicyClassName(classOf[ConstantSizeRegionSplitPolicy].getName)

    val splitKeys = new mutable.MutableList[Array[Byte]]
    for (i <- 0 to regionCount) {
      val regionSplitStr = StringUtils.leftPad((i*(numOfSalts/regionCount)).toString, 4, "0")
      splitKeys += Bytes.toBytes(regionSplitStr)
    }
    admin.createTable(tableDescriptor, splitKeys.toArray)
  }
} 

package com.webank.wedatasphere.linkis.entrance.execute

import com.webank.wedatasphere.linkis.common.log.LogUtils
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.entrance.exception.EntranceErrorException
import com.webank.wedatasphere.linkis.protocol.utils.TaskUtils
import com.webank.wedatasphere.linkis.scheduler.exception.WaitForNextAskExecutorException
import com.webank.wedatasphere.linkis.scheduler.queue.Job
import org.apache.commons.lang.StringUtils
import org.springframework.stereotype.Component


trait EntranceExecutorRuler {

  def setEngineManager(engineManager: EngineManager): Unit
  def getEngineManager: EngineManager
  def rule(engines: Array[EntranceEngine], job: Job): Array[EntranceEngine]
  def cloneNew(): EntranceExecutorRuler

}
trait AbstractEntranceExecutorRuler extends EntranceExecutorRuler {
  private var engineManager: EngineManager = _
  protected val specialKey: String

  override def setEngineManager(engineManager: EngineManager): Unit = this.engineManager = engineManager

  override def getEngineManager: EngineManager = engineManager

  override def rule(engines: Array[EntranceEngine], job: Job): Array[EntranceEngine] = job match {
    case entranceJob: EntranceJob =>
      val specialMap = TaskUtils.getSpecialMap(entranceJob.getParams)
      val specialValue = specialMap.get(specialKey)
      if(specialValue != null && StringUtils.isNotBlank(specialValue.toString)) {
        val log = new java.lang.StringBuilder
        val es = rule(engines, specialValue, log)
        if(log.length() > 0) job.getLogListener.foreach(_.onLogUpdate(job, LogUtils.generateSystemInfo(log.toString)))
        es
      }
      else if(specialMap.containsKey(specialKey)) throw new EntranceErrorException(28000, "非法的special configuration: " + specialKey + ", 参数值不能为空！")
      else engines
    case _ => engines
  }

  protected def rule(engines: Array[EntranceEngine], specialValue: Any, log: java.lang.StringBuilder): Array[EntranceEngine]

  override def toString: String = specialKey
}
@Component
class FixedInstanceEntranceExecutorRuler extends AbstractEntranceExecutorRuler with Logging {
  override val specialKey: String = FixedInstanceEntranceExecutorRuler.FIXED_INSTANCE
  override def rule(engines: Array[EntranceEngine], specialValue: Any, log: java.lang.StringBuilder): Array[EntranceEngine] = {
    log.append(s"You have enabled the fixed execution engine function! This time will be submitted to the engine $specialValue.(您开启了固定执行引擎功能！本次将提交给引擎$specialValue。)")
    val es = engines.filter(_.getModuleInstance.getInstance == specialValue)
    if(es.isEmpty && getEngineManager.get(specialValue.toString).isEmpty)
      throw new EntranceErrorException(28001, s"The last used engine $specialValue no longer exists, and this fixed execution engine operation cannot be completed!(沿用上次的引擎$specialValue 已不存在，无法完成本次固定执行引擎操作！)")
    else if(es.isEmpty) {
      throw new WaitForNextAskExecutorException(s"Please note: the execution engine ${specialValue} is currently not idle, waiting for the state to flip...(请注意：执行引擎${specialValue}目前不处于空闲状态，等待状态翻转...)")
    }
    es
  }
  override def cloneNew(): FixedInstanceEntranceExecutorRuler = new FixedInstanceEntranceExecutorRuler
}
object FixedInstanceEntranceExecutorRuler {
  val FIXED_INSTANCE = "fixedInstance"
}
@Component
class ExceptInstanceEntranceExecutorRuler extends AbstractEntranceExecutorRuler {
  override val specialKey: String = ExceptInstanceEntranceExecutorRuler.EXCEPT_INSTANCES
  override def rule(engines: Array[EntranceEngine], specialValue: Any, log: java.lang.StringBuilder): Array[EntranceEngine] = {
    log.append("Please note: This submitted script will exclude the following engines(请注意：本次提交的脚本，将排除以下引擎)：" + specialValue)
    val exceptEngines = ExceptInstanceEntranceExecutorRuler.deserializable(specialValue.toString).toSet
    engines.filterNot(e => exceptEngines.contains(e.getModuleInstance.getInstance))
  }
  override def cloneNew(): ExceptInstanceEntranceExecutorRuler = this
}
object ExceptInstanceEntranceExecutorRuler {
  val EXCEPT_INSTANCES = "exceptInstances"
  def serializable(instances: Array[String]): String = instances.mkString(",")
  def deserializable(instances: String): Array[String] = instances.split(",")
} 

package com.webank.wedatasphere.linkis.engine.hive.hook

import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorContext, EngineHook}
import com.webank.wedatasphere.linkis.engine.hive.executor.HiveEngineExecutor
import com.webank.wedatasphere.linkis.server.JMap
import org.apache.commons.lang.StringUtils
import org.slf4j.LoggerFactory



class HiveAddJarsEngineHook extends EngineHook {

  private var jars:String = _
  private val JARS = "jars"
  private val logger = LoggerFactory.getLogger(classOf[HiveAddJarsEngineHook])
  private val addSql = "add jar "
  override def beforeCreateEngine(params: JMap[String, String]): JMap[String, String] = {
    import scala.collection.JavaConversions._
//    params foreach {
//      case (k, v) => logger.info(s"params key is $k, value is $v")
//    }
    params foreach {
      case (key,value) => if (JARS.equals(key)) jars = value
    }
    logger.info(s"jarArray is {}", jars)
    params
  }

  override def afterCreatedEngine(executor: EngineExecutor): Unit = {
    if (StringUtils.isEmpty(jars)) {
      logger.warn("hive added jars is empty")
      return
    }
    jars.split(",") foreach{
       jar =>
         try{
           logger.info("begin to run hive sql {}", addSql + jar)
           executor.asInstanceOf[HiveEngineExecutor].executeLine(new EngineExecutorContext(executor), addSql + jar)
         }catch{
           case t:Throwable => logger.error(s"run hive sql ${addSql + jar} failed", t)
         }
    }
  }
} 

package com.webank.wedatasphere.linkis.enginemanager.hive.process

import java.nio.file.Paths

import com.webank.wedatasphere.linkis.common.conf.Configuration
import com.webank.wedatasphere.linkis.enginemanager.conf.EnvConfiguration.{DEFAULT_JAVA_OPTS, JAVA_HOME, engineGCLogPath}
import com.webank.wedatasphere.linkis.enginemanager.hive.conf.HiveEngineConfiguration
import com.webank.wedatasphere.linkis.enginemanager.impl.UserEngineResource
import com.webank.wedatasphere.linkis.enginemanager.process.JavaProcessEngineBuilder
import com.webank.wedatasphere.linkis.enginemanager.{AbstractEngineCreator, EngineResource}
import com.webank.wedatasphere.linkis.protocol.engine.RequestEngine
import org.apache.commons.lang.StringUtils
import org.slf4j.LoggerFactory

import scala.collection.mutable.ArrayBuffer


  override protected def classpathCheck(jarOrFiles: Array[String]): Unit = {
    for(jarOrFile <- jarOrFiles){
      checkJarOrFile(jarOrFile)
    }
  }
  //todo Check the jar of the classpath(对classpath的jar进行检查)
  private def checkJarOrFile(jarOrFile:String):Unit = {

  }


  override def build(engineRequest: EngineResource, request: RequestEngine): Unit = {
    this.request = request
    userEngineResource = engineRequest.asInstanceOf[UserEngineResource]
    val javaHome = JAVA_HOME.getValue(request.properties)
    if(StringUtils.isEmpty(javaHome)) {
      warn("We cannot find the java home, use java to run storage repl web server.")
      commandLine += "java"
    } else {
      commandLine += Paths.get(javaHome, "bin/java").toAbsolutePath.toFile.getAbsolutePath
    }
    if (request.properties.containsKey(HiveEngineConfiguration.HIVE_CLIENT_MEMORY.key)){
      val settingClientMemory = request.properties.get(HiveEngineConfiguration.HIVE_CLIENT_MEMORY.key)
      if (!settingClientMemory.toLowerCase().endsWith("g")){
        request.properties.put(HiveEngineConfiguration.HIVE_CLIENT_MEMORY.key, settingClientMemory + "g")
      }
      //request.properties.put(HiveEngineConfiguration.HIVE_CLIENT_MEMORY.key, request.properties.get(HiveEngineConfiguration.HIVE_CLIENT_MEMORY.key)+"g")
    }
    val clientMemory = HiveEngineConfiguration.HIVE_CLIENT_MEMORY.getValue(request.properties).toString
    if (clientMemory.toLowerCase().endsWith("g")){
      commandLine += ("-Xmx" + clientMemory.toLowerCase())
      commandLine += ("-Xms" + clientMemory.toLowerCase())
    }else{
      commandLine += ("-Xmx" + clientMemory + "g")
      commandLine += ("-Xms" + clientMemory + "g")
    }
    val javaOPTS = getExtractJavaOpts
    val alias = getAlias(request)
    if(StringUtils.isNotEmpty(DEFAULT_JAVA_OPTS.getValue))
      DEFAULT_JAVA_OPTS.getValue.format(engineGCLogPath(port, userEngineResource.getUser, alias)).split("\\s+").foreach(commandLine += _)
    if(StringUtils.isNotEmpty(javaOPTS)) javaOPTS.split("\\s+").foreach(commandLine += _)
    //engineLogJavaOpts(port, alias).trim.split(" ").foreach(commandLine += _)
    if(Configuration.IS_TEST_MODE.getValue) {
      val port = AbstractEngineCreator.getNewPort
      info(s"$toString open debug mode with port $port.")
      commandLine += s"-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$port"
    }
    var classpath = getClasspath(request.properties, getExtractClasspath)
    classpath = classpath ++ request.properties.get("jars").split(",")
    classpathCheck(classpath)
    commandLine += "-Djava.library.path=/appcom/Install/hadoop/lib/native"
    commandLine += "-cp"
    commandLine += classpath.mkString(":")
    commandLine += "com.webank.wedatasphere.linkis.engine.DataWorkCloudEngineApplication"
  }


//  override def build(engineRequest: EngineResource, request: RequestEngine): Unit = {
//    import scala.collection.JavaConversions._
//    request.properties foreach {case (k, v) => LOG.info(s"request key is $k, value is $v")}
//    this.request = request
//    super.build(engineRequest, request)
//
//  }

  override protected val addApacheConfigPath: Boolean = true
} 

package com.webank.wedatasphere.linkis.entrance.executer

import com.webank.wedatasphere.linkis.entrance.conf.JDBCConfiguration
import org.apache.commons.lang.StringUtils

import scala.collection.mutable.ArrayBuffer

object JDBCSQLCodeParser {

  val separator = ";"
  val defaultLimit: Int = JDBCConfiguration.ENGINE_DEFAULT_LIMIT.getValue

  def parse(code: String): Array[String] = {
    val codeBuffer = new ArrayBuffer[String]()

    def appendStatement(sqlStatement: String): Unit = {
      codeBuffer.append(sqlStatement)
    }

    if (StringUtils.contains(code, separator)) {
      StringUtils.split(code, ";").foreach {
        case s if StringUtils.isBlank(s) =>
        case s if isSelectCmdNoLimit(s) => appendStatement(s + " limit " + defaultLimit);
        case s => appendStatement(s);
      }
    } else {
      code match {
        case s if StringUtils.isBlank(s) =>
        case s if isSelectCmdNoLimit(s) => appendStatement(s + " limit " + defaultLimit);
        case s => appendStatement(s);
      }
    }
    codeBuffer.toArray
  }

  def isSelectCmdNoLimit(cmd: String): Boolean = {
    var code = cmd.trim
    if (!cmd.split("\\s+")(0).equalsIgnoreCase("select")) return false
    if (code.contains("limit")) code = code.substring(code.lastIndexOf("limit")).trim
    else if (code.contains("LIMIT")) code = code.substring(code.lastIndexOf("LIMIT")).trim.toLowerCase
    else return true
    val hasLimit = code.matches("limit\\s+\\d+\\s*;?")
    if (hasLimit) {
      if (code.indexOf(";") > 0) code = code.substring(5, code.length - 1).trim
      else code = code.substring(5).trim
      val limitNum = code.toInt
      if (limitNum > defaultLimit) throw new IllegalArgumentException("We at most allowed to limit " + defaultLimit + ", but your SQL has been over the max rows.")
    }
    !hasLimit
  }


} 

package com.webank.wedatasphere.linkis.engine.util

import org.apache.commons.lang.StringUtils
/.+\\s*".r
    val symbolRegex3 = "\\s*--.+\\s*".r
    val blankRegex = "\\s*".r
    code.split("\n").foreach {
      case blankRegex() | setRegex() | symbolRegex1() | symbolRegex2() | symbolRegex3() | restartRegex() | kindRegex() =>
      case str => msg ++= str ++ "\n"
    }
    StringUtils.strip(msg.toString)
  }
} 

package com.webank.wedatasphere.linkis.engine.mdq

import java.util

import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration
import com.webank.wedatasphere.linkis.engine.exception.MDQErrorException
import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
import com.webank.wedatasphere.linkis.engine.extension.SparkPreExecutionHook
import com.webank.wedatasphere.linkis.engine.spark.common.SparkKind
import com.webank.wedatasphere.linkis.protocol.mdq.{DDLRequest, DDLResponse}
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.storage.utils.StorageUtils
import javax.annotation.PostConstruct
import org.apache.commons.lang.StringUtils
import org.springframework.stereotype.Component


@Component
class MDQPreExecutionHook extends SparkPreExecutionHook with Logging{

  @PostConstruct
  def  init(): Unit ={
    SparkPreExecutionHook.register(this)
  }

  override def hookName: String = "MDQPreHook"

  override def callPreExecutionHook(engineExecutorContext: EngineExecutorContext, code: String): String = {

    val runType: String = engineExecutorContext.getProperties.get("runType") match {
      case value:String => value
      case _ => ""
    }
    if(StringUtils.isEmpty(runType) || ! SparkKind.FUNCTION_MDQ_TYPE.equalsIgnoreCase(runType)) return code
    val sender = Sender.getSender(SparkConfiguration.MDQ_APPLICATION_NAME.getValue)
    val params = new util.HashMap[String,Object]()
    params.put("user", StorageUtils.getJvmUser)
    params.put("code", code)
    sender.ask(DDLRequest(params)) match {
      case DDLResponse(postCode) => postCode
      case _ => throw new MDQErrorException(40010, "向MDQ服务请求解析为可以执行的sql时失败")
    }
  }
} 

package com.webank.wedatasphere.linkis.engine.mdq

import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration
import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
import com.webank.wedatasphere.linkis.engine.extension.SparkPostExecutionHook
import com.webank.wedatasphere.linkis.engine.spark.common.SparkKind
import com.webank.wedatasphere.linkis.protocol.mdq.{DDLCompleteResponse, DDLExecuteResponse}
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteResponse, SuccessExecuteResponse}
import com.webank.wedatasphere.linkis.storage.utils.StorageUtils
import javax.annotation.PostConstruct
import org.apache.commons.lang.StringUtils
import org.springframework.stereotype.Component


@Component
class MDQPostExecutionHook extends SparkPostExecutionHook with Logging{

  @PostConstruct
  def  init(): Unit ={
    SparkPostExecutionHook.register(this)
  }

  override def hookName: String = "MDQPostHook"

  override def callPostExecutionHook(engineExecutorContext: EngineExecutorContext, executeResponse: ExecuteResponse, code: String): Unit = {
    val runType: String = engineExecutorContext.getProperties.get("runType") match {
      case value:String => value
      case _ => ""
    }
    if(StringUtils.isEmpty(runType) || ! SparkKind.FUNCTION_MDQ_TYPE.equalsIgnoreCase(runType)) return
    val sender = Sender.getSender(SparkConfiguration.MDQ_APPLICATION_NAME.getValue)
    executeResponse match {
      case SuccessExecuteResponse() =>
        sender.ask(DDLExecuteResponse(true, code, StorageUtils.getJvmUser)) match {
          case DDLCompleteResponse(status) => if (! status)
            warn(s"执行建表失败:$code")
        }
      case _=> warn(s"执行建表失败:$code")
    }
  }
} 

package com.webank.wedatasphere.linkis.engine.cs

import java.util.regex.Pattern

import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.cs.client.service.CSTableService
import com.webank.wedatasphere.linkis.cs.common.entity.metadata.CSTable
import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils
import com.webank.wedatasphere.linkis.engine.exception.ExecuteError
import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
import org.apache.commons.lang.StringUtils
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.execution.datasources.csv.DolphinToSpark

import scala.collection.mutable.ArrayBuffer


  def getCSTable(csTempTable:String,  contextIDValueStr: String, nodeNameStr: String):CSTable = {
    CSTableService.getInstance().getUpstreamSuitableTable(contextIDValueStr, nodeNameStr, csTempTable)
  }

  def registerTempTable(csTable: CSTable):Unit = {
    val spark = SparkSession.builder().enableHiveSupport().getOrCreate()
    info(s"Start to create  tempView to sparkSession viewName(${csTable.getName}) location(${csTable.getLocation})")
    DolphinToSpark.createTempView(spark, csTable.getName, csTable.getLocation, true)
    info(s"Finished to create  tempView to sparkSession viewName(${csTable.getName}) location(${csTable.getLocation})")
  }
} 

package com.webank.wedatasphere.linkis.engine.spark.common

import org.apache.commons.lang.StringUtils

/.+\\s*".r
    val symbolRegex3 = "\\s*--.+\\s*".r
    val blankRegex = "\\s*".r
    code.split("\n").foreach {
      case blankRegex() | setRegex() | symbolRegex1() | symbolRegex2() | symbolRegex3() | restartRegex() | kindRegex() =>
      case str => msg ++= str ++ "\n"
    }
    StringUtils.strip(msg.toString)
  }
} 

package com.webank.wedatasphere.linkis.enginemanager.process

import com.webank.wedatasphere.linkis.common.conf.DWCArgumentsParser
import com.webank.wedatasphere.linkis.common.utils.Utils
import org.apache.commons.lang.StringUtils


class SparkCommonProcessEngine(override val processBuilder: ProcessEngineBuilder,
                                override val dwcArgumentsParser: DWCArgumentsParser,
                                override val timeout: Long) extends CommonProcessEngine(processBuilder,dwcArgumentsParser,timeout)
{
  def this(processBuilder: ProcessEngineBuilder, dwcArgumentsParser: DWCArgumentsParser)=this(processBuilder, dwcArgumentsParser, -1)
  private var yarnAppId: String = _

  override def dealStartupLog(line: String): Unit = {
    println(getPort + ": " + line)
    if(StringUtils.isEmpty(yarnAppId)) {
      if (line.contains("application_")) {
        val appIdStr = line.split("\\s+").filter(x => x.startsWith("application_")).toList
        if (appIdStr.size > 0) {
          yarnAppId = appIdStr(0)
          info("Get yarn application id is "+ yarnAppId)
        }
      }
    }
  }

  override def shutdown(): Unit ={
   if(StringUtils.isNotEmpty(yarnAppId)){
     info(s"try to kill yarn app with id($yarnAppId).")
     Utils.tryQuietly(
     Utils.exec(Array(JavaProcessEngineBuilder.sudoUserScript.getValue, getUser, s"yarn  application -kill  $yarnAppId"), 3000l)
     )
   }
    super.shutdown()
  }

} 

package com.webank.wedatasphere.linkis.engine.shell.executor

import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorFactory}
import com.webank.wedatasphere.linkis.engine.shell.exception.NoCorrectUserException
import com.webank.wedatasphere.linkis.server.JMap
import org.apache.commons.lang.StringUtils
import org.springframework.stereotype.Component


@Component
class ShellEngineExecutorFactory extends EngineExecutorFactory with Logging{
  override def createExecutor(options: JMap[String, String]): EngineExecutor = {
    //todo 可能会有一些设置环境变量的操作
    import scala.collection.JavaConverters._
    options.asScala foreach {
      case (k, v) => info(s"key is $k, value is $v")
    }
    val user:String = System.getProperty("user.name")
    if (StringUtils.isEmpty(user)) throw NoCorrectUserException()

    new ShellEngineExecutor(user)
  }
} 

package com.webank.wedatasphere.linkis.entrance.parser

import java.util
import java.util.Date

import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
import com.webank.wedatasphere.linkis.entrance.exception.EntranceIllegalParamException
import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant
import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
import com.webank.wedatasphere.linkis.protocol.task.Task
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState
import org.apache.commons.lang.StringUtils
import org.slf4j.LoggerFactory


    if (EntranceConfiguration.DEFAULT_REQUEST_APPLICATION_NAME.getValue.equals(creator) && StringUtils.isEmpty(source.get(TaskConstant.SCRIPTPATH)) &&
      StringUtils.isEmpty(executionCode))
      throw new EntranceIllegalParamException(20007, "param executionCode and scriptPath can not be empty at the same time")
    var runType:String = null
    if (StringUtils.isNotEmpty(executionCode)) {
      runType = params.get(TaskConstant.RUNTYPE).asInstanceOf[String]
      if (StringUtils.isEmpty(runType)) runType = EntranceConfiguration.DEFAULT_RUN_TYPE.getValue
      //If formatCode is not empty, we need to format it(如果formatCode 不为空的话，我们需要将其进行格式化)
      if (formatCode) executionCode = format(executionCode)
      task.setExecutionCode(executionCode)
    }
    task.setSource(source)
    task.setEngineType(runType)
    //为了兼容代码，让engineType和runType都有同一个属性
    task.setRunType(runType)
    task.setExecuteApplicationName(executeApplicationName)
    task.setRequestApplicationName(creator)
    task.setStatus(SchedulerEventState.Inited.toString)
    task
  }
  //todo to format code using proper way
  private def format(code:String):String = code

} 

package com.webank.wedatasphere.linkis.entrance

import com.webank.wedatasphere.linkis.common.exception.{DWCException, DWCRuntimeException, ErrorException}
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.entrance.exception.{EntranceErrorException, SubmitFailedException}
import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob
import com.webank.wedatasphere.linkis.entrance.log.LogReader
import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
import com.webank.wedatasphere.linkis.scheduler.queue.{Job, SchedulerEventState}
import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.exception.ExceptionUtils


  def execute(params: java.util.Map[String, Any]): String = {
    if(!params.containsKey(EntranceServer.DO_NOT_PRINT_PARAMS_LOG)) info("received a request: " + params)
    else params.remove(EntranceServer.DO_NOT_PRINT_PARAMS_LOG)
    var task = getEntranceContext.getOrCreateEntranceParser().parseToTask(params)
    task match {
      case t: RequestPersistTask => if(StringUtils.isBlank(t.getRequestApplicationName))
        throw new EntranceErrorException(20038, "requestApplicationName cannot be empty.")
      case _ =>
    }
    //After parse the map into a task, we need to store it in the database, and the task can get a unique taskID.
    //将map parse 成 task 之后，我们需要将它存储到数据库中，task可以获得唯一的taskID
    getEntranceContext.getOrCreatePersistenceManager().createPersistenceEngine().persist(task)
    val logAppender = new java.lang.StringBuilder()
    Utils.tryThrow(getEntranceContext.getOrCreateEntranceInterceptors().foreach(int => task = int.apply(task, logAppender))) { t =>
      val error = t match {
        case error: ErrorException => error
        case t1:Throwable => val exception = new EntranceErrorException(20039, "解析task失败！原因：" + ExceptionUtils.getRootCauseMessage(t1))
          exception.initCause(t1)
          exception
        case _ => new EntranceErrorException(20039, "解析task失败！原因：" + ExceptionUtils.getRootCauseMessage(t))
      }
      task match {
        case t: RequestPersistTask =>
          t.setErrCode(error.getErrCode)
          t.setErrDesc(error.getDesc)
          t.setStatus(SchedulerEventState.Failed.toString)
          t.setProgress(1.0f)
        case _ =>
      }
      getEntranceContext.getOrCreatePersistenceManager().createPersistenceEngine().updateIfNeeded(task)
      error
    }
    getEntranceContext.getOrCreatePersistenceManager().createPersistenceEngine().updateIfNeeded(task)
    val job = getEntranceContext.getOrCreateEntranceParser().parseToJob(task)
    job.init()
    job.setLogListener(getEntranceContext.getOrCreateLogManager())
    job.setProgressListener(getEntranceContext.getOrCreatePersistenceManager())
    job.setJobListener(getEntranceContext.getOrCreatePersistenceManager())
    job match {
      case entranceJob: EntranceJob => entranceJob.setEntranceListenerBus(getEntranceContext.getOrCreateEventListenerBus)
      case _ =>
    }
    Utils.tryCatch{
      if(logAppender.length() > 0) job.getLogListener.foreach(_.onLogUpdate(job, logAppender.toString.trim))
    }{
      t => logger.error("Failed to write init log, reason: ", t)
    }

    Utils.tryThrow{
      getEntranceContext.getOrCreateScheduler().submit(job)
    }{t =>
      job.onFailure("Submitting the query failed!(提交查询失败！)", t)
      val _task = getEntranceContext.getOrCreateEntranceParser().parseToTask(job)
      getEntranceContext.getOrCreatePersistenceManager().createPersistenceEngine().updateIfNeeded(_task)
      t match {
        case e: DWCException => e
        case e: DWCRuntimeException => e
        case t: Throwable =>
          new SubmitFailedException(30009, "Submitting the query failed!(提交查询失败！)" + ExceptionUtils.getRootCauseMessage(t), t)
      }
    }
    task match {
      case requestPersistTask:RequestPersistTask => logger.info(s"Job ${job.getId} submitted and taskID is ${requestPersistTask.getTaskID}")
      case _ => info(s"Job $job submitted!")
    }
    job.getId
  }

  def logReader(execId: String): LogReader

  def getJob(execId: String): Option[Job] = getEntranceContext.getOrCreateScheduler().get(execId).map(_.asInstanceOf[Job])

  private[entrance] def getEntranceWebSocketService: Option[EntranceWebSocketService] = if(ServerConfiguration.BDP_SERVER_SOCKET_MODE.getValue) {
    if(entranceWebSocketService.isEmpty) synchronized {
      if(entranceWebSocketService.isEmpty) {
        entranceWebSocketService = Some(new EntranceWebSocketService)
        entranceWebSocketService.foreach(_.setEntranceServer(this))
        entranceWebSocketService.foreach(getEntranceContext.getOrCreateEventListenerBus.addListener)
      }
    }
    entranceWebSocketService
  } else None

}
object EntranceServer {
  val DO_NOT_PRINT_PARAMS_LOG = "doNotPrintParamsLog"
} 

package com.webank.wedatasphere.linkis.entrance.log

import org.apache.commons.lang.StringUtils

class CacheLogWriter(logPath:String,
                     charset:String,
                     sharedCache:Cache,
                     user: String)
  extends AbstractLogWriter(logPath, user, charset) {

  def getCache:Option[Cache] = Some(sharedCache)

  private def cache(msg:String): Unit = {
    this synchronized {
      val removed = sharedCache.cachedLogs.add(msg)
      if (removed != null){
        val logs = sharedCache.cachedLogs.toList
        val sb = new StringBuilder
        sb.append(removed).append("\n")
        logs.filter(_ != null).foreach(log => sb.append(log).append("\n"))
        sharedCache.cachedLogs.fakeClear()
        super.write(sb.toString())
      }
    }
  }
  override def write(msg: String): Unit = {
    if (StringUtils.isBlank(msg)){
      cache("")
    }else{
      val rows = msg.split("\n")
      rows.foreach(row => {
        if (row == null) cache("") else cache(row)
      })
    }
  }

  override def flush(): Unit = {
    sharedCache.cachedLogs.toList.filter(StringUtils.isNotEmpty).foreach(super.write)
    sharedCache.cachedLogs.clear()
    super.flush()
  }

} 

package com.webank.wedatasphere.linkis.entrance.log

import java.io.{Closeable, Flushable, OutputStream}
import java.util

import com.webank.wedatasphere.linkis.common.io.FsPath
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.entrance.exception.EntranceErrorException
import com.webank.wedatasphere.linkis.storage.FSFactory
import com.webank.wedatasphere.linkis.storage.utils.FileSystemUtils
import org.apache.commons.io.IOUtils
import org.apache.commons.lang.StringUtils


abstract class LogWriter(charset: String) extends Closeable with Flushable with Logging {

  private var firstWrite = true

  protected val outputStream: OutputStream

  def write(msg: String): Unit = synchronized {
    val log = if (!firstWrite) "\n" + msg else {
      firstWrite = false
      msg
    }
    Utils.tryQuietly({
      outputStream.write(log.getBytes(charset))
      outputStream.flush()
    }, t => {
      warn("error when write query log to outputStream.", t)
      info(msg)
    })
  }



  def flush(): Unit = Utils.tryQuietly(outputStream.flush(),  t => {
    warn("Error encounters when flush log,", t)
  })


  def close(): Unit = {
    flush()
    if (outputStream != null) IOUtils.closeQuietly(outputStream)

  }
}

abstract class AbstractLogWriter(logPath: String,
                                 user: String,
                                 charset: String) extends LogWriter(charset) {
  if(StringUtils.isBlank(logPath)) throw new EntranceErrorException(20301, "logPath cannot be empty.")
  protected val fileSystem = FSFactory.getFs(new FsPath(logPath))
  fileSystem.init(new util.HashMap[String, String]())

  protected val outputStream: OutputStream = {
    FileSystemUtils.createNewFile(new FsPath(logPath), true)
    fileSystem.write(new FsPath(logPath), true)
  }

  override def close(): Unit = {
    super.close()
    if (fileSystem != null) Utils.tryQuietly(fileSystem.close(), t => {
      warn("Error encounters when closing fileSystem", t)
    })
  }
} 

package com.webank.wedatasphere.linkis.entrance.scheduler

import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob
import com.webank.wedatasphere.linkis.entrance.persistence.HaPersistenceTask
import com.webank.wedatasphere.linkis.protocol.config.{RequestQueryAppConfig, ResponseQueryConfig}
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.scheduler.queue.parallelqueue.ParallelGroup
import com.webank.wedatasphere.linkis.scheduler.queue.{Group, GroupFactory, SchedulerEvent}
import com.webank.wedatasphere.linkis.server.JMap
import org.apache.commons.lang.StringUtils
import org.slf4j.{Logger, LoggerFactory}


class EntranceGroupFactory extends GroupFactory {

  private val groupNameToGroups = new JMap[String, Group]
  private val logger:Logger = LoggerFactory.getLogger(classOf[EntranceGroupFactory])
  override def getOrCreateGroup(groupName: String): Group = {
    if(!groupNameToGroups.containsKey(groupName)) synchronized{
      //TODO Query the database and get initCapacity, maxCapacity, maxRunningJobs, maxAskExecutorTimes(查询数据库，拿到initCapacity、maxCapacity、maxRunningJobs、maxAskExecutorTimes)
      val initCapacity = 100
      val maxCapacity = 100
      var maxRunningJobs =  EntranceConfiguration.WDS_LINKIS_INSTANCE.getValue
      val maxAskExecutorTimes = EntranceConfiguration.MAX_ASK_EXECUTOR_TIME.getValue.toLong
      if (groupName.split("_").length < 2){
        logger.warn(s"name style of group: $groupName is not correct, we will set default value for the group")
      }else{
        val sender:Sender = Sender.getSender(EntranceConfiguration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME.getValue)
        val creator = groupName.split("_")(0)
        val username = groupName.split("_")(1)
        val engineName = EntranceConfiguration.ENGINE_SPRING_APPLICATION_NAME.getValue
        val engineType = if (engineName.trim().toLowerCase().contains("engine")) engineName.substring(0, engineName.length - "engine".length) else "spark"
        logger.info(s"Getting parameters for $groupName(正在为 $groupName 获取参数) username: $username, creator:$creator, engineType: $engineType")
        val keyAndValue = sender.ask(RequestQueryAppConfig(username, creator, engineType)).asInstanceOf[ResponseQueryConfig].getKeyAndValue
        try{
          maxRunningJobs = Integer.parseInt(keyAndValue.get(EntranceConfiguration.WDS_LINKIS_INSTANCE.key))
        }catch{
          case t:Throwable => logger.warn("Get maxRunningJobs from configuration server failed! Next use the default value to continue.",t)
        }
      }
      logger.info("groupName: {} =>  maxRunningJobs is {}", groupName, maxRunningJobs)
      val group = new ParallelGroup(groupName, initCapacity, maxCapacity)
      group.setMaxRunningJobs(maxRunningJobs)
      group.setMaxAskExecutorTimes(maxAskExecutorTimes)
      if(!groupNameToGroups.containsKey(groupName)) groupNameToGroups.put(groupName, group)
    }
    groupNameToGroups.get(groupName)
  }


  override def getGroupNameByEvent(event: SchedulerEvent): String = event match {
    case job: EntranceJob =>
      job.getTask match {
        case HaPersistenceTask(task) =>
          "HA"
        case _ =>EntranceGroupFactory.getGroupName(job.getCreator, job.getUser)
      }
  }
}
object EntranceGroupFactory {
  def getGroupName(creator: String, user: String): String = {
    if (StringUtils.isNotEmpty(creator)) creator + "_" + user
    else EntranceConfiguration.DEFAULT_REQUEST_APPLICATION_NAME.getValue + "_" + user
  }
} 

package com.webank.wedatasphere.linkis.bml.hook

import java.io.File
import java.util

import com.webank.wedatasphere.linkis.bml.client.{BmlClient, BmlClientFactory}
import com.webank.wedatasphere.linkis.bml.exception.BmlHookDownloadException
import com.webank.wedatasphere.linkis.bml.utils.BmlHookUtils
import com.webank.wedatasphere.linkis.common.exception.ErrorException
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.engine.ResourceExecuteRequest
import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
import com.webank.wedatasphere.linkis.engine.extension.EnginePreExecuteHook
import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteRequest
import org.apache.commons.lang.StringUtils

import scala.collection.JavaConversions._

class BmlEnginePreExecuteHook extends EnginePreExecuteHook with Logging{
  override val hookName: String = "BmlEnginePreExecuteHook"

  val RESOURCES_STR = "resources"

  val RESOURCE_ID_STR = "resourceId"

  val VERSION_STR = "version"

  val FILE_NAME_STR = "fileName"

  val processUser:String = System.getProperty("user.name")

  val defaultUser:String = "hadoop"

  val bmlClient:BmlClient = if (StringUtils.isNotEmpty(processUser))
    BmlClientFactory.createBmlClient(processUser) else BmlClientFactory.createBmlClient(defaultUser)

  val seperator:String = File.separator

  val pathType:String = "file://"

  override def callPreExecuteHook(engineExecutorContext: EngineExecutorContext, executeRequest: ExecuteRequest, code: String): String = {
    val workDir = BmlHookUtils.getCurrentWorkDir
    val jobId = engineExecutorContext.getJobId
    executeRequest match {
      case resourceExecuteRequest:ResourceExecuteRequest => val resources = resourceExecuteRequest.resources
        if (null == resources) return code
        resources foreach {
          case resource:util.Map[String, Object] => val fileName = resource.get(FILE_NAME_STR).toString
            val resourceId = resource.get(RESOURCE_ID_STR).toString
            val version = resource.get(VERSION_STR).toString
            val fullPath = if (workDir.endsWith(seperator)) pathType + workDir + fileName else
              pathType + workDir + seperator + fileName
            val response = Utils.tryCatch{
              bmlClient.downloadResource(processUser, resourceId, version, fullPath, true)
            }{
              case error:ErrorException => logger.error("download resource for {} failed", error)
                throw error
              case t:Throwable => logger.error(s"download resource for $jobId failed", t)
                val e1 = BmlHookDownloadException(t.getMessage)
                e1.initCause(t)
                throw t
            }
            if (response.isSuccess){
              logger.info(s"for job $jobId resourceId $resourceId version $version download to path $fullPath ok")
            }else{
              logger.warn(s"for job $jobId resourceId $resourceId version $version download to path $fullPath Failed")
            }
          case _ => logger.warn("job resource cannot download")
        }
      case _ =>
    }
    if (StringUtils.isNotBlank(code)) code else executeRequest.code
  }
} 

package com.webank.wedatasphere.linkis.entranceclient.conf

import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
import com.webank.wedatasphere.linkis.entranceclient
import org.apache.commons.lang.StringUtils
import org.springframework.context.annotation.{Condition, ConditionContext}
import org.springframework.core.`type`.AnnotatedTypeMetadata


class MultiEntranceCondition extends Condition {
  override def matches(conditionContext: ConditionContext, annotatedTypeMetadata: AnnotatedTypeMetadata): Boolean =
    entranceclient.conf.MultiEntranceCondition.isMultiEntranceApplication
}
class SingleEntranceCondition extends Condition {
  override def matches(conditionContext: ConditionContext, annotatedTypeMetadata: AnnotatedTypeMetadata): Boolean =
    !entranceclient.conf.MultiEntranceCondition.isMultiEntranceApplication
}
object MultiEntranceCondition {
  def isMultiEntranceApplication: Boolean = StringUtils.isNotBlank(EntranceConfiguration.ENGINE_MANAGER_SPRING_APPLICATION_NAME.getValue)
} 

package com.webank.wedatasphere.linkis.ujes.jdbc

import java.util
import java.util.Properties

import com.webank.wedatasphere.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy
import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfigBuilder
import com.webank.wedatasphere.linkis.ujes.client.UJESClient
import UJESSQLDriverMain._
import org.apache.commons.lang.StringUtils


object UJESClientFactory {

  private val ujesClients = new util.HashMap[String, UJESClient]

  def getUJESClient(props: Properties): UJESClient = {
    val host = props.getProperty(HOST)
    val port = props.getProperty(PORT)
    val serverUrl = if(StringUtils.isNotBlank(port)) s"http://$host:$port" else "http://" + host
    if(ujesClients.containsKey(serverUrl)) ujesClients.get(serverUrl)
    else serverUrl.intern synchronized {
      if(ujesClients.containsKey(serverUrl)) return ujesClients.get(serverUrl)
      val ujesClient = createUJESClient(serverUrl, props)
      ujesClients.put(serverUrl, ujesClient)
      ujesClient
    }
  }

  private def createUJESClient(serverUrl: String, props: Properties): UJESClient = {
    val clientConfigBuilder = DWSClientConfigBuilder.newBuilder()
    clientConfigBuilder.addUJESServerUrl(serverUrl)
    clientConfigBuilder.setAuthTokenKey(props.getProperty(USER))
    clientConfigBuilder.setAuthTokenValue(props.getProperty(PASSWORD))
    clientConfigBuilder.setAuthenticationStrategy(new StaticAuthenticationStrategy())
    clientConfigBuilder.readTimeout(100000)
    clientConfigBuilder.maxConnectionSize(20)
    clientConfigBuilder.readTimeout(10000)
    val params = props.getProperty(PARAMS)
    var versioned = false
    if(StringUtils.isNotBlank(params)) {
      var enableDiscovery = false
      params.split(PARAM_SPLIT).foreach { kv =>
        kv.split(KV_SPLIT) match {
          case Array(VERSION, v) =>
            clientConfigBuilder.setDWSVersion(v)
            versioned = true
          case Array(MAX_CONNECTION_SIZE, v) =>
            clientConfigBuilder.maxConnectionSize(v.toInt)
          case Array(READ_TIMEOUT, v) =>
            clientConfigBuilder.readTimeout(v.toLong)
          case Array(ENABLE_DISCOVERY, v) =>
            clientConfigBuilder.discoveryEnabled(v.toBoolean)
            enableDiscovery = true
          case Array(ENABLE_LOADBALANCER, v) if enableDiscovery =>
            clientConfigBuilder.loadbalancerEnabled(v.toBoolean)
          case _ =>
        }
      }
    }
    if(!versioned) clientConfigBuilder.setDWSVersion("v" + DEFAULT_VERSION)
    UJESClient(clientConfigBuilder.build())
  }

} 

package com.webank.wedatasphere.linkis.engine.cs


import java.util
import java.util.regex.Pattern

import com.webank.wedatasphere.linkis.cs.client.service.CSResourceService
import com.webank.wedatasphere.linkis.engine.PropertiesExecuteRequest
import org.apache.commons.lang.StringUtils

import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer


class CSResourceParser {

  private val pb = Pattern.compile("cs://[^\\s\"]+[$\\s]{0,1}", Pattern.CASE_INSENSITIVE)

  private val PREFIX = "cs://"

  private def getPreFixResourceNames(code: String): Array[String] = {
    val bmlResourceNames = new ArrayBuffer[String]()
    val mb = pb.matcher(code)
    while (mb.find) bmlResourceNames.append(mb.group.trim)
    bmlResourceNames.toArray
  }

  def parse(executeRequest: PropertiesExecuteRequest, code: String, contextIDValueStr: String, nodeNameStr: String): String = {

    //TODO getBMLResource peaceWong
    val bmlResourceList = CSResourceService.getInstance().getUpstreamBMLResource(contextIDValueStr, nodeNameStr)

    val parsedResources = new util.ArrayList[util.Map[String, Object]]()
    val preFixResourceNames = getPreFixResourceNames(code)

    val preFixNames = new ArrayBuffer[String]()
    val parsedNames = new ArrayBuffer[String]()
    preFixResourceNames.foreach { preFixResourceName =>
      val resourceName = preFixResourceName.replace(PREFIX, "").trim
      val bmlResourceOption = bmlResourceList.find(_.getDownloadedFileName.equals(resourceName))
      if (bmlResourceOption.isDefined) {
        val bmlResource = bmlResourceOption.get
        val map = new util.HashMap[String, Object]()
        map.put("resourceId", bmlResource.getResourceId)
        map.put("version", bmlResource.getVersion)
        map.put("fileName", resourceName)
        parsedResources.add(map)
        preFixNames.append(preFixResourceName)
        parsedNames.append(resourceName)
      }

    }
    executeRequest.properties.put("resources", parsedResources)
    StringUtils.replaceEach(code, preFixNames.toArray, parsedNames.toArray)
  }

} 

package com.webank.wedatasphere.linkis.engine.cs

import java.util.Date

import com.webank.wedatasphere.linkis.common.io.resultset.ResultSetWriter
import com.webank.wedatasphere.linkis.common.io.{MetaData, Record}
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.cs.client.service.CSTableService
import com.webank.wedatasphere.linkis.cs.client.utils.{ContextServiceUtils, SerializeHelper}
import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.{ContextScope, ContextType}
import com.webank.wedatasphere.linkis.cs.common.entity.metadata.{CSColumn, CSTable}
import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextKey
import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils
import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
import com.webank.wedatasphere.linkis.storage.domain.Column
import com.webank.wedatasphere.linkis.storage.utils.StorageUtils
import org.apache.commons.lang.StringUtils


object CSTableRegister extends Logging{

  def registerTempTable(engineExecutorContext: EngineExecutorContext,
                        writer: ResultSetWriter[_ <: MetaData, _ <: Record], alias: String, columns: Array[Column]): Unit = {

    val contextIDValueStr = ContextServiceUtils.getContextIDStrByMap(engineExecutorContext.getProperties)
    val nodeNameStr = ContextServiceUtils.getNodeNameStrByMap(engineExecutorContext.getProperties)

    if (StringUtils.isNotBlank(contextIDValueStr) && StringUtils.isNotBlank(nodeNameStr)) {
      info(s"Start to register TempTable nodeName:$nodeNameStr")
      writer.flush()
      val tableName = if (StringUtils.isNotBlank(alias)) s"${CSCommonUtils.CS_TMP_TABLE_PREFIX}${nodeNameStr}_${alias}" else {
        var i = 1;
        var rsName: String = null;
        while (StringUtils.isEmpty(rsName)) {
          val tmpTable = s"${CSCommonUtils.CS_TMP_TABLE_PREFIX}${nodeNameStr}_rs${i}"
          i = i + 1
          val contextKey = new CommonContextKey
          contextKey.setContextScope(ContextScope.FRIENDLY)
          contextKey.setContextType(ContextType.METADATA)
          contextKey.setKey(CSCommonUtils.getTableKey(nodeNameStr, tmpTable))
          val table = CSTableService.getInstance().getCSTable(contextIDValueStr, SerializeHelper.serializeContextKey(contextKey))
          if (null == table) {
            rsName = tmpTable
          }
        }
        rsName
      }
      val csTable = new CSTable
      csTable.setName(tableName)
      csTable.setAlias(alias)
      csTable.setAvailable(true)
      csTable.setComment("cs temp table")
      csTable.setCreateTime(new Date())
      csTable.setCreator(StorageUtils.getJvmUser)
      csTable.setExternalUse(true)
      csTable.setImport(false)
      csTable.setLocation(writer.toString)
      csTable.setPartitionTable(false)
      csTable.setView(true)
      val csColumns = columns.map { column =>
        val csColumn = new CSColumn
        csColumn.setName(column.columnName)
        csColumn.setType(column.dataType.typeName)
        csColumn.setComment(column.comment)
        csColumn
      }
      csTable.setColumns(csColumns)
      val contextKey = new CommonContextKey
      contextKey.setContextScope(ContextScope.PUBLIC)
      contextKey.setContextType(ContextType.METADATA)
      contextKey.setKey(CSCommonUtils.getTableKey(nodeNameStr, tableName))
      CSTableService.getInstance().putCSTable(contextIDValueStr, SerializeHelper.serializeContextKey(contextKey), csTable)
      info(s"Finished to register TempTable nodeName:$nodeNameStr")
    }
  }
} 

package com.webank.wedatasphere.linkis.engine

import java.text.SimpleDateFormat
import java.util.Date

import com.webank.wedatasphere.linkis.DataWorkCloudApplication
import com.webank.wedatasphere.linkis.common.conf.DWCArgumentsParser
import com.webank.wedatasphere.linkis.common.utils.Utils
import com.webank.wedatasphere.linkis.engine.conf.EngineConfiguration
import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration
import org.apache.commons.lang.StringUtils
import org.slf4j.LoggerFactory


object DataWorkCloudEngineApplication {

  val userName:String = System.getProperty("user.name")
  val hostName:String = Utils.getComputerName
  val appName:String = EngineConfiguration.ENGINE_SPRING_APPLICATION_NAME.getValue
  val prefixName:String = EngineConfiguration.ENGINE_LOG_PREFIX.getValue
  val timeStamp:Long = System.currentTimeMillis()
  private val timeFormat = new SimpleDateFormat("yyyy-MM-dd_HH:mm:ss")
  private val dateFormat = new SimpleDateFormat("yyyy-MM-dd")
  val time:String = timeFormat.format(new Date(timeStamp))
  val date:String = dateFormat.format(new Date(timeStamp))

  val isTimeStampSuffix:Boolean = "true".equalsIgnoreCase(EngineConfiguration.ENGINE_LOG_TIME_STAMP_SUFFIX.getValue)
  val shortLogFile:String =
    if (isTimeStampSuffix) appName + "_" + hostName + "_" + userName + "_"  + time + ".log"
    else appName + "_" + hostName + "_" + userName + ".log"
  val logName:String =
    if(isTimeStampSuffix) prefixName + "/" + userName + "/" + shortLogFile
    else prefixName + "/" + shortLogFile
  System.setProperty("engineLogFile", logName)
  System.setProperty("shortEngineLogFile", shortLogFile)
//  System.setProperty("engineLogFile", logName)
//  val context:LoggerContext = LogManager.getContext(false).asInstanceOf[LoggerContext]
//  val path:String = getClass.getResource("/").getPath
//  val log4j2XMLFile:File = new File(path + "/log4j2-engine.xml")
//  val configUri:URI = log4j2XMLFile.toURI
//  context.setConfigLocation(configUri)
  private val logger = LoggerFactory.getLogger(getClass)
  logger.info(s"Now log4j2 Rolling File is set to be $logName")
  logger.info(s"Now shortLogFile is set to be $shortLogFile")
  def main(args: Array[String]): Unit = {
    val parser = DWCArgumentsParser.parse(args)
    DWCArgumentsParser.setDWCOptionMap(parser.getDWCConfMap)
    val existsExcludePackages = ServerConfiguration.BDP_SERVER_EXCLUDE_PACKAGES.getValue
    if(StringUtils.isEmpty(existsExcludePackages))
      DataWorkCloudApplication.setProperty(ServerConfiguration.BDP_SERVER_EXCLUDE_PACKAGES.key, "com.webank.wedatasphere.linkis.enginemanager")
    else
      DataWorkCloudApplication.setProperty(ServerConfiguration.BDP_SERVER_EXCLUDE_PACKAGES.key, existsExcludePackages + ",com.webank.wedatasphere.linkis.enginemanager")
    DataWorkCloudApplication.main(DWCArgumentsParser.formatSpringOptions(parser.getSpringConfMap))
  }
} 

package com.webank.wedatasphere.linkis.engine.execute.hook

import java.io.File

import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineHook}
import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteRequest, RunTypeExecuteRequest}
import com.webank.wedatasphere.linkis.server.JMap
import org.apache.commons.io.FileUtils
import org.apache.commons.lang.StringUtils

import scala.collection.mutable.ArrayBuffer


@Deprecated
//changed to UdfLoadEngineHook
abstract class CodeGeneratorEngineHook extends EngineHook with Logging{ self =>
  val udfPathProp = "udf.paths"
  protected var creator: String = _
  protected var user: String = _
  protected var initSpecialCode: String = _
  protected val runType: String

  protected def acceptCodeType(line: String): Boolean

  protected def generateCode(): Array[String] = {
    val codeBuffer = new ArrayBuffer[String]
    val statementBuffer = new ArrayBuffer[String]
    var accept = true
    initSpecialCode.split("\n").foreach{
      case "" =>
      case l if l.startsWith("%") =>
        if(acceptCodeType(l)){
          accept = true
          codeBuffer.append(statementBuffer.mkString("\n"))
          statementBuffer.clear()
        }else{
          accept = false
        }
      case l if accept => statementBuffer.append(l)
      case _ =>
    }
    if(statementBuffer.nonEmpty) codeBuffer.append(statementBuffer.mkString("\n"))
    codeBuffer.toArray
  }

  override def beforeCreateEngine(params: JMap[String, String]): JMap[String, String] = {
    creator = params.get("creator")
    user = params.get("user")
    initSpecialCode = StringUtils.split(params.get(udfPathProp), ",").map(readFile).mkString("\n")
    params
  }

  override def afterCreatedEngine(executor: EngineExecutor): Unit = {
    generateCode().foreach {
      case "" =>
      case c: String =>
        info("Submit udf registration to engine, code: " + c)
        executor.execute(new ExecuteRequest with RunTypeExecuteRequest{
          override val code: String = c
          override val runType: String = self.runType
        })
        info("executed code: " + c)
    }
  }

  protected def readFile(path: String): String = {
    info("read file: " + path)
    val file = new File(path)
    if(file.exists()){
      FileUtils.readFileToString(file)
    } else {
      info("udf file: [" + path + "] doesn't exist, ignore it.")
      ""
    }
  }
}
@Deprecated
class SqlCodeGeneratorEngineHook extends CodeGeneratorEngineHook{
  override val runType = "sql"
  override protected def acceptCodeType(line: String): Boolean = {
    line.startsWith("%sql")
  }
}
@Deprecated
class PythonCodeGeneratorEngineHook extends CodeGeneratorEngineHook{
  override val runType = "python"
  override protected def acceptCodeType(line: String): Boolean = {
    line.startsWith("%python")
  }
}
@Deprecated
class ScalaCodeGeneratorEngineHook extends CodeGeneratorEngineHook{
  override val runType = "scala"
  override protected def acceptCodeType(line: String): Boolean = {
    line.startsWith("%scala")
  }
} 

class JobInfoAction extends GetAction with UJESJobAction {
  private var taskId: String = _
  override def suffixURLs: Array[String] = Array("jobhistory", taskId, "get")
}
object JobInfoAction {
  def builder(): Builder = new Builder
  class Builder private[JobInfoAction]() {
    private var taskId: String = _
    private var user: String = _
    def setTaskId(taskId: String): Builder = {
      this.taskId = taskId
      this
    }
    def setUser(user: String): Builder = {
      this.user = user
      this
    }
    def setTaskId(jobExecuteResult: JobExecuteResult): Builder = {
      this.user = jobExecuteResult.getUser
      setTaskId(jobExecuteResult.getTaskID)
    }
    def build(): JobInfoAction = {
      if(StringUtils.isBlank(taskId)) throw new UJESClientBuilderException("taskId is needed!")
      if(StringUtils.isBlank(user)) throw new UJESClientBuilderException("user is needed!")
      val jobInfoAction = new JobInfoAction
      jobInfoAction.taskId = taskId
      jobInfoAction.setUser(user)
      jobInfoAction
    }
  }
} 

package com.webank.wedatasphere.linkis.enginemanager.hook

import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.enginemanager.{Engine, EngineHook}
import com.webank.wedatasphere.linkis.enginemanager.conf.EngineManagerConfiguration.ENGINE_UDF_APP_NAME
import com.webank.wedatasphere.linkis.protocol.engine.RequestEngine
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.udf.api.rpc.{RequestUdfTree, ResponseUdfTree}
import com.webank.wedatasphere.linkis.udf.entity.{UDFInfo, UDFTree}
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.io.FileUtils
import org.apache.commons.lang.StringUtils
import org.codehaus.jackson.map.ObjectMapper

import scala.collection.JavaConversions._
import scala.collection.mutable

class JarLoaderEngineHook extends EngineHook with Logging{

  override def beforeCreateSession(requestEngine: RequestEngine): RequestEngine = {
    info("start loading UDFs")
    val udfInfos = extractUdfInfos(requestEngine).filter{info => info.getUdfType == 0 && info.getExpire == false && StringUtils.isNotBlank(info.getPath) && isJarExists(info) && info.getLoad == true }
    // add to class path
    val jars = new mutable.HashSet[String]()
    udfInfos.foreach{udfInfo => jars.add("file://" + udfInfo.getPath)}
    val jarPaths = jars.mkString(",")
    if(StringUtils.isBlank(requestEngine.properties.get("jars"))){
      requestEngine.properties.put("jars", jarPaths)
    } else {
      requestEngine.properties.put("jars", requestEngine.properties.get("jars") + "," + jarPaths)
    }
    info("added jars: " + jarPaths)
    //jars.foreach(fetchRemoteFile)
    //info("copied jars.")
    info("end loading UDFs")
    requestEngine
  }

  override def afterCreatedSession(engine: Engine, requestEngine: RequestEngine): Unit = {
  }

  protected def isJarExists(udfInfo: UDFInfo) : Boolean = {
    true
//    if(FileUtils.getFile(udfInfo.getPath).exists()){
//      true
//    } else {
//      info(s"The jar file [${udfInfo.getPath}] of UDF [${udfInfo.getUdfName}] doesn't exist, ignore it.")
//      false
//    }
  }

  protected def extractUdfInfos(requestEngine: RequestEngine): mutable.ArrayBuffer[UDFInfo] = {
    val udfInfoBuilder = new mutable.ArrayBuffer[UDFInfo]
    val userName = requestEngine.user
    val udfTree = queryUdfRpc(userName)
    extractUdfInfos(udfInfoBuilder, udfTree, userName)
    udfInfoBuilder
  }

  protected def extractUdfInfos(udfInfoBuilder: mutable.ArrayBuffer[UDFInfo], udfTree: UDFTree, userName: String) : Unit = {
    if(CollectionUtils.isNotEmpty(udfTree.getUdfInfos)){
      for(udfInfo <- udfTree.getUdfInfos){
        udfInfoBuilder.append(udfInfo)
      }
    }
    if(CollectionUtils.isNotEmpty(udfTree.getChildrens)){
      for(child <- udfTree.getChildrens){
        var childInfo = child
        if(TreeType.specialTypes.contains(child.getUserName)){
          childInfo = queryUdfRpc(userName, child.getId, child.getUserName)
        } else {
          childInfo = queryUdfRpc(userName, child.getId, TreeType.SELF)
        }
        extractUdfInfos(udfInfoBuilder, childInfo, userName)
      }
    }
  }

  private def queryUdfRpc(userName: String, treeId: Long = -1, treeType: String = "self"): UDFTree = {
    val udfTree = Sender.getSender(ENGINE_UDF_APP_NAME.getValue)
      .ask(RequestUdfTree(userName, treeType, treeId, "udf"))
      .asInstanceOf[ResponseUdfTree]
      .udfTree
    //info("got udf tree:" + new ObjectMapper().writer().withDefaultPrettyPrinter().writeValueAsString(udfTree))
    udfTree
  }
} 

package com.webank.wedatasphere.linkis.enginemanager

import com.webank.wedatasphere.linkis.enginemanager.impl.{UserEngineResource, UserTimeoutEngineResource}
import com.webank.wedatasphere.linkis.protocol.engine.{RequestEngine, TimeoutRequestEngine}
import com.webank.wedatasphere.linkis.resourcemanager.Resource
import org.apache.commons.lang.StringUtils


trait AbstractEngineResourceFactory extends EngineResourceFactory {

  protected def getRequestResource(properties: java.util.Map[String, String]): Resource

  override def createEngineResource(request: RequestEngine): EngineResource = {
    val user = if(StringUtils.isEmpty(request.user)) request.creator
      else request.user
    val engineResource = request match {
      case timeoutRequest: TimeoutRequestEngine =>
        val engineResource = new UserTimeoutEngineResource
        engineResource.setTimeout(timeoutRequest.timeout)
        engineResource
      case _ => new UserEngineResource
    }
    engineResource.setCreator(request.creator)
    engineResource.setUser(user)
    engineResource.setResource(getRequestResource(request.properties))
    engineResource
  }
} 

package com.webank.wedatasphere.linkis.variable.service

import java.lang.Long
import java.util

import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.protocol.variable.ResponseQueryVariable
import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper
import com.webank.wedatasphere.linkis.variable.dao.VarMapper
import com.webank.wedatasphere.linkis.variable.entity.{VarKey, VarKeyUser, VarKeyValueVO}
import com.webank.wedatasphere.linkis.variable.exception.VariableException
import org.apache.commons.lang.StringUtils
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
import org.springframework.transaction.annotation.Transactional




  private def insertGlobalVariable(saveVariable: VarKeyValueVO, userName: String): Unit = {
    val newKey = new VarKey
    newKey.setApplicationID(-1L)
    newKey.setKey(saveVariable.getKey)
    varMapper.insertKey(newKey)
    val newValue = new VarKeyUser
    newValue.setApplicationID(-1L)
    newValue.setKeyID(newKey.getId)
    newValue.setUserName(userName)
    newValue.setValue(saveVariable.getValue)
    varMapper.insertValue(newValue)
  }

  private def updateGlobalVariable(saveVariable: VarKeyValueVO, valueID: Long): Unit = {
    varMapper.updateValue(valueID, saveVariable.getValue)
  }

  @Transactional
  override def saveGlobalVaraibles(globalVariables: util.List[_], userVariables: util.List[VarKeyValueVO], userName: String): Unit = {
    import scala.collection.JavaConversions._
    import scala.util.control.Breaks._
    val saves = globalVariables.map(f => BDPJettyServerHelper.gson.fromJson(BDPJettyServerHelper.gson.toJson(f), classOf[VarKeyValueVO]))
    saves.foreach {
      f =>
        if (StringUtils.isBlank(f.getKey) || StringUtils.isBlank(f.getValue)) throw new VariableException("key或value不能为空")
        var flag = true
        breakable {
          for (ele <- userVariables) {
            if (f.getKey.equals(ele.getKey)) {
              flag = false
              updateGlobalVariable(f, ele.getValueID)
              break()
            }
          }
        }
        if (flag) insertGlobalVariable(f, userName)
    }
    userVariables.foreach {
      f =>
        var flag = true
        breakable {
          for (ele <- saves) {
            if (ele.getKey.equals(f.getKey)) {
              flag = false
              break()
            }
          }
          if (flag) removeGlobalVariable(f.getKeyID)
        }
    }

  }
} 

package com.webank.wedatasphere.linkis.configuration.validate

import org.apache.commons.lang.StringUtils


class RegexValidator extends Validator{
  override var kind: String = "Regex"

  override def validate(value: String, range: String): Boolean = {
    value matches(range.r.regex)
  }
}

object RegexValidator {
  def main (args: Array[String] ): Unit = {
   print(StringUtils.isEmpty(""))
}
} 

package com.webank.wedatasphere.linkis.gateway.ujes.route

import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.gateway.exception.TooManyServiceException
import com.webank.wedatasphere.linkis.gateway.http.GatewayContext
import com.webank.wedatasphere.linkis.gateway.route.AbstractGatewayRouter
import com.webank.wedatasphere.linkis.gateway.ujes.parser.EntranceExecutionGatewayParser
import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant
import org.apache.commons.lang.StringUtils
import org.springframework.stereotype.Component


@Component
class EntranceGatewayRouter extends AbstractGatewayRouter {

  protected def findEntranceService(parsedServiceId: String) = findService(parsedServiceId, list => {
    val services = list.filter(_.toLowerCase.contains("entrance"))
    if(services.length == 1) Some(services.head)
    else if(services.isEmpty) None
    else {
      val errorMsg = new TooManyServiceException(s"Cannot find a correct serviceId for parsedServiceId $parsedServiceId, service list is: " + services)
      warn("", errorMsg)
      throw errorMsg
    }
  })
  override def route(gatewayContext: GatewayContext): ServiceInstance = {
    gatewayContext.getGatewayRoute.getRequestURI match {
      case EntranceGatewayRouter.ENTRANCE_REGEX(_) =>
        val creator = gatewayContext.getGatewayRoute.getParams.get(TaskConstant.REQUESTAPPLICATIONNAME)
        val applicationName = gatewayContext.getGatewayRoute.getServiceInstance.getApplicationName
        val serviceId = if(StringUtils.isBlank(creator) || creator == "IDE")
          findEntranceService(applicationName)
        else findEntranceService(creator).orElse {
          warn(s"Cannot find a service which named $creator, now redirect to $applicationName entrance.")
          findEntranceService(applicationName)
        }
        serviceId.map(ServiceInstance(_, gatewayContext.getGatewayRoute.getServiceInstance.getInstance)).orNull
      case _ => null
    }
  }
}
object EntranceGatewayRouter {
  val ENTRANCE_REGEX = (EntranceExecutionGatewayParser.ENTRANCE_HEADER + ".+").r
} 

package com.webank.wedatasphere.linkis.httpclient.dws.authentication

import com.webank.wedatasphere.linkis.common.utils.ByteTimeUtils
import com.webank.wedatasphere.linkis.httpclient.authentication.{AbstractAuthenticationStrategy, AuthenticationAction, AuthenticationResult}
import com.webank.wedatasphere.linkis.httpclient.dws.exception.AuthenticationFailedException
import com.webank.wedatasphere.linkis.httpclient.dws.request.DWSAuthenticationAction
import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSAuthenticationResult
import com.webank.wedatasphere.linkis.httpclient.request.{Action, UserAction, UserPwdAction}
import org.apache.commons.lang.StringUtils
import org.apache.http.HttpResponse


class StaticAuthenticationStrategy(override protected val sessionMaxAliveTime: Long) extends AbstractAuthenticationStrategy {
   def this() = this(ByteTimeUtils.timeStringAsMs("2h"))

  override protected def getAuthenticationAction(requestAction: Action, serverUrl: String): AuthenticationAction = {
    val action = new DWSAuthenticationAction(serverUrl)
    def pwd: String = if(StringUtils.isNotBlank(getClientConfig.getAuthTokenValue)) getClientConfig.getAuthTokenValue
      else throw new AuthenticationFailedException("the value of authTokenValue in ClientConfig must be exists, since no password is found to login.")
    requestAction match {
      case userPwd: UserPwdAction =>
        action.addRequestPayload("userName", userPwd.getUser)
        action.addRequestPayload("password", userPwd.getPassword.getOrElse(pwd))
      case userAction: UserAction =>
        action.addRequestPayload("userName", userAction.getUser)
        action.addRequestPayload("password", pwd)
      case _ =>
        if(StringUtils.isBlank(getClientConfig.getAuthTokenKey))
          throw new AuthenticationFailedException("the value of authTokenKey in ClientConfig must be exists, since no user is found to login.")
        action.addRequestPayload("userName", getClientConfig.getAuthTokenKey)
        action.addRequestPayload("password", pwd)
    }
    action
  }

  override def getAuthenticationResult(response: HttpResponse, requestAction: AuthenticationAction): AuthenticationResult = {
    new DWSAuthenticationResult(response, requestAction.serverUrl)
  }
} 

package com.webank.wedatasphere.linkis.httpclient.dws.config

import com.webank.wedatasphere.linkis.httpclient.config.ClientConfigBuilder
import com.webank.wedatasphere.linkis.httpclient.dws.exception.UnknownVersionException
import org.apache.commons.lang.StringUtils


class DWSClientConfigBuilder private() extends ClientConfigBuilder {

  private var dwsVersion: String = _

  def setDWSVersion(dwsVersion: String): this.type = {
    this.dwsVersion = dwsVersion
    this
  }

  override def build(): DWSClientConfig = {
    if(StringUtils.isBlank(dwsVersion)) throw new UnknownVersionException
    val clientConfig = new DWSClientConfig(super.build())
    clientConfig.setDWSVersion(dwsVersion)
    clientConfig
  }
}
object DWSClientConfigBuilder {
  def newBuilder(): DWSClientConfigBuilder = new DWSClientConfigBuilder
}