java.net.SocketTimeoutException Scala Examples

package com.wavesplatform.dex.time

import java.net.{InetAddress, SocketTimeoutException}
import java.util.concurrent.RejectedExecutionException
import java.util.concurrent.atomic.AtomicBoolean

import com.wavesplatform.dex.domain.utils.ScorexLogging
import monix.eval.Task
import monix.execution.schedulers.SchedulerService
import monix.execution.{ExecutionModel, Scheduler}
import org.apache.commons.net.ntp.NTPUDPClient

import scala.concurrent.duration.DurationInt

class NTP(ntpServer: String) extends Time with ScorexLogging with AutoCloseable {

  private val offsetPanicThreshold = 1000000L
  private val ExpirationTimeout    = 60.seconds
  private val RetryDelay           = 10.seconds
  private val ResponseTimeout      = 10.seconds

  private val duringShutdown = new AtomicBoolean(false)

  private implicit val scheduler: SchedulerService = Scheduler.singleThread(
    name = "time-impl",
    daemonic = false,
    executionModel = ExecutionModel.AlwaysAsyncExecution,
    reporter = {
      case _: RejectedExecutionException if duringShutdown.get() => // ignore
      case e: Throwable                                          => log.error("An uncaught error", e)
    }
  )

  private val client = new NTPUDPClient()
  client.setDefaultTimeout(ResponseTimeout.toMillis.toInt)

  @volatile private var offset = 0L
  private val updateTask: Task[Unit] = {
    def newOffsetTask: Task[Option[(InetAddress, java.lang.Long)]] = Task {
      try {
        val info = client.getTime(InetAddress.getByName(ntpServer))
        info.computeDetails()
        Option(info.getOffset).map { offset =>
          val r = if (Math.abs(offset) > offsetPanicThreshold) throw new Exception("Offset is suspiciously large") else offset
          (info.getAddress, r)
        }
      } catch {
        case _: SocketTimeoutException =>
          None
        case t: Throwable =>
          log.warn("Problems with NTP: ", t)
          None
      }
    }

    newOffsetTask.flatMap {
      case None if !scheduler.isShutdown => updateTask.delayExecution(RetryDelay)
      case Some((server, newOffset)) if !scheduler.isShutdown =>
        log.trace(s"Adjusting time with $newOffset milliseconds, source: ${server.getHostAddress}.")
        offset = newOffset
        updateTask.delayExecution(ExpirationTimeout)
      case _ => Task.unit
    }
  }

  def correctedTime(): Long = System.currentTimeMillis() + offset

  private var txTime: Long = 0

  def getTimestamp(): Long = {
    txTime = Math.max(correctedTime(), txTime + 1)
    txTime
  }

  private val taskHandle = updateTask.runAsync {
    case Left(e) => log.error(s"Error executing task", e)
    case _       =>
  }

  override def close(): Unit = if (duringShutdown.compareAndSet(false, true)) {
    log.info("Shutting down Time")
    taskHandle.cancel()
    if (client.isOpen) client.close()
    scheduler.shutdown()
  }
} 

package com.wixpress.build.codota

import java.net.SocketTimeoutException

import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.wixpress.build.codota.CodotaThinClient.{PathAttemptResult, RetriesLoop}
import org.slf4j.LoggerFactory

import scala.util.{Failure, Success, Try}
import scalaj.http.{Http, HttpOptions, HttpResponse}

class CodotaThinClient(token: String,
                       codePack: String,
                       baseURL: String = CodotaThinClient.DefaultBaseURL,
                       maxRetries: Int = 5) {

  private val logger = LoggerFactory.getLogger(getClass)
  private val timeout: Int = 1000
  private val mapper = new ObjectMapper()
    .registerModule(DefaultScalaModule)
    .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
    .configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, true)

  private val requestURLTemplate = s"$baseURL/api/codenav/artifact"

  def pathFor(artifactName: String): Option[String] = {
    retriesLoop(artifactName).collectFirst {
      case Success(p) => p
    }.flatten
  }

  private def retriesLoop(artifactName: String): RetriesLoop = {
    (1 to maxRetries).toStream.map(pathForArtifact(_, artifactName))
  }

  private def pathForArtifact(retry: Int, artifactName: String): PathAttemptResult = {
    pathForArtifact(artifactName) match {
      case Success(p) => Success(p)
      case Failure(e: SocketTimeoutException) if retry < maxRetries =>
        handleSocketTimeout(retry, artifactName, e)
      case Failure(e) => throw e
    }
  }

  private def handleSocketTimeout(retry: Int, artifactName: String, e: SocketTimeoutException) = {
    logger.warn(s"($retry/$maxRetries) Timeout when trying to get path for $artifactName")
    Thread.sleep(3000 / ((maxRetries - retry) + 1))
    Failure(e)
  }

  private def pathForArtifact(artifactName: String): PathAttemptResult = {
    for {
      response <- requestFor(artifactName)
      body <- responseBody(response)
      path <- extractPath(body, artifactName)
    } yield path
  }

  private def requestFor(artifactName: String) = {
    Try {
      Http(s"$requestURLTemplate/$artifactName/metadata")
        .param("codePack", codePack)
        .header("Authorization", s"bearer $token")
        .option(HttpOptions.readTimeout(timeout))
        .asString
    }
  }

  private def responseBody(resp: HttpResponse[String]) = {
    Try {
      val body = resp.body
      val code = resp.code

      code match {
        case 200 => body
        case 401 => throw NotAuthorizedException(body)
        case 404 if body.contains(codePack) => throw CodePackNotFoundException(body)
        case 403 => throw MissingCodePackException()
        case 404 => throw MetaDataOrArtifactNotFound(body)
      }
    }
  }

  private def extractPath(body: String, artifactName: String) = {
    Try {
      val metaData = mapper.readValue(body, classOf[ArtifactMetadata])
      Option(metaData.path)
    }
  }
}

object CodotaThinClient {
  val DefaultBaseURL = "https://gateway.codota.com"
  type PathAttemptResult = Try[Option[String]]
  type RetriesLoop = Stream[PathAttemptResult]
}

case class ArtifactMetadata(path: String) 

package com.wixpress.build.codota

import java.net.SocketTimeoutException

import org.specs2.specification.BeforeAfterAll
import org.specs2.specification.core.Fragments


class CodotaThinClientIT extends CodotaThinClientContract with BeforeAfterAll {
  val codePack = "some_code_pack"
  val validToken = "someValidToken"
  val artifactName = "some.group.artifact-name"
  val validArtifactWithoutMetaData = "some.group.artifact-without-metadata"
  val matchingPath = "some/path/to/artifact"

  override def testData: CodotaThinClientTestData = {
    CodotaThinClientTestData(
      codotaUrl = codotaFakeServer.url,
      codePack = codePack,
      validToken = validToken,
      validArtifact = artifactName,
      matchingPath = matchingPath,
      validArtifactWithoutMetaData = validArtifactWithoutMetaData
    )
  }

  val artifactsToPaths = Map(artifactName -> Some(matchingPath), validArtifactWithoutMetaData -> None)
  val codotaFakeServer = new CodotaFakeServer(codePack, validToken, artifactsToPaths)

  override protected def stressTests: Fragments = {
    "retry in case of timeouts" in {
      codotaFakeServer.delayTheNextNCalls(n = 1)

      client.pathFor(artifactName) must beSome(testData.matchingPath)
    }

    "throw TimeoutException in case still getting timeout after given max retries" in {
      val fastToGiveUpClient = new CodotaThinClient(validToken, codePack, codotaFakeServer.url, maxRetries = 2)

      codotaFakeServer.delayTheNextNCalls(n = 3)
      fastToGiveUpClient.pathFor(artifactName) must throwA[SocketTimeoutException]
    }
  }

  override def beforeAll(): Unit = codotaFakeServer.start()

  override def afterAll(): Unit = codotaFakeServer.stop()
} 

package bad.robot.temperature.server

import java.net.{DatagramPacket, DatagramSocket, SocketTimeoutException}

import bad.robot.temperature.server.DiscoveryServer._
import bad.robot.temperature.{Error, Timeout, UnexpectedError}

import scala.concurrent.duration.Duration
import scalaz.\/

object Socket {

  implicit class SocketOps(socket: DatagramSocket) {
    def await(timeout: Duration = Duration(0, "seconds")): Error \/ DatagramPacket = {
      val packet = new DatagramPacket(new Array[Byte](BufferSize), BufferSize)
      socket.setSoTimeout(timeout.toMillis.toInt)
      \/.fromTryCatchNonFatal {
        socket.receive(packet)
        packet
      }.leftMap {
        case _: SocketTimeoutException => Timeout(s"socket timed out after $timeout")
        case e: Throwable              => UnexpectedError(e.getMessage)
      }
    }
  }

} 

package org.crowdcrawler.proxycrawler.checker

import java.io.IOException
import java.net.SocketTimeoutException

import com.typesafe.scalalogging.Logger
import org.apache.http.annotation.ThreadSafe
import org.apache.http.conn.ConnectTimeoutException
import org.crowdcrawler.proxycrawler.ProxyInfo
import org.slf4j.LoggerFactory

import scala.collection.parallel.ForkJoinTaskSupport
import scala.concurrent.forkjoin.ForkJoinPool


@ThreadSafe
object ProxyChecker {
  private val LOGGER = Logger(LoggerFactory.getLogger(ProxyChecker.getClass.getName))


  
  private def check(proxyInfo: ProxyInfo): ProxyInfo = {
    val start = System.currentTimeMillis
    try {
      LOGGER.info("Executing request via proxy " + proxyInfo)
      val (statusCode, bytes) = proxyInfo.schema match {
        case "HTTP" =>
          HttpProxyChecker.check(proxyInfo.host, proxyInfo.port)
        case "HTTPS" =>
          HttpsProxyChecker.check(proxyInfo.host, proxyInfo.port)
        case "SOCKS" | "SOCKS4" | "SOCKS5" =>
          SocksProxyChecker.check(proxyInfo.host, proxyInfo.port)
        case other => throw new IllegalArgumentException("Unsupported schema " + other)
      }
      val end = System.currentTimeMillis
      LOGGER.info("Time elapsed " + (end - start) + " milliseconds")

      if (statusCode != 200) {
        LOGGER.error("HTTP status code is " + statusCode)
        ProxyInfo(proxyInfo.host, proxyInfo.port, proxyInfo.schema, -1, proxyInfo.location, proxyInfo.from)
      } else {
        if (bytes > 0) {
          val speed = (bytes / ((end - start) / 1000.0)).toInt
          LOGGER.info("Speed is " + speed + " bytes/s")
          ProxyInfo(proxyInfo.host, proxyInfo.port, proxyInfo.schema, speed, proxyInfo.location, proxyInfo.from)
        } else {
          LOGGER.error("HTTP status code is 200 but the proxy failed to retrieve HTML source code")
          if (proxyInfo.speed >= 0) {
            ProxyInfo(proxyInfo.host, proxyInfo.port, proxyInfo.schema, -1, proxyInfo.location, proxyInfo.from)
          } else {
            ProxyInfo(proxyInfo.host, proxyInfo.port, proxyInfo.schema, proxyInfo.speed - 1,
              proxyInfo.location, proxyInfo.from)
          }
        }
      }
    } catch {
      case e: IOException =>
        val end = System.currentTimeMillis
        if (e.isInstanceOf[ConnectTimeoutException] || e.isInstanceOf[SocketTimeoutException]) {
          LOGGER.info(e.getClass.getName + " : " + e.getMessage)
          LOGGER.info("Time elapsed " + (end - start) + " milliseconds")
        } else {
          LOGGER.error(e.getClass.getName + " : " + e.getMessage)
          LOGGER.error("Time elapsed " + (end - start) + " milliseconds")
        }

        if (proxyInfo.speed >= 0) {
          ProxyInfo(proxyInfo.host, proxyInfo.port, proxyInfo.schema, -1, proxyInfo.location, proxyInfo.from)
        } else {
          ProxyInfo(proxyInfo.host, proxyInfo.port, proxyInfo.schema, proxyInfo.speed - 1,
            proxyInfo.location, proxyInfo.from)
        }
    }
  }

} 

package turbo.crawler.power

import java.io.IOException
import java.io.StringReader
import java.net.BindException
import java.net.SocketException
import java.net.SocketTimeoutException

import org.apache.commons.httpclient.ConnectTimeoutException
import org.apache.commons.httpclient.Header
import org.cyberneko.html.parsers.DOMParser
import org.w3c.dom.Document
import org.xml.sax.InputSource

import turbo.crawler.FetchRejectedException
import turbo.crawler.Fetchable
import turbo.crawler.IO
import turbo.crawler.Logable
import turbo.crawler.ResourceHasAlreadyBeenFetchedException
import turbo.crawler.StringAdapter
import turbo.crawler.io.HttpReturns

/**
 * Event driven fetcher
 * @author mclaren
 *
 */
class EventDrivenFetcher[T <: Fetchable](eventId: String) extends Logable with MessageDriven with IO with StringAdapter {
  def fetch(fetchUrl: String  ,
    contentFilter: String => String  ,
    parseDocument: Document => List[T])(hasRejected: Document => Boolean)  = {
    val _retry = (msg: String) => {
      logger.info("Retry " + msg)
      Thread.sleep(3000)
      this.fetch(fetchUrl, contentFilter, parseDocument)(hasRejected)(howToContinue)(referer)
    }
    var httpReturns: HttpReturns = null
    try {
      val dom = new DOMParser
      httpReturns = this.fromUrl(fetchUrl, Array[Header](new Header("Referer", referer(fetchUrl))))
      dom.parse(new InputSource(new StringReader(contentFilter(httpReturns.body))))
      var document = dom.getDocument

      //检查是否被屏蔽
      if (hasRejected(document)) throw new FetchRejectedException(fetchUrl)

      parseDocument(document).foreach(x => fireEvent(new Evt(eventId + "_COMPLETION", x)))

    } catch {
      case e: SocketTimeoutException => _retry(e.getMessage)
      case e: SocketException => _retry(e.getMessage)
      case e: ConnectTimeoutException => _retry(e.getMessage)
      case e: IOException => {
        logger.info("Oh网络错误with代理：" + httpReturns.proxy.ip + ":" + httpReturns.proxy.port)
        howToContinue(fetchUrl, httpReturns.proxy)
        //10秒之内只允许出现一次重拨
        _retry(e.getMessage)
      }
      case e: BindException => _retry(e.getMessage)
      case e: FetchRejectedException => {
        logger.info("Oh 惨遭屏蔽~")
        howToContinue(e.getFetchUrl, httpReturns.proxy)
        //10秒之内只允许出现一次重拨
        _retry(e.getMessage)
      }
      case e: ResourceHasAlreadyBeenFetchedException =>
      case e: Exception => {
        logger.error("Unknown exception has been occurred", e)
      }
    }
  }
} 

package turbo.crawler.power

import java.io.IOException
import java.io.StringReader
import java.net.SocketException
import java.net.SocketTimeoutException

import org.apache.commons.httpclient.ConnectTimeoutException
import org.apache.commons.httpclient.Header
import org.cyberneko.html.parsers.DOMParser
import org.w3c.dom.Document
import org.xml.sax.InputSource

import turbo.crawler.FetchRejectedException
import turbo.crawler.Logable
import turbo.crawler.ResourceHasAlreadyBeenFetchedException
import turbo.crawler.io.HttpReturns
import turbo.crawler.io.InternetIO

/**
 * 分页支持
 * @author mclaren
 *
 */
object pages extends Logable with InternetIO {
  def apply(fetchUrl: String, contentFilter: String => String, checkBoundary: Document => Int, urlFactory: (String, Int) => String)(hasRejected: Document => Boolean)(howToContinue: (String, turbo.crawler.io.Proxy) => Unit): List[String] = {
    var value = new ValueRef[Int](0)
    resetBoundary(fetchUrl, value, contentFilter, checkBoundary, urlFactory)(hasRejected)(howToContinue)
    var rts = List[String]()
    value.get
    for (i <- 1 to value.get) {
      rts = rts.+:(urlFactory(fetchUrl, i))
    }
    rts
  }

  private def resetBoundary(fetchUrl: String, lastPage: ValueRef[Int], contentFilter: String => String = x => x, checkBoundary: Document => Int, urlFactory: (String, Int) => String)(hasRejected: Document => Boolean  ): Unit = {
    val _retry = (() => {
      Thread.sleep(3000)
      resetBoundary(fetchUrl, lastPage, contentFilter, checkBoundary, urlFactory)(hasRejected)(howToContinue)
    })

    var httpReturns: HttpReturns = null
    try {
      var domp = new DOMParser
      httpReturns = this.fromUrl(fetchUrl, Array[Header]())
      domp.parse(new InputSource(new StringReader(contentFilter(httpReturns.body))))
      var document = domp.getDocument

      if (hasRejected(document)) throw new FetchRejectedException(fetchUrl, httpReturns.proxy)

      lastPage.set(checkBoundary(document))
    } catch {
      case e: SocketTimeoutException => _retry()
      case e: SocketException => _retry()
      case e: ConnectTimeoutException => _retry()
      case e: IOException => _retry()
      case e: FetchRejectedException => {
        logger.info("Oh 惨遭屏蔽~")
        howToContinue(e.getFetchUrl, httpReturns.proxy)
        _retry()
      }
      case e: ResourceHasAlreadyBeenFetchedException =>
      case e: Exception => {
        logger.error("Unknown exception has been occurred", e)
      }
    }
  }
}
class ValueRef[M](v: M) {
  var value = v
  def set(vv: M) = this.value = vv
  def get = value
} 

package com.wavesplatform.utils

import java.net.{InetAddress, SocketTimeoutException}

import monix.eval.Task
import monix.execution.ExecutionModel
import monix.execution.schedulers.SchedulerService
import org.apache.commons.net.ntp.NTPUDPClient

import scala.concurrent.duration.DurationInt

trait Time {
  def correctedTime(): Long

  def getTimestamp(): Long
}

class NTP(ntpServer: String) extends Time with ScorexLogging with AutoCloseable {

  log.info("Initializing time")

  private val offsetPanicThreshold = 1000000L
  private val ExpirationTimeout    = 60.seconds
  private val RetryDelay           = 10.seconds
  private val ResponseTimeout      = 10.seconds

  private implicit val scheduler: SchedulerService =
    Schedulers.singleThread(name = "time-impl", reporter = log.error("Error in NTP", _), ExecutionModel.AlwaysAsyncExecution)

  private val client = new NTPUDPClient()
  client.setDefaultTimeout(ResponseTimeout.toMillis.toInt)

  @volatile private var offset = 0L
  private val updateTask: Task[Unit] = {
    def newOffsetTask: Task[Option[(InetAddress, java.lang.Long)]] = Task {
      try {
        client.open()
        val info = client.getTime(InetAddress.getByName(ntpServer))
        info.computeDetails()
        Option(info.getOffset).map { offset =>
          val r = if (Math.abs(offset) > offsetPanicThreshold) throw new Exception("Offset is suspiciously large") else offset
          (info.getAddress, r)
        }
      } catch {
        case _: SocketTimeoutException =>
          None
        case t: Throwable =>
          log.warn("Problems with NTP: ", t)
          None
      } finally {
        client.close()
      }
    }

    newOffsetTask.flatMap {
      case None if !scheduler.isShutdown => updateTask.delayExecution(RetryDelay)
      case Some((server, newOffset)) if !scheduler.isShutdown =>
        log.trace(s"Adjusting time with $newOffset milliseconds, source: ${server.getHostAddress}.")
        offset = newOffset
        updateTask.delayExecution(ExpirationTimeout)
      case _ => Task.unit
    }
  }

  def correctedTime(): Long = System.currentTimeMillis() + offset

  private var txTime: Long = 0

  def getTimestamp(): Long = {
    txTime = Math.max(correctedTime(), txTime + 1)
    txTime
  }

  private val taskHandle = updateTask.runAsyncLogErr

  override def close(): Unit = {
    log.info("Shutting down Time")
    taskHandle.cancel()
    scheduler.shutdown()
  }
}