scala.collection.SortedSet Scala Examples

// Copyright 2017, Yahoo Holdings Inc.
// Licensed under the terms of the Apache License 2.0. Please see LICENSE file in project root for terms.
package com.yahoo.maha.core

import com.yahoo.maha.core.fact.CostMultiplier
import com.yahoo.maha.core.request.ReportingRequest
import grizzled.slf4j.Logging

import scala.collection.SortedSet


trait FactCostEstimator extends Logging {
  def isGrainKey(grainKey: String): Boolean
  def grainPrefix(schemaRequiredEntity: String, entity:String):  String = if(schemaRequiredEntity == entity) {
    schemaRequiredEntity
  } else {
    s"$schemaRequiredEntity-$entity"
  }
  def allPrefix(entity: String): String = s"*-$entity"
  def getSchemaBasedGrainRows(grainKey: String, request:ReportingRequest, filters: scala.collection.mutable.Map[String, Filter], defaultRowCount: Long): Option[Long]
  def getAllBasedGrainRows(grainKey: String, request:ReportingRequest, filters: scala.collection.mutable.Map[String, Filter]): Option[Long]
  def getDefaultRows(defaultRowCount: Long, request:ReportingRequest, filters: scala.collection.mutable.Map[String, Filter]) = {
    defaultRowCount * (request.numDays + 1)
  }
  def getRowsEstimate(schemaRequiredEntitySet:Set[(String, Filter)]
                      , dimensionsCandidates: SortedSet[DimensionCandidate]
                      , factDimList: List[String]
                      , request: ReportingRequest
                      , filters: scala.collection.mutable.Map[String, Filter]
                      , defaultRowCount:Long): RowsEstimate = {
    val schemaBasedGrainKeys = schemaRequiredEntitySet.map {
      case (requiredEntity, filter) =>
        dimensionsCandidates.headOption.map(entity => grainPrefix(requiredEntity, entity.dim.name)).getOrElse(requiredEntity)
    }
    val schemaBasedResult = schemaBasedGrainKeys.filter(isGrainKey).flatMap {
      grainKey =>
        val grainRows = getSchemaBasedGrainRows(grainKey, request, filters, defaultRowCount)
        if(request.isDebugEnabled) {
          info(s"schemaBasedResult grainKey=$grainKey grainRows=$grainRows")
        }
        grainRows
    }
    val (isGrainOptimized, rows) = if(schemaBasedResult.nonEmpty) {
      (true, schemaBasedResult.min)
    } else (false, getDefaultRows(defaultRowCount, request, filters))
    //all based grain key, take highest grain for fact table
    val  allBasedResult = factDimList.headOption.map(allPrefix).filter(isGrainKey).flatMap{
      grainKey =>
        val grainRows = getAllBasedGrainRows(grainKey, request, filters)
        if(request.isDebugEnabled) {
          info(s"allBasedResult grainKey=$grainKey grainRows=$grainRows")
        }
        grainRows
    }
    val (isScanOptimized, scanRows) = if(allBasedResult.nonEmpty) {
      (true, allBasedResult.max)
    } else (false, Long.MaxValue)

    val estimate = RowsEstimate(rows, isGrainOptimized, scanRows, isScanOptimized)
    if(request.isDebugEnabled) {
      info(s"$estimate")
    }
    estimate
  }
  def getCostEstimate(rowsEstimate: RowsEstimate, rowCostMultiplierOption: Option[CostMultiplier]) : Long = {
    val cost = for {
      rowCostMultiplier <- rowCostMultiplierOption
      costMultiplier <- rowCostMultiplier.rows.find(rowsEstimate.rows)
    } yield (costMultiplier * rowsEstimate.rows).longValue()
    cost.getOrElse(Long.MaxValue)
  }
}

trait DimCostEstimator {
  def getCardinalityEstimate(grainKey: String, request: ReportingRequest,filters: scala.collection.mutable.Map[String, Filter]): Option[Long]
}

class DefaultDimEstimator extends DimCostEstimator {
  def getCardinalityEstimate(grainKey: String, request: ReportingRequest,filters: scala.collection.mutable.Map[String, Filter]): Option[Long] = None
}

class DefaultFactEstimator(grainKeySet: Set[String] = Set.empty
                           , defaultRowCount: Long = 100
                          ) extends FactCostEstimator {
  def isGrainKey(grainKey: String): Boolean = grainKeySet(grainKey)
  def getGrainRows(grainKey: String, request:ReportingRequest, filters: scala.collection.mutable.Map[String, Filter], defaultRowCount: Long): Option[Long] = {
    Option((defaultRowCount * (request.numDays + 1)).longValue())
  }
  def getSchemaBasedGrainRows(grainKey: String, request:ReportingRequest, filters: scala.collection.mutable.Map[String, Filter], defaultRowCount: Long): Option[Long] = {
    getGrainRows(grainKey, request, filters, defaultRowCount)
  }
  def getAllBasedGrainRows(grainKey: String, request:ReportingRequest, filters: scala.collection.mutable.Map[String, Filter]): Option[Long] = {
    getGrainRows(grainKey, request, filters, defaultRowCount)
  }
} 

package org.perftester.results

import java.nio.file.{Files, Path}

import org.perftester.results.rows._

import scala.collection.SortedSet
import collection.JavaConverters._

object ResultReader {
  def readResults(id: String, file: Path, iterations: Int): RunResult = {
    val lines    = Files.readAllLines(file).asScala
    val asValues = lines.map(_.split(',').map(_.trim).toList).toList
    val info = asValues.collectFirst {
      case values if values(0) == "info" => InfoRow.parse(values)
    }.get
    val dataRows = asValues.flatMap(DataRow(_, info.version))

    val gcInfo = dataRows.collect {
      case gc: GCDataRow => gc
    }

    def gcEvents(start: Long, end: Long) = gcInfo.collect {
      case gc @ GCDataRow(startNs, endNs, _, _, _, _, _, _)
          if (startNs > start && startNs < end) || (endNs > start && endNs < end) =>
        gc
    }

    val background = dataRows
      .collect {
        case row: BackgroundPhaseRow => row
      }
      .groupBy { row: BackgroundPhaseRow =>
        (row.runId, row.phaseName)
      }
      .withDefaultValue(Nil)

    val rows = dataRows
      .collect {
        case row: MainPhaseRow =>
          PhaseResults(row,
                       background((row.runId, row.phaseName)),
                       gcEvents(row.startNs, row.endNs))
      }
      .sortBy(r => (r.iterationId, r.phaseId))
    val allPhases     = rows.groupBy(_.phaseName).keySet

    RunResult(id, rows, allPhases)
  }
} 

package org.perftester.results

import ammonite.ops.{Path, read}
import org.perftester.TestConfig
import org.perftester.results.rows._

import scala.collection.SortedSet

object ResultReader {
  def readResults(testConfig: TestConfig, file: Path, iterations: Int): RunResult = {
    val lines    = read.lines ! file
    val asValues = lines.map(_.split(',').map(_.trim).toList).toList
    val info = asValues.collectFirst {
      case values if values(0) == "info" => InfoRow.parse(values)
    }.get
    val dataRows = asValues.flatMap(DataRow(_, info.version))

    val gcInfo = dataRows.collect {
      case gc: GCDataRow => gc
    }

    def gcEvents(start: Long, end: Long) = gcInfo.collect {
      case gc @ GCDataRow(startNs, endNs, _, _, _, _, _, _)
          if (startNs > start && startNs < end) || (endNs > start && endNs < end) =>
        gc
    }

    val background = dataRows
      .collect {
        case row: BackgroundPhaseRow => row
      }
      .groupBy { row: BackgroundPhaseRow =>
        (row.runId, row.phaseName)
      }
      .withDefaultValue(Nil)

    val rows = dataRows
      .collect {
        case row: MainPhaseRow =>
          PhaseResults(row,
                       background((row.runId, row.phaseName)),
                       gcEvents(row.startNs, row.endNs))
      }
      .sortBy(r => (r.iterationId, r.phaseId))
    val allIterations = (1 to iterations).to[SortedSet]
    val allPhases     = rows.groupBy(_.phaseName).keySet
    RunResult(testConfig, info, rows, allIterations, allPhases)
  }
} 

package org.perftester.results

import org.perftester.TestConfig
import org.perftester.results.rows.InfoRow

import scala.collection.SortedSet

case class RunResult(testConfig: TestConfig,
                     info: InfoRow,
                     rawData: Seq[PhaseResults],
                     iterations: SortedSet[Int],
                     phases: Set[String]) {

  rawData foreach { r =>
    require(phases(r.phaseName), r.phaseName)
  }
}
case class RunDetails(cycleNumber: Int, testId: Int, runResult: RunResult)
    extends Ordered[RunDetails] {
  override def compare(that: RunDetails): Int = {
    val major = this.testId compareTo that.testId
    if (major != 0) major else this.cycleNumber compareTo that.cycleNumber
  }
} 

import scala.collection.SortedSet
import scala.collection.mutable.HashMap
def mapStrIndex(str:String)={
  var indexMap = new HashMap[Char,SortedSet[Int]]()
  var i = 0
  str.toCharArray.foreach {
    c =>
      indexMap.get(c) match {
        case Some(result) => indexMap(c) = result + i
        case None => indexMap += (c -> SortedSet {
          i
        })
      }
      i += 1
  }
  indexMap

}
println(mapStrIndex("Mississippi")) 

package org.mmadt.storage.obj.value.strm.util

import org.mmadt.language.obj.value.Value
import org.mmadt.language.obj.value.strm.Strm
import org.mmadt.language.obj.{Obj, _}
import org.mmadt.storage.StorageFactory._

import scala.collection.SortedSet
import scala.collection.immutable.ListSet


class MultiSet[A <: Obj](val baseSet: ListSet[A] = ListSet.empty[A]) extends Seq[A] {
  def get(a: A): Option[A] = baseSet.find(b => a.asInstanceOf[Value[_]].g.equals(b.asInstanceOf[Value[_]].g))
  def put(a: A): MultiSet[A] = {
    val oldObj: Option[A] = this.get(a)
    new MultiSet[A](oldObj.map(x => baseSet - x).getOrElse(baseSet) + oldObj.map(x => x.hardQ(plusQ(a, x))).getOrElse(a))
  }
  def objSize: Long = baseSet.size
  def qSize: IntQ = baseSet.foldRight(qZero)((a, b) => plusQ(a.q, b))
  override def length: scala.Int = objSize.toInt
  override def apply(idx: scala.Int): A = this.baseSet.toSeq.apply(idx)
  override def iterator: Iterator[A] = this.baseSet.iterator
  override def toSeq: Seq[A] = baseSet.toSeq

  override def hashCode: scala.Int = baseSet.hashCode()
  override def equals(other: Any): Boolean = other match {
    case multiSet: MultiSet[_] => multiSet.baseSet == this.baseSet
    case _ => false
  }
}

object MultiSet {
  def put[A <: Obj](objs: A*): MultiSet[A] = objs.foldLeft(new MultiSet[A])((a, b) => a.put(b))
  def test(a: Obj, b: Obj): Boolean = MultiSet(a.toStrm.values) == MultiSet(b.toStrm.values)
  def apply[A <: Obj](objs: Seq[A]): MultiSet[A] = objs.flatMap {
    case astrm: Strm[A] => astrm.values
    case x => List(x)
  }.foldLeft(new MultiSet[A])((a, b) => a.put(b))
} 

package com.gilazaria.subsearch.output

import com.gilazaria.subsearch.model.Record

import scala.collection.SortedSet
import scala.concurrent.Future

trait Logger {
  // Controller
  def logHeader(header: String)
  def logConfig(threads: Int, wordlistSize: Int, resolverslistSize: Int)
  def logTarget(hostname: String)

  // Authoritative Scanner
  def logAuthoritativeScanStarted()
  def logAuthoritativeNameServer(nameServer: String)
  def logAuthoritativeScanCompleted()
  def logAddingAuthNameServersToResolvers(totalResolversSize: Int)

  // Zone Transfer Scanner
  def logStartedZoneTransfer()
  def logNameServersNotVulnerableToZoneTransfer()
  def logNameServerVulnerableToZoneTransfer(nameServer: String)
  def logZoneTransferCompleted()

  // Additional Scanners
  def logAdditionalScansStarted()
  def logAdditionalScannerError(scannerName: String, msg: String)
  def logAdditionalScannerFoundSubdomains(scannerName: String, subdomains: Set[String])
  def logAdditionalScansCompleted()

  // Subdomain Bruteforce Scanner
  def logStartedSubdomainSearch()
  def logTaskCompleted()
  def logTaskFailed()
  def logPausingThreads()
  def logPauseOptions()
  def logInvalidPauseOption()
  def logNotEnoughResolvers()
  def logTimedOutScan(subdomain: String, resolver: String, duration: String)
  def logBlacklistedResolver(resolver: String)
  def logScanCancelled()
  def logScanForceCancelled()
  def logLastRequest(subdomain: String, numberOfRequestsSoFar: Int, totalNumberOfSubdomains: Int)

  // Records
  def logRecords(records: SortedSet[Record])
  def logRecordsDuringScan(records: SortedSet[Record])

  // Utility
  def completedLoggingFuture: Future[Unit]
} 

package com.gilazaria.subsearch.output

import com.gilazaria.subsearch.model.Record
import com.gilazaria.subsearch.utils.{File, TimeUtils}

import scala.collection.SortedSet
import scala.concurrent.Future
import scala.concurrent.ExecutionContext.Implicits.global

class CSVOutput(private val file: Option[File]) extends Output {
  override def print(string: String): Unit = {}

  

  private var saveToFileFuture: Future[Unit] = Future(Unit)
  override def printRecords(records: SortedSet[Record]) = {
    if (file.isDefined) {
      saveToFileFuture = saveToFileFuture.map {
        _ =>
          val lines = records.map(record => s"${TimeUtils.timestampNow},${record.name},${record.recordType},${record.data}")
          file.get.write(lines.mkString("\n") + "\n")
      }
    }
  }

  override def writingToFileFuture: Future[Unit] = {
    saveToFileFuture
  }
}

object CSVOutput {
  def create(fileOption: Option[File]): Option[CSVOutput] =
    if (fileOption.isDefined) Some(new CSVOutput(fileOption))
    else None
} 

package com.gilazaria.subsearch.output

import com.gilazaria.subsearch.model.Record
import com.gilazaria.subsearch.utils.TimeUtils

import scala.collection.SortedSet
import scala.concurrent.Future
import scala.concurrent.ExecutionContext.Implicits.global

trait Output {
  def print(string: String)
  def println(): Unit = println("")
  def println(string: String): Unit = print(s"$string\n")

  def printSuccess(string: String) = printSuccessWithoutTime(prependTime(string))
  def printStatus(string: String) = printStatusWithoutTime(prependTime(string))
  def printInfo(string: String) = printInfoWithoutTime(prependTime(string))
  def printError(string: String) = printErrorWithoutTime(prependTime(string))

  def printSuccessWithoutTime(string: String) = println(string)
  def printStatusWithoutTime(string: String) = println(string)
  def printInfoWithoutTime(string: String) = println(string)
  def printErrorWithoutTime(string: String) = println(string)

  def printSuccessDuringScan(string: String) = printSuccess(string)
  def printStatusDuringScan(string: String) = printStatus(string)
  def printInfoDuringScan(string: String) = printInfo(string)
  def printErrorDuringScan(string: String) = printError(string)

  // Utility

  final def prependTime(string: String): String =
    s"${TimeUtils.currentTimePretty} $string"

  def writingToFileFuture: Future[Unit] = Future(Unit)

  // Application Specific

  def printHeader(header: String) = {
    println(header)
    println()
  }

  def printConfig(config: List[(String, String)], separator: String) = {
    val string: String =
      config
        .map((tuple: (String, String)) => tuple._1 + tuple._2)
        .mkString(separator)

    println(string)
    println()
  }

  def printTarget(text: String, hostname: String) = {
    println(s"$text$hostname")
    println()
  }

  def printTaskCompleted(text: String) = {
    println()
    printStatusWithoutTime(text)
    println()
  }

  def printTaskFailed(text: String) = {
    println()
    printErrorWithoutTime(text)
    println()
  }

  def printPausingThreads(text: String) = {}

  def printPauseOptions(text: String) = {}

  def printInvalidPauseOptions(text: String) = {}

  def printLastRequest(text: String) = {}

  def printLastRequest() = {}

  def printRecords(records: SortedSet[Record])

  def printRecordsDuringScan(records: SortedSet[Record]) = printRecords(records)
} 

package com.gilazaria.subsearch.output

import com.gilazaria.subsearch.model.{Record, RecordType}
import com.gilazaria.subsearch.utils.File

import scala.collection.SortedSet
import scala.concurrent.Future
import scala.concurrent.ExecutionContext.Implicits.global

class StandardOutput(private val file: Option[File], private val verbose: Boolean) extends Output {
  private var saveToFileFuture: Future[Unit] = Future(Unit)

  override def print(string: String): Unit =  {
    if (file.isDefined) {
      saveToFileFuture = saveToFileFuture.map {
        _ => file.get.write(string)
      }
    }
  }

  override def writingToFileFuture: Future[Unit] = {
    saveToFileFuture
  }

  override def printRecords(records: SortedSet[Record]) = {
    if (verbose) printRecordsVerbose(records)
    else printRecordsNormal(records)
  }

  protected def printRecordsVerbose(records: SortedSet[Record]) = {
    val lines: List[String] =
      records
        .map(_.name)
        .toList
        .flatMap {
          subdomain =>
            val subdomainRecords: SortedSet[Record] = records.filter(_.name == subdomain)
            val recordTypes: SortedSet[RecordType] = subdomainRecords.map(_.recordType)

            recordTypes.flatMap {
              recordType =>
                subdomainRecords.filter(_.recordType == recordType).map {
                  case Record(_, _, data) =>
                    val msg = formatRecordTypeAndSubdomainForPrinting(recordType, subdomain)

                    if (recordType.isOneOf("A", "AAAA", "CNAME", "NS", "SRV"))
                      s"$msg  ->  $data"
                    else if (recordType.stringValue == "MX")
                      s"$msg  @@  $data"
                    else
                      s"$msg  --  $data"
                }
            }
        }

    if (lines.nonEmpty)
      println(lines.mkString("\n"))
  }

  protected def formatRecordTypeAndSubdomainForPrinting(recordType: RecordType, subdomain: String): String =
    prependTime(f"${recordType.toString}%-7s:  $subdomain")

  protected def printRecordsNormal(records: SortedSet[Record]) = {
    val lines: List[String] =
      records
        .map(_.name)
        .toList
        .map(subdomain => (subdomain, records.filter(_.name == subdomain).map(_.recordType)))
        .map((data: (String, SortedSet[RecordType])) => s"${data._2.mkString(", ")}:  ${data._1}")

    if (lines.nonEmpty)
      printSuccess(lines.mkString("\n"))
  }
}

object StandardOutput {
  def create(fileOption: Option[File], verbose: Boolean): Option[StandardOutput] =
    if (fileOption.isDefined) Some(new StandardOutput(fileOption, verbose))
    else None
} 

package com.gilazaria.subsearch.connection

import com.gilazaria.subsearch.model.{Record, RecordType}
import com.gilazaria.subsearch.utils.IterableUtils._
import org.xbill.DNS._

import scala.annotation.tailrec
import scala.util.Try
import scala.collection.JavaConverters._
import scala.collection.SortedSet

class DNSLookupImpl extends DNSLookup {
  import DNSLookupImpl.{HostNotFoundException, ServerFailureException}

  override def performQueryOfType(hostname: String, resolver: String, recordType: RecordType): Try[SortedSet[Record]] =
    if (recordType == RecordType.AXFR)
      Try(performZoneTransfer(hostname, resolver))
    else
      Try(performLookup(hostname, resolver, recordType))

  private[connection] def performZoneTransfer(hostname: String, resolver: String): SortedSet[Record] = {
    val transfer: ZoneTransferInFactory = ZoneTransferInFactoryImpl.newAXFR(new Name(hostname), resolver, null)
    val xbillRecords: Set[org.xbill.DNS.Record] = xbillRecordsFromTransfer(transfer).getOrElse(Set.empty)

    recordsFromXbillRecords(xbillRecords)
      .filter(dnsRecord => !dnsRecord.name.startsWith(hostname))
  }

  private[connection] def xbillRecordsFromTransfer(transfer: ZoneTransferInFactory): Try[Set[org.xbill.DNS.Record]] = {
    Try {
      Option(transfer.run())
        .map(_.asScala.toSet)
        .getOrElse(Set.empty)
        .asInstanceOf[Set[org.xbill.DNS.Record]]
    }
  }

  private[connection] def performLookup(hostname: String, resolver: String, recordType: RecordType): SortedSet[Record] = {
    val lookup: LookupFactory = new LookupFactoryImpl(hostname, recordType.intValue)
    lookup.setResolver(new SimpleResolver(resolver))
    query(hostname, resolver, lookup)
  }

  @tailrec
  private[connection] final def query(hostname: String, resolver: String, lookup: LookupFactory, attempt: Int = 1): SortedSet[Record] = {
    lookup.run()

    lookup.getResult match {
      case Lookup.SUCCESSFUL =>
        val xbillRecords =
          Option(lookup.getAnswers)
            .map(_.toSet)
            .getOrElse(Set.empty)

        recordsFromXbillRecords(xbillRecords)

      case Lookup.HOST_NOT_FOUND =>
        throw new HostNotFoundException(s"The hostname $hostname was not found.")

      case Lookup.UNRECOVERABLE =>
        throw new ServerFailureException(s"There was a data or server error with the resolver $resolver.")

      case Lookup.TYPE_NOT_FOUND =>
        SortedSet.empty

      case Lookup.TRY_AGAIN =>
        if (attempt >= 3) SortedSet.empty
        else query(hostname, resolver, lookup, attempt + 1)
    }
  }

  private[connection] def recordsFromXbillRecords(xbillRecords: Set[org.xbill.DNS.Record]): SortedSet[Record] =
    xbillRecords
      .map(xbillRecord => Record(xbillRecord.getName.toString, RecordType.fromInt(xbillRecord.getType), xbillRecord.rdataToString))
      .toSortedSet
}

object DNSLookupImpl {
  def create(): DNSLookup =
    new DNSLookupImpl()

  case class HostNotFoundException(msg: String) extends Exception(msg)
  case class ServerFailureException(msg: String) extends Exception(msg)
} 

package com.gilazaria.subsearch.core.subdomainscanner

import akka.actor.ActorRef
import com.gilazaria.subsearch.model.Record

import scala.collection.SortedSet
import scala.util.Try

trait Message

object DispatcherMessage extends Message {
  case object ResumeScanning
  case object PauseScanning
  case object NotifyOnCompletion
  case object AvailableForScan

  case class PriorityScanSubdomain(subdomain: String)
  case class CompletedScan(subdomain: String, resolver: String)
  case class FailedScan(subdomain: String, resolver: String)
}

object ScannerMessage extends Message {
  case object ScanAvailable

  case class Scan(subdomain: String, resolver: String, attempt: Int)
  case class ScanComplete(records: Try[SortedSet[Record]], subdomain: String, resolver: String)
  case class ScanFailed(cause: Throwable, subdomain: String, resolver: String, attempt: Int)
}

object ListenerMessage extends Message {
  case object PausingScanning
  case object ResumedScanning
  case object NotEnoughResolvers

  case class FoundSubdomain(subdomain: String, records: SortedSet[Record])
  case class LastScan(subdomain: String, requestsSoFar: Int, totalRequests: Int)
  case class TaskCompleted(master: Option[ActorRef])
  case class TaskFailed(master: Option[ActorRef])
  case class ScanTimeout(subdomain: String, resolver: String, attempt: Int)
  case class BlacklistedResolver(resolver: String)
} 

package com.gilazaria.subsearch.core

import com.gilazaria.subsearch.output.Logger
import com.gilazaria.subsearch.connection.{DNSLookupImpl, DNSLookup}
import com.gilazaria.subsearch.model.{Record, RecordType}

import scala.concurrent.{ExecutionContext, Future}
import scala.collection.SortedSet

class AuthoritativeScanner private[core] (private val logger: Logger, private val lookup: DNSLookup)(implicit ec: ExecutionContext) {
  def performLookupOnHostname(hostname: String, resolver: String): Future[Set[String]] = {
    logger.logAuthoritativeScanStarted()

    dataFromQuery(hostname, resolver, RecordType.NS)
      .flatMap(nameservers => ipsForNameServers(nameservers, resolver))
      .map(printAuthoritativeNameServers)
  }

  private[core] def ipsForNameServers(nameServers: Set[String], resolver: String): Future[Set[String]] = {
    Future
      .sequence(nameServers.map(ns => dataFromQuery(ns, resolver, RecordType.A)))
      .map(_.flatten)
  }

  private[core] def dataFromQuery(hostname: String, resolver: String, recordType: RecordType): Future[Set[String]] = {
    Future {
      lookup
        .performQueryOfType(hostname, resolver, recordType)
        .getOrElse(SortedSet.empty[Record])
        .map(_.data)
        .toSet
    }
  }

  private[core] def printAuthoritativeNameServers(nameServers: Set[String]) = {
    nameServers.foreach(logger.logAuthoritativeNameServer)
    logger.logAuthoritativeScanCompleted()
    nameServers
  }
}

object AuthoritativeScanner {
  def create(logger: Logger)(implicit ec: ExecutionContext): AuthoritativeScanner =
    new AuthoritativeScanner(logger, DNSLookupImpl.create())
} 

package com.gilazaria.subsearch.core

import com.gilazaria.subsearch.connection.{DNSLookupImpl, DNSLookup}
import com.gilazaria.subsearch.model.{Record, RecordType}
import com.gilazaria.subsearch.output.Logger

import scala.concurrent.{ExecutionContext, Future}
import scala.util.Try
import scala.collection.SortedSet

class ZoneTransferScanner private[core] (private val logger: Logger, private val lookup: DNSLookup)(implicit ec: ExecutionContext) {
  def performLookup(hostname: String, resolvers: Set[String]): Future[Set[String]] = {
    logger.logStartedZoneTransfer()

    zoneTransfersForHostnameAndResolvers(hostname, resolvers)
      .map(records => recordsEndingWithHostname(hostname, records))
      .map(printFoundRecords)
      .map(namesFromRecords)
  }

  private[core] def zoneTransfersForHostnameAndResolvers(hostname: String, resolvers: Set[String]): Future[SortedSet[Record]] =
    Future
      .sequence(resolvers.map(resolver => zoneTransferForHostnameAndResolver(hostname, resolver)))
      .map(flattenRecords)

  private[core] def zoneTransferForHostnameAndResolver(hostname: String, resolver: String): Future[SortedSet[Record]] = {
    val lookupFut: Future[SortedSet[Record]] =
      Future {
        lookup
          .performQueryOfType(hostname, resolver, RecordType.AXFR)
          .getOrElse(SortedSet.empty[Record])
      }

    lookupFut
      .andThen {
        case records: Try[SortedSet[Record]] =>
          if (records.getOrElse(SortedSet.empty[Record]).nonEmpty)
            logger.logNameServerVulnerableToZoneTransfer(resolver)
      }
  }

  private[core] def flattenRecords(set: Set[SortedSet[Record]]): SortedSet[Record] =
    if (set.isEmpty) SortedSet.empty[Record]
    else set.reduce(_ ++ _)

  private[core] def recordsEndingWithHostname(hostname: String, records: SortedSet[Record]): SortedSet[Record] =
    records.filter(_.name.endsWith(hostname))

  private[core] def printFoundRecords(records: SortedSet[Record]): SortedSet[Record] = {
    if (records.isEmpty)
      logger.logNameServersNotVulnerableToZoneTransfer()
    else
      logger.logRecords(records)

    logger.logZoneTransferCompleted()

    records
  }

  private[core] def namesFromRecords(records: SortedSet[Record]): Set[String] =
    records.map(_.name).toSet

}

object ZoneTransferScanner {
  def create(logger: Logger)(implicit ec: ExecutionContext): ZoneTransferScanner =
    new ZoneTransferScanner(logger, DNSLookupImpl.create())
} 

package com.spotify.featran.transformers

import com.spotify.featran.{FeatureBuilder, FlatReader, FlatWriter}
import com.twitter.algebird.HLL

import scala.collection.SortedSet


  def fromSettings(setting: Settings): Transformer[Seq[String], HLL, Int] = {
    val hashBucketSize = setting.params("hashBucketSize").toInt
    val sizeScalingFactor = setting.params("sizeScalingFactor").toDouble
    HashNHotEncoder(setting.name, hashBucketSize, sizeScalingFactor)
  }
}

private[featran] class HashNHotEncoder(name: String, hashBucketSize: Int, sizeScalingFactor: Double)
    extends BaseHashHotEncoder[Seq[String]](name, hashBucketSize, sizeScalingFactor) {
  override def prepare(a: Seq[String]): HLL =
    a.map(hllMonoid.toHLL(_)).fold(hllMonoid.zero)(hllMonoid.plus)

  override def buildFeatures(a: Option[Seq[String]], c: Int, fb: FeatureBuilder[_]): Unit =
    a match {
      case Some(xs) =>
        var prev = -1
        SortedSet(xs.map(HashEncoder.bucket(_, c)): _*).foreach { curr =>
          val gap = curr - prev - 1
          if (gap > 0) fb.skip(gap)
          fb.add(name + '_' + curr, 1.0)
          prev = curr
        }
        val gap = c - prev - 1
        if (gap > 0) fb.skip(gap)
      case None => fb.skip(c)
    }

  override def flatRead[T: FlatReader]: T => Option[Any] = FlatReader[T].readStrings(name)

  override def flatWriter[T](implicit fw: FlatWriter[T]): Option[Seq[String]] => fw.IF =
    fw.writeStrings(name)
} 

package com.swissborg.lithium

package utils

import cats._
import com.swissborg.lithium.strategy._

import scala.collection.SortedSet


  lazy val isResolved: Boolean = {
    val nonEmptyPartitions = partitions.filter(_.nonEmpty)

    val allReachableNodes = nonEmptyPartitions.forall(_.forall {
      case _: ReachableNode => true
      case _                => false
    })

    lazy val noSplitBrain = if (nonEmptyPartitions.lengthCompare(1) > 0) {
      nonEmptyPartitions.tail
        .foldLeft((true, nonEmptyPartitions.head)) {
          case ((noSplitBrain, expectedPartition), partition) =>
            (noSplitBrain && expectedPartition.iterator.sameElements(partition.iterator), expectedPartition)
        }
        ._1
    } else {
      // 1 or less non-empty partitions
      true
    }

    allReachableNodes && noSplitBrain
  }
}

object PostResolution {
  val empty: PostResolution = PostResolution(List.empty)

  def one(nodes: SortedSet[Node]): PostResolution = PostResolution(List(nodes))

  def fromDecision(worldView: WorldView)(decision: Decision): PostResolution =
    decision match {
      // In all these cases the entire partition will down itself.
      case _: Decision.DownReachable                        => PostResolution.empty
      case Decision.DownThese(_: Decision.DownReachable, _) => PostResolution.empty
      case Decision.DownThese(_, _: Decision.DownReachable) => PostResolution.empty

      case _ =>
        val nodesAfterDowning = worldView.nodes.toSortedSet -- decision.nodesToDown
        PostResolution.one(nodesAfterDowning)
    }

  implicit val postResolutionMonoid: Monoid[PostResolution] = new Monoid[PostResolution] {
    override def empty: PostResolution = PostResolution.empty

    override def combine(x: PostResolution, y: PostResolution): PostResolution =
      PostResolution(x.partitions ++ y.partitions)
  }
}