com.amazonaws.services.s3.AmazonS3 Scala Examples

package org.hatdex.hat.api.service

import com.amazonaws.auth.{ AWSStaticCredentialsProvider, BasicAWSCredentials }
import com.amazonaws.services.s3.model.ObjectMetadata
import com.amazonaws.services.s3.{ AmazonS3, AmazonS3ClientBuilder }
import org.specs2.mock.Mockito

import scala.concurrent.duration._

case class FileManagerS3Mock() extends Mockito {
  val s3Configuration = AwsS3Configuration("hat-storage-test", "testAwsAccessKey", "testAwsSecret", "eu-west-1", 5.minutes)
  private val awsCreds: BasicAWSCredentials = new BasicAWSCredentials(s3Configuration.accessKeyId, s3Configuration.secretKey)
  val mockS3client: AmazonS3 = spy(AmazonS3ClientBuilder.standard()
    .withRegion("eu-west-1")
    .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
    .build())

  private val s3ObjectMetadata = new ObjectMetadata()
  s3ObjectMetadata.setContentLength(123456L)
  doReturn(s3ObjectMetadata).when(mockS3client).getObjectMetadata("hat-storage-test", "hat.hubofallthings.net/testFile")
  doNothing.when(mockS3client).deleteObject("hat-storage-test", "hat.hubofallthings.net/deleteFile")
} 

package org.marvin.artifact.manager

import java.io.File

import akka.Done
import akka.actor.{ActorSystem, Props}
import akka.testkit.{ImplicitSender, TestKit}
import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model.GetObjectRequest
import com.typesafe.config.ConfigFactory
import org.apache.hadoop.fs.Path
import org.marvin.artifact.manager.ArtifactSaver.{SaveToLocal, SaveToRemote}
import org.marvin.fixtures.MetadataMock
import org.marvin.model.EngineMetadata
import org.scalamock.scalatest.MockFactory
import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}


class ArtifactS3SaverTest extends TestKit(
  ActorSystem("ArtifactS3SaverTest", ConfigFactory.parseString("""akka.loggers = ["akka.testkit.TestEventListener"]""")))
  with ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll with MockFactory {

  override def afterAll {
    TestKit.shutdownActorSystem(system)
  }

  "s3 saver" should {
    "receive SaveToLocal message" in {
      val metadata = MetadataMock.simpleMockedMetadata()
      val _s3Client = mock[AmazonS3]
      val actor = system.actorOf(Props(new ArtifactS3SaverMock(metadata, _s3Client, true)))

      val protocol = "protocol"
      val artifactName = "model"

      (_s3Client.getObject(_ : GetObjectRequest, _ : File)).expects(*, *).once()

      actor ! SaveToLocal(artifactName, protocol)

      expectMsg(Done)
    }

    "receive SaveToRemote message" in {
      val metadata = MetadataMock.simpleMockedMetadata()
      val _s3Client = mock[AmazonS3]
      val actor = system.actorOf(Props(new ArtifactS3SaverMock(metadata, _s3Client, true)))

      val protocol = "protocol"
      val artifactName = "model"

      (_s3Client.putObject(_ : String, _: String, _ : File)).expects(metadata.s3BucketName, *, *).once()

      actor ! SaveToRemote(artifactName, protocol)

      expectMsg(Done)
    }
  }

    "call preStart method wth success" in {
      val metadata = MetadataMock.simpleMockedMetadata()
      try{
        system.actorOf(Props(new ArtifactS3Saver(metadata)))
        assert(true)
      }catch {
        case _: Throwable =>
          assert(false)
      }
    }

  class ArtifactS3SaverMock(metadata: EngineMetadata, _s3Client: AmazonS3, _isRemote: Boolean) extends ArtifactS3Saver(metadata) {
    def _preStart(): Unit = super.preStart()
    override def preStart(): Unit = {
      s3Client = _s3Client
    }

    override def validatePath(path: Path, isRemote: Boolean): Boolean = {
      if (_isRemote) true
      else false
    }
  }
} 

package org.marvin.artifact.manager

import java.io.File

import akka.Done
import akka.actor.{Actor, ActorLogging}
import com.amazonaws.services.s3.model.GetObjectRequest
import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
import org.apache.hadoop.fs.Path
import org.marvin.artifact.manager.ArtifactSaver.{SaveToLocal, SaveToRemote}
import org.marvin.model.EngineMetadata

class ArtifactS3Saver(metadata: EngineMetadata) extends Actor with ActorLogging {
  var s3Client: AmazonS3 = _

  override def preStart() = {
    log.info(s"${this.getClass().getCanonicalName} actor initialized...")

    //Create S3 Client with default credential informations(Environment Variable)
    s3Client = AmazonS3ClientBuilder.standard.withRegion(System.getenv("AWS_DEFAULT_REGION")).build

    log.info("Amazon S3 client initialized...")
  }

  def generatePaths(artifactName: String, protocol: String): Map[String, Path] = {
    var artifactsRemotePath: String = null
    if(metadata.artifactsRemotePath.startsWith("/")){
      artifactsRemotePath = metadata.artifactsRemotePath.substring(1)
    }
    Map(
      "localPath" -> new Path(s"${metadata.artifactsLocalPath}/${metadata.name}/$artifactName"),
      "remotePath" -> new Path(s"${artifactsRemotePath}/${metadata.name}/${metadata.version}/$artifactName/$protocol")
    )
  }

  def validatePath(path: Path, isRemote: Boolean): Boolean = {
    if (isRemote) {
      s3Client.doesObjectExist(metadata.s3BucketName, path.toString)
    } else {
      new java.io.File(path.toString).exists
    }
  }

  override def receive: Receive = {
    case SaveToLocal(artifactName, protocol) =>
      log.info("Receive message and starting to working...")
      val uris = generatePaths(artifactName, protocol)
      val localToSave = new File(uris("localPath").toString)

      // Validate if the protocol is correct
      if (validatePath(uris("remotePath"), true)) {
        log.info(s"Copying files from ${metadata.s3BucketName}: ${uris("remotePath")} to ${uris("localPath")}")
        //Get artifact named "uris("remotePath")" from S3 Bucket and save it to local
        s3Client.getObject(new GetObjectRequest(metadata.s3BucketName, uris("remotePath").toString), localToSave)
        log.info(s"File ${uris("localPath")} saved!")
      }
      else {
        log.error(s"Invalid protocol: ${protocol}, save process canceled!")
      }

      sender ! Done

    case SaveToRemote(artifactName, protocol) =>
      log.info("Receive message and starting to working...")
      val uris = generatePaths(artifactName, protocol)
      val fileToUpload = new File(uris("localPath").toString)

      // Validate if the protocol is correct
      if (validatePath(uris("localPath"), false)) {
        log.info(s"Copying files from ${uris("localPath")} to ${metadata.s3BucketName}: ${uris("remotePath")}")
        //Get local artifact and save to S3 Bucket with name "uris("remotePath")"
        s3Client.putObject(metadata.s3BucketName, uris("remotePath").toString, fileToUpload)
        log.info(s"File ${uris("localPath")} saved!")
      }
      else {
        log.error(s"Invalid protocol: ${protocol}, save process canceled!")
      }

      sender ! Done

    case _ =>
      log.warning("Received a bad format message...")
  }
} 

package fs2
package aws

import java.util.concurrent.Executors

import cats.effect.{ ContextShift, IO }
import com.amazonaws.services.s3.AmazonS3
import fs2.aws.internal.S3Client
import fs2.aws.s3._
import org.mockito.MockitoSugar._
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

import scala.concurrent.ExecutionContext

class S3Spec extends AnyFlatSpec with Matchers {

  private val blockingEC                        = ExecutionContext.fromExecutorService(Executors.newFixedThreadPool(6))
  implicit val ec: ExecutionContext             = ExecutionContext.global
  implicit val ioContextShift: ContextShift[IO] = IO.contextShift(ec)

  implicit val s3Client: S3Client[IO] = fs2.aws.utils.s3TestClient
  val mockS3                          = mock[AmazonS3]

  ignore should "stdout the jsonfile" in {
    readS3FileMultipart[IO]("resources", "jsontest.json", 25, mockS3).compile.toVector.unsafeRunSync should be(
      Vector()
    )
  }

  "Downloading the JSON test file by chunks" should "return the same content" in {
    readS3FileMultipart[IO]("resources", "jsontest.json", 25, mockS3)
      .through(fs2.text.utf8Decode)
      .through(fs2.text.lines)
      .compile
      .toVector
      .unsafeRunSync
      .reduce(_ + _)
      .concat("") should be(
      """{"test": 1}{"test": 2}{"test": 3}{"test": 4}{"test": 5}{"test": 6}{"test": 7}{"test": 8}"""
    )
  }

  "Downloading the JSON test file" should "return the same content" in {
    readS3File[IO]("resources", "jsontest.json", blockingEC, mockS3)
      .through(fs2.text.utf8Decode)
      .through(fs2.text.lines)
      .compile
      .toVector
      .unsafeRunSync
      .reduce(_ + _)
      .concat("") should be(
      """{"test": 1}{"test": 2}{"test": 3}{"test": 4}{"test": 5}{"test": 6}{"test": 7}{"test": 8}"""
    )
  }

  "Downloading the versioned JSON test file" should "return the same content" in {
    readS3VersionedFile[IO]("resources", "jsontest.json", version = "ABC", blockingEC, mockS3)
      .through(fs2.text.utf8Decode)
      .through(fs2.text.lines)
      .compile
      .toVector
      .unsafeRunSync
      .reduce(_ + _)
      .concat("") should be(
      """{"this": 1}{"is": 2}{"versioned": 3}{"content": 4}"""
    )
  }

  "big chunk size but small entire text" should "be trimmed to content" in {
    readS3FileMultipart[IO]("resources", "jsontest1.json", 25, mockS3)
      .through(fs2.text.utf8Decode)
      .through(fs2.text.lines)
      .compile
      .toVector
      .unsafeRunSync
      .reduce(_ + _)
      .concat("") should be("""{"test": 1}""")
  }
} 

package fs2.aws

import java.io._

import cats.effect.{ Effect, IO }
import com.amazonaws.SdkClientException
import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model.{ AmazonS3Exception, GetObjectRequest, S3ObjectInputStream }
import fs2.aws.internal._
import org.apache.http.client.methods.HttpRequestBase
import scala.io.Source

package object utils {
  val s3TestClient: S3Client[IO] = new S3Client[IO] {

    override def client: AmazonS3 =
      throw new NotImplementedError("s3 client shouldn't be used in this test client")

    override def getObjectContentOrError(
      getObjectRequest: GetObjectRequest
    )(implicit e: Effect[IO]): IO[Either[Throwable, InputStream]] =
      getObjectRequest match {
        case goe: GetObjectRequest => {
          IO[Either[Throwable, ByteArrayInputStream]] {
            val fileContent: Array[Byte] =
              try {
                Source.fromResource(goe.getKey).mkString.getBytes
              } catch {
                case _: FileNotFoundException => throw new AmazonS3Exception("File not found")
                case e: Throwable             => throw e
              }
            goe.getRange match {
              case Array(x, y) =>
                if (y > fileContent.length)
                  Right(new ByteArrayInputStream(fileContent.slice(x.toInt, fileContent.length)))
                else Right(new ByteArrayInputStream(fileContent.slice(x.toInt, y.toInt)))
            }

          } map {
            case Left(e) => Left(e)
            case Right(is) =>
              Thread.sleep(500) // simulate a call to S3
              Right(new S3ObjectInputStream(is, new HttpRequestBase {
                def getMethod = ""
              }))
          }
        }
        case _ => throw new SdkClientException("Invalid GetObjectRequest")
      }

    override def getObjectContent(
      getObjectRequest: GetObjectRequest
    )(implicit e: Effect[IO]): IO[InputStream] =
      IO[ByteArrayInputStream] {
        val fileContent: Array[Byte] =
          try {
            val testS3Resource = Option(getObjectRequest.getVersionId) match {
              case Some(version) => s"${getObjectRequest.getKey}_v$version"
              case None          => getObjectRequest.getKey
            }
            Source.fromResource(testS3Resource).mkString.getBytes
          } catch {
            case _: FileNotFoundException => throw new AmazonS3Exception("File not found")
            case e: Throwable             => throw e
          }
        new ByteArrayInputStream(fileContent)

      }.map { is =>
        Thread.sleep(500) // simulate a call to S3
        new S3ObjectInputStream(is, new HttpRequestBase {
          def getMethod = ""
        })
      }
  }

} 

package fs2.aws.internal

import java.io.InputStream

import cats.effect.Effect
import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model._

import scala.collection.JavaConverters._
import scala.util.control.Exception

private[aws] object S3Client {
  def apply[F[_]](s3: AmazonS3) = new S3ClientImpl[F](s3)
}

private[aws] class S3ClientImpl[F[_]](c: AmazonS3) extends S3Client[F] {
  override def client: AmazonS3 = c
}

private[aws] trait S3Client[F[_]] {

  def client: AmazonS3

  def getObjectContentOrError(
    getObjectRequest: GetObjectRequest
  )(implicit F: Effect[F]): F[Either[Throwable, InputStream]] =
    F.delay(Exception.nonFatalCatch either client.getObject(getObjectRequest).getObjectContent)

  def getObjectContent(getObjectRequest: GetObjectRequest)(implicit F: Effect[F]): F[InputStream] =
    F.delay(client.getObject(getObjectRequest).getObjectContent)

  def initiateMultipartUpload(
    initiateMultipartUploadRequest: InitiateMultipartUploadRequest
  )(implicit F: Effect[F]): F[InitiateMultipartUploadResult] =
    F.delay(client.initiateMultipartUpload(initiateMultipartUploadRequest))

  def uploadPart(uploadPartRequest: UploadPartRequest)(implicit F: Effect[F]): F[UploadPartResult] =
    F.delay(client.uploadPart(uploadPartRequest))

  def completeMultipartUpload(
    completeMultipartUploadRequest: CompleteMultipartUploadRequest
  )(implicit F: Effect[F]): F[CompleteMultipartUploadResult] =
    F.delay(client.completeMultipartUpload(completeMultipartUploadRequest))

  def s3ObjectSummaries(
    listObjectsV2Request: ListObjectsV2Request
  )(implicit F: Effect[F]): F[List[S3ObjectSummary]] =
    F.delay(client.listObjectsV2(listObjectsV2Request).getObjectSummaries.asScala.toList)

  def getObject(objectRequest: GetObjectRequest)(implicit F: Effect[F]): F[S3Object] =
    F.delay(client.getObject(objectRequest))
} 

package com.klibisz.elastiknn.benchmarks

import com.amazonaws.services.s3.AmazonS3
import com.klibisz.elastiknn.api.{Mapping, NearestNeighborsQuery}
import io.circe.parser.decode
import io.circe.syntax._
import zio._
import codecs._
import com.amazonaws.services.s3.model.{ListObjectsV2Request, ListObjectsV2Result}
import zio.blocking._
import zio.stream._

import scala.annotation.tailrec
import scala.collection.JavaConverters._

object ResultClient {

  trait Service {
    def find(dataset: Dataset, mapping: Mapping, query: NearestNeighborsQuery, k: Int): IO[Throwable, Option[BenchmarkResult]]
    def save(result: BenchmarkResult): IO[Throwable, Unit]
    def all(): Stream[Throwable, BenchmarkResult]
  }

  def s3(bucket: String, keyPrefix: String): ZLayer[Has[AmazonS3] with Blocking, Nothing, ResultClient] = {

    def validChars(s: String): String = s.map { c =>
      if (c.isLetter && c <= 'z' || c.isDigit || c == '.') c
      else '-'
    }

    def genKey(dataset: Dataset, mapping: Mapping, query: NearestNeighborsQuery, k: Int): String = {
      val suffix = validChars(s"res-${dataset.toString}-${mapping.toString}-${query.toString}-$k.json")
      if (keyPrefix.nonEmpty) s"$keyPrefix/$suffix".replace("//", "/")
      else suffix
    }

    ZLayer.fromServices[AmazonS3, Blocking.Service, Service] {
      case (client, blocking) =>
        new Service {
          override def find(dataset: Dataset,
                            mapping: Mapping,
                            query: NearestNeighborsQuery,
                            k: Int): IO[Throwable, Option[BenchmarkResult]] = {
            val key = genKey(dataset, mapping, query, k)
            for {
              ex <- blocking.effectBlocking(client.doesObjectExist(bucket, key))
              res: Option[BenchmarkResult] <- if (ex) {
                for {
                  body <- blocking.effectBlocking(client.getObjectAsString(bucket, key))
                  dec <- ZIO.fromEither(decode[BenchmarkResult](body))
                } yield Some(dec)
              } else ZIO.effectTotal(None)
            } yield res
          }

          override def save(result: BenchmarkResult): IO[Throwable, Unit] = {
            val key = genKey(result.dataset, result.mapping, result.query, result.k)
            for {
              bucketExists <- blocking.effectBlocking(client.doesBucketExistV2(bucket))
              _ <- if (bucketExists) ZIO.succeed(()) else blocking.effectBlocking(client.createBucket(bucket))
              _ <- blocking.effectBlocking(client.putObject(bucket, key, result.asJson.spaces2SortKeys))
            } yield ()
          }

          override def all(): Stream[Throwable, BenchmarkResult] = {

            @tailrec
            def readAllKeys(req: ListObjectsV2Request, agg: Vector[String] = Vector.empty): Vector[String] = {
              val res = client.listObjectsV2(req)
              val keys = res.getObjectSummaries.asScala.toVector.map(_.getKey).filter(_.endsWith(".json"))
              if (res.isTruncated) readAllKeys(req.withContinuationToken(res.getNextContinuationToken), agg ++ keys)
              else agg ++ keys
            }

            val req = new ListObjectsV2Request().withBucketName(bucket).withPrefix(keyPrefix)

            Stream
              .fromIterableM(blocking.effectBlocking(readAllKeys(req)))
              .mapM(key => blocking.effectBlocking(client.getObjectAsString(bucket, key)))
              .mapM(body => ZIO.fromEither(decode[BenchmarkResult](body)))
          }
        }
    }

  }

} 

package com.klibisz.elastiknn.benchmarks

import java.util.zip.GZIPInputStream

import com.amazonaws.services.s3.AmazonS3
import com.klibisz.elastiknn.api.{ElasticsearchCodec, Vec}
import com.klibisz.elastiknn.benchmarks.Dataset._
import io.circe
import zio._
import zio.stream._

import scala.io.Source
import scala.util.Random
import scala.util.hashing.MurmurHash3

object DatasetClient {

  trait Service {
    def streamTrain(dataset: Dataset, limit: Option[Int] = None): Stream[Throwable, Vec]
    def streamTest(dataset: Dataset, limit: Option[Int] = None): Stream[Throwable, Vec]
  }

  
  def s3(bucket: String, keyPrefix: String): ZLayer[Has[AmazonS3], Throwable, DatasetClient] = ZLayer.fromService[AmazonS3, Service] {
    client =>
      new Service {
        private def stream(dataset: Dataset, name: String, limit: Option[Int]): Stream[Throwable, Vec] =
          dataset match {
            case r: RandomSparseBool =>
              implicit val rng: Random = new Random(MurmurHash3.orderedHash(Seq(r.dims, name)))
              Stream
                .range(0, if (name == "train") r.train else r.test)
                .map(_ => Vec.SparseBool.random(r.dims, r.bias))
            case r: RandomDenseFloat =>
              implicit val rng: Random = new Random(MurmurHash3.orderedHash(Seq(r.dims, name)))
              Stream
                .range(0, if (name == "train") r.train else r.test)
                .map(_ => Vec.DenseFloat.random(r.dims))
            case _ =>
              def parseDecode(s: String): Either[circe.Error, Vec] =
                ElasticsearchCodec.parse(s).flatMap(j => ElasticsearchCodec.decode[Vec](j.hcursor))
              val obj = client.getObject(bucket, s"$keyPrefix/${dataset.name}/${name}.json.gz")
              val iterManaged = Managed.makeEffect(Source.fromInputStream(new GZIPInputStream(obj.getObjectContent)))(_.close())
              val lines = Stream.fromIteratorManaged(iterManaged.map(src => limit.map(n => src.getLines.take(n)).getOrElse(src.getLines())))
              val rawJson = lines.map(_.dropWhile(_ != '{'))
              rawJson.mapM(s => ZIO.fromEither(parseDecode(s)))
          }

        override def streamTrain(dataset: Dataset, limit: Option[Int]): Stream[Throwable, Vec] =
          stream(dataset, "train", limit)

        override def streamTest(dataset: Dataset, limit: Option[Int]): Stream[Throwable, Vec] =
          stream(dataset, "test", limit)
      }
  }

} 

package com.klibisz.elastiknn.benchmarks

import com.amazonaws.ClientConfiguration
import com.amazonaws.auth.{AWSCredentials, AWSStaticCredentialsProvider}
import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}

object S3Utils {

  
  def minioClient(): AmazonS3 = {
    val endpointConfig = new EndpointConfiguration("http://localhost:9000", "us-east-1")
    val clientConfig = new ClientConfiguration()
    clientConfig.setSignerOverride("AWSS3V4SignerType")
    AmazonS3ClientBuilder.standard
      .withPathStyleAccessEnabled(true)
      .withEndpointConfiguration(endpointConfig)
      .withClientConfiguration(clientConfig)
      .withCredentials(new AWSStaticCredentialsProvider(new AWSCredentials {
        override def getAWSAccessKeyId: String = "minioadmin"
        override def getAWSSecretKey: String = "minioadmin"
      }))
      .build()
  }

  def defaultClient(): AmazonS3 = AmazonS3ClientBuilder.defaultClient()

} 

package crossroad0201.aws.flywaylambda.deploy

import java.nio.file.{Files, Path, Paths}
import java.util.{Properties => JProperties}

import com.amazonaws.services.lambda.runtime.Context
import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model.S3ObjectSummary

import scala.annotation.tailrec
import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
import scala.util.Try

class S3SourceFlywayDeployer(s3Client: AmazonS3, srcBucketName: String, srcPrefix: String, flywayConfFileName: String) extends FlywayDeployer {

  def deploy(implicit context: Context): Try[FlywayDeployment] = Try {
    val logger = context.getLogger

    val tmpDir = Files.createDirectories(Paths.get("/tmp", context.getAwsRequestId))
    Files.createDirectories(Paths.get(tmpDir.toString, srcPrefix))

    @tailrec
    def deployInternal(objects: List[S3ObjectSummary], acc: (Option[JProperties], ListBuffer[Path])): (Option[JProperties], Seq[Path]) = {
      def loadConf(key: String) = {
        val o = s3Client.getObject(srcBucketName, key)
        val props = new JProperties
        props.load(o.getObjectContent)
        logger.log(s"Flyway configuration loaded. s3://$srcBucketName/$key")
        (Some(props), acc._2)
      }
      def createDir(key: String) = {
        val dir = Files.createDirectories(Paths.get(tmpDir.toString, key))
        logger.log(s"Dir created. $dir")
        acc
      }
      def createSqlFile(key: String) = {
        val o = s3Client.getObject(srcBucketName, key)
        val file = Paths.get(tmpDir.toString, key)
        val fileSize = Files.copy(o.getObjectContent, file)
        logger.log(s"SQL file created. $file($fileSize Byte)")
        acc._2 += file
        acc
      }

      objects match {
        case Nil => (acc._1, acc._2)
        case x :: xs =>
          val _acc = x.getKey match {
            case key if key.endsWith(flywayConfFileName) => loadConf(key)
            case key if key.endsWith("/") => createDir(key)
            case key if key.endsWith(".sql") => createSqlFile(key)
            case _ => acc
          }
          deployInternal(xs, _acc)
      }
    }

    val objectSummaries = {
      val objects = s3Client.listObjects(srcBucketName, srcPrefix)
      objects.getObjectSummaries.asScala.toList.sortWith { (x, y) =>
        x.getKey.compareTo(y.getKey) < 1
      }
    }

    logger.log(s"Deploying Flyway resources from $srcBucketName/$srcPrefix... ${objectSummaries.map(_.getKey).mkString(", ")}")

    deployInternal(objectSummaries, (None, ListBuffer())) match {
      case (Some(conf), sqlFiles) =>
        FlywayDeployment(
          srcBucketName,
          srcPrefix,
          conf,
          s"filesystem:${Paths.get(tmpDir.toString, srcPrefix).toString}",
          sqlFiles)
      case _ => throw new IllegalStateException(s"$flywayConfFileName does not exists.")
    }
  }

} 

package crossroad0201.aws.flywaylambda

import java.io.{BufferedOutputStream, InputStream, OutputStream, PrintWriter}

import com.amazonaws.regions.{Region, Regions}
import com.amazonaws.services.lambda.runtime.{Context, RequestStreamHandler}
import com.amazonaws.services.s3.{AmazonS3, AmazonS3Client}

import scala.io.{BufferedSource, Codec}
import scala.util.{Failure, Success, Try}

class InvokeMigrationHandler extends RequestStreamHandler with S3MigrationHandlerBase {
  type BucketName = String
  type Prefix = String
  type ConfFileName = String

  override def handleRequest(input: InputStream, output: OutputStream, context: Context): Unit = {
    def parseInput: Try[(BucketName, Prefix, ConfFileName)] = Try {
      import spray.json._
      import DefaultJsonProtocol._

      val json = new BufferedSource(input)(Codec("UTF-8")).mkString
      val jsObj = JsonParser(json).toJson.asJsObject
      jsObj.getFields(
        "bucket_name",
        "prefix"
      ) match {
        case Seq(JsString(b), JsString(p)) => {
          jsObj.getFields(
            "flyway_conf"
          ) match {
            case Seq(JsString(c)) => (b, p, c)
            case _ => (b, p, "flyway.conf")
          }
        }
        case _ => throw new IllegalArgumentException(s"Missing require key [bucketName, prefix]. - $json")
      }
    }

    val logger = context.getLogger

    implicit val s3Client: AmazonS3 = new AmazonS3Client().withRegion(Region.getRegion(Regions.fromName(sys.env("AWS_REGION"))))

    (for {
      i <- parseInput
      _ = { logger.log(s"Flyway migration start. by invoke lambda function(${i._1}, ${i._2}, ${i._3}).") }
      r <- migrate(i._1, i._2, i._3)(context, s3Client)
    } yield r) match {
      case Success(r) =>
        logger.log(r)
        val b = r.getBytes("UTF-8")
        val bout = new BufferedOutputStream(output)
        Stream.continually(bout.write(b))
        bout.flush()
      case Failure(e) =>
        e.printStackTrace()
        val w = new PrintWriter(output)
        w.write(e.toString)
        w.flush()
    }
  }

} 

package crossroad0201.aws.flywaylambda

import com.amazonaws.regions.{Region, Regions}
import com.amazonaws.services.lambda.runtime.events.S3Event
import com.amazonaws.services.lambda.runtime.{Context, RequestHandler}
import com.amazonaws.services.s3.{AmazonS3, AmazonS3Client}

import scala.util.{Failure, Success}

class S3EventMigrationHandler extends RequestHandler[S3Event, Unit] with S3MigrationHandlerBase {

  override def handleRequest(event: S3Event, context: Context): Unit = {
    val logger = context.getLogger

    implicit val s3Client: AmazonS3 = new AmazonS3Client().withRegion(Region.getRegion(Regions.fromName(event.getRecords.get(0).getAwsRegion)))

    logger.log(s"Flyway migration start. by ${event.getRecords.get(0).getEventName} s3://${event.getRecords.get(0).getS3.getBucket.getName}/${event.getRecords.get(0).getS3.getObject.getKey}")

    val s3 = event.getRecords.get(0).getS3
    val migrationPrefix = {
      val objectKey = s3.getObject.getKey
      objectKey.substring(0, objectKey.lastIndexOf("/"))
    }

    migrate(s3.getBucket.getName, migrationPrefix)(context, s3Client) match {
      case Success(r) => logger.log(r)
      case Failure(e) => e.printStackTrace()
    }
  }

} 

package crossroad0201.aws.flywaylambda

import java.text.SimpleDateFormat
import java.util.Date

import com.amazonaws.services.lambda.runtime.Context
import com.amazonaws.services.s3.AmazonS3
import crossroad0201.aws.flywaylambda.deploy.{FlywayDeployment, S3SourceFlywayDeployer}
import crossroad0201.aws.flywaylambda.migration.{FlywayMigrator, MigrationInfo, MigrationResult}
import spray.json.DefaultJsonProtocol

import scala.util.Try

object MigrationResultProtocol extends DefaultJsonProtocol {
  import spray.json._

  implicit val DateFormat = new RootJsonFormat[Date] {
    override def write(value: Date): JsValue = if (value == null) JsNull else JsString(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'").format(value))
    override def read(json: JsValue): Date = ???
  }
  implicit val migrationInfoFormat = jsonFormat6(MigrationInfo.apply)
  implicit val migrationResultFormat = jsonFormat5(MigrationResult.apply)
}

trait S3MigrationHandlerBase extends FlywayMigrator {

  type ResultJson = String
  type ResultStoredPath = String

  protected def migrate(bucketName: String, prefix: String, flywayConfFileName: String = "flyway.conf")(implicit context: Context, s3Client: AmazonS3): Try[ResultJson] = {
    val logger = context.getLogger

    def resultJson(result: MigrationResult): ResultJson = {
      import MigrationResultProtocol._
      import spray.json._

      result.toJson.prettyPrint
    }

    def storeResult(deployment: FlywayDeployment, result: MigrationResult): ResultStoredPath = {
      val jsonPath = s"${deployment.sourcePrefix}/migration-result.json"
      s3Client.putObject(deployment.sourceBucket, jsonPath, resultJson(result))
      jsonPath
    }

    for {
      // Deploy Flyway resources.
      d <- new S3SourceFlywayDeployer(s3Client, bucketName, prefix, flywayConfFileName).deploy
      _ = {
        logger.log(
          s"""--- Flyway configuration ------------------------------------
             |flyway.url      = ${d.url}
             |flyway.user     = ****
             |flyway.password = ****
             |
             |SQL locations   = ${d.location}
             |SQL files       = ${d.sqlFiles.mkString(", ")}
             |-------------------------------------------------------------
              """.stripMargin)
      }

      // Migrate DB.
      r = migrate(d)
      _ = {
        logger.log(s"${r.message}!. ${r.appliedCount} applied.")
        r.infos.foreach { i =>
          logger.log(s"Version=${i.version}, Type=${i.`type`}, State=${i.state} InstalledAt=${i.installedAt} ExecutionTime=${i.execTime} Description=${i.description}")
        }
      }

      // Store migration result.
      storedPath = storeResult(d, r)
      _ = logger.log(s"Migration result stored to $bucketName/$storedPath.")

    } yield resultJson(r)
  }

} 

package cool.graph.aws

import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials}
import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
import com.amazonaws.services.kinesis.{AmazonKinesis, AmazonKinesisClientBuilder}
import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
import com.amazonaws.services.sns.{AmazonSNS, AmazonSNSAsyncClientBuilder}

object AwsInitializers {
  lazy val accessKeyId = sys.env.getOrElse("AWS_ACCESS_KEY_ID", "")
  lazy val accessKey   = sys.env.getOrElse("AWS_SECRET_ACCESS_KEY", "")
  lazy val credentials = new BasicAWSCredentials(accessKeyId, accessKey)

  def createKinesis(): AmazonKinesis = {
    AmazonKinesisClientBuilder
      .standard()
      .withCredentials(new AWSStaticCredentialsProvider(credentials))
      .withEndpointConfiguration(new EndpointConfiguration(sys.env("KINESIS_ENDPOINT"), sys.env("AWS_REGION")))
      .build()
  }

  def createSns(): AmazonSNS = {
    AmazonSNSAsyncClientBuilder.standard
      .withCredentials(new AWSStaticCredentialsProvider(credentials))
      .withEndpointConfiguration(new EndpointConfiguration(sys.env("SNS_ENDPOINT"), sys.env("AWS_REGION")))
      .build
  }

  def createS3(): AmazonS3 = {
    AmazonS3ClientBuilder.standard
      .withCredentials(new AWSStaticCredentialsProvider(credentials))
      .withEndpointConfiguration(new EndpointConfiguration(sys.env("FILEUPLOAD_S3_ENDPOINT"), sys.env("AWS_REGION")))
      .build
  }

  def createExportDataS3(): AmazonS3 = {
    AmazonS3ClientBuilder.standard
      .withCredentials(new AWSStaticCredentialsProvider(credentials))
      .withEndpointConfiguration(new EndpointConfiguration(sys.env("DATA_EXPORT_S3_ENDPOINT"), sys.env("AWS_REGION")))
      .build
  }

  // This is still in the old SBS AWS account
  def createS3Fileupload(): AmazonS3 = {
    val credentials = new BasicAWSCredentials(
      sys.env("FILEUPLOAD_S3_AWS_ACCESS_KEY_ID"),
      sys.env("FILEUPLOAD_S3_AWS_SECRET_ACCESS_KEY")
    )

    AmazonS3ClientBuilder.standard
      .withCredentials(new AWSStaticCredentialsProvider(credentials))
      .withEndpointConfiguration(new EndpointConfiguration(sys.env("FILEUPLOAD_S3_ENDPOINT"), sys.env("AWS_REGION")))
      .build
  }
} 

package cool.graph.shared.functions.lambda

import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials}
import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
import cool.graph.shared.models.Project
import play.api.libs.json.Json
import software.amazon.awssdk.auth.{AwsCredentials, StaticCredentialsProvider}
import software.amazon.awssdk.regions.Region
import software.amazon.awssdk.services.lambda.LambdaAsyncClient

object LambdaDeploymentAccount {
  implicit val lambdaDeploymentBucket        = Json.format[LambdaDeploymentBucket]
  implicit val lambdaDeploymentAccountFormat = Json.format[LambdaDeploymentAccount]
}

case class LambdaDeploymentAccount(
    id: String,
    accessKeyID: String,
    accessKey: String,
    deployIamArn: String,
    deploymentEnabled: Boolean,
    deploymentBuckets: Vector[LambdaDeploymentBucket]
) {
  lazy val credentialsProvider = new StaticCredentialsProvider(new AwsCredentials(accessKeyID, accessKey))
  lazy val s3Credentials       = new BasicAWSCredentials(accessKeyID, accessKey)

  def bucket(project: Project): String = {
    val region = getRegion(project.region.toString)
    deploymentBuckets.find(_.region == region).getOrElse(sys.error("Region is not supported for lambda deployment")).deploymentBucket
  }

  def lambdaClient(project: Project): LambdaAsyncClient =
    LambdaAsyncClient
      .builder()
      .region(Region.of(project.region.toString))
      .credentialsProvider(credentialsProvider)
      .build()

  def s3Client(project: Project): AmazonS3 = {
    val region = getRegion(project.region.toString)
    AmazonS3ClientBuilder.standard
      .withCredentials(new AWSStaticCredentialsProvider(s3Credentials))
      .withEndpointConfiguration(new EndpointConfiguration(s"s3-$region.amazonaws.com", region))
      .build
  }

  private def getRegion(region: String) = Region.of(region).toString
}

case class LambdaDeploymentBucket(region: String, deploymentBucket: String) 

package org.embulk.output.s3_parquet.aws

import com.amazonaws.client.builder.AwsClientBuilder
import com.amazonaws.services.glue.{AWSGlue, AWSGlueClientBuilder}
import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
import com.amazonaws.services.s3.transfer.{
  TransferManager,
  TransferManagerBuilder
}

object Aws {

  trait Task
      extends AwsCredentials.Task
      with AwsEndpointConfiguration.Task
      with AwsClientConfiguration.Task
      with AwsS3Configuration.Task

  def apply(task: Task): Aws = {
    new Aws(task)
  }

}

class Aws(task: Aws.Task) {

  def withS3[A](f: AmazonS3 => A): A = {
    val builder: AmazonS3ClientBuilder = AmazonS3ClientBuilder.standard()
    AwsS3Configuration(task).configureAmazonS3ClientBuilder(builder)
    val svc = createService(builder)
    try f(svc)
    finally svc.shutdown()
  }

  def withTransferManager[A](f: TransferManager => A): A = {
    withS3 { s3 =>
      val svc = TransferManagerBuilder.standard().withS3Client(s3).build()
      try f(svc)
      finally svc.shutdownNow(false)
    }
  }

  def withGlue[A](f: AWSGlue => A): A = {
    val builder: AWSGlueClientBuilder = AWSGlueClientBuilder.standard()
    val svc = createService(builder)
    try f(svc)
    finally svc.shutdown()
  }

  def createService[S <: AwsClientBuilder[S, T], T](
      builder: AwsClientBuilder[S, T]
  ): T = {
    AwsEndpointConfiguration(task).configureAwsClientBuilder(builder)
    AwsClientConfiguration(task).configureAwsClientBuilder(builder)
    builder.setCredentials(AwsCredentials(task).createAwsCredentialsProvider)

    builder.build()
  }
} 

package org.hatdex.hat.api.service

import javax.inject.Inject

import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model.{ GeneratePresignedUrlRequest, SSEAlgorithm }
import com.google.inject.name.Named
import com.typesafe.config.Config
import org.hatdex.hat.resourceManagement.HatServer
import play.api.{ ConfigLoader, Logger }

import scala.concurrent.Future
import scala.concurrent.duration.FiniteDuration

trait FileManager {
  def getUploadUrl(filename: String, maybeContentType: Option[String])(implicit hatServer: HatServer): Future[String]
  def getContentUrl(filename: String)(implicit hatServer: HatServer): Future[String]
  def getFileSize(fileName: String)(implicit hatServer: HatServer): Future[Long]
  def deleteContents(filename: String)(implicit hatServer: HatServer): Future[Unit]
}

case class AwsS3Configuration(
    bucketName: String,
    accessKeyId: String,
    secretKey: String,
    region: String,
    signedUrlExpiry: FiniteDuration)

object AwsS3Configuration {
  implicit val configLoader: ConfigLoader[AwsS3Configuration] = new ConfigLoader[AwsS3Configuration] {
    def load(rootConfig: Config, path: String): AwsS3Configuration = {
      val config = rootConfig.getConfig(path)
      AwsS3Configuration(
        bucketName = config.getString("bucketName"),
        accessKeyId = config.getString("accessKeyId"),
        secretKey = config.getString("secretKey"),
        region = config.getString("region"),
        signedUrlExpiry = ConfigLoader.finiteDurationLoader.load(config, "signedUrlExpiry"))
    }
  }
}

class FileManagerS3 @Inject() (
    awsS3Configuration: AwsS3Configuration,
    @Named("s3client-file-manager") s3client: AmazonS3)(implicit ec: RemoteExecutionContext) extends FileManager {

  private val logger = Logger(this.getClass)
  private val bucketName = awsS3Configuration.bucketName

  def getUploadUrl(fileName: String, maybeContentType: Option[String])(implicit hatServer: HatServer): Future[String] = {
    val expiration = org.joda.time.DateTime.now().plus(awsS3Configuration.signedUrlExpiry.toMillis)

    val generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, s"${hatServer.domain}/$fileName")
      .withMethod(com.amazonaws.HttpMethod.PUT)
      .withExpiration(expiration.toDate)
      .withSSEAlgorithm(SSEAlgorithm.AES256)

    // TODO: to be replaced with mandatory validated content MIME type in API v2.7
    val generatePresignedUrlRequestWithContent = maybeContentType.map { contentType =>
      generatePresignedUrlRequest.withContentType(contentType)
    }.getOrElse(generatePresignedUrlRequest)

    val url = Future(s3client.generatePresignedUrl(generatePresignedUrlRequestWithContent))
    url.map(_.toString)
  }

  def getContentUrl(fileName: String)(implicit hatServer: HatServer): Future[String] = {
    val expiration = org.joda.time.DateTime.now().plus(awsS3Configuration.signedUrlExpiry.toMillis)

    val generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, s"${hatServer.domain}/$fileName")
      .withMethod(com.amazonaws.HttpMethod.GET)
      .withExpiration(expiration.toDate)

    val url = Future(s3client.generatePresignedUrl(generatePresignedUrlRequest))
    url.map(_.toString)
  }

  def getFileSize(fileName: String)(implicit hatServer: HatServer): Future[Long] = {
    logger.debug(s"Getting file size for $bucketName ${hatServer.domain}/$fileName")
    Future(s3client.getObjectMetadata(bucketName, s"${hatServer.domain}/$fileName"))
      .map { metadata => Option(metadata.getContentLength).getOrElse(0L) }
      .recover { case _ => 0L }
  }

  def deleteContents(fileName: String)(implicit hatServer: HatServer): Future[Unit] = {
    Future(s3client.deleteObject(bucketName, s"${hatServer.domain}/$fileName"))
  }
} 

package org.hatdex.hat.modules

import com.amazonaws.auth.{ AWSStaticCredentialsProvider, BasicAWSCredentials }
import com.amazonaws.services.s3.{ AmazonS3, AmazonS3ClientBuilder }
import com.google.inject.name.Named
import com.google.inject.{ AbstractModule, Provides }
import net.codingwell.scalaguice.ScalaModule
import org.hatdex.hat.api.service.{ AwsS3Configuration, FileManager, FileManagerS3 }
import play.api.Configuration
import play.api.libs.concurrent.AkkaGuiceSupport

class FileManagerModule extends AbstractModule with ScalaModule with AkkaGuiceSupport {

  override def configure() = {
    bind[FileManager].to[FileManagerS3]
    ()
  }

  @Provides
  def provideCookieAuthenticatorService(configuration: Configuration): AwsS3Configuration = {
    import AwsS3Configuration.configLoader
    configuration.get[AwsS3Configuration]("storage.s3Configuration")
  }

  @Provides @Named("s3client-file-manager")
  def provides3Client(configuration: AwsS3Configuration): AmazonS3 = {
    val awsCreds: BasicAWSCredentials = new BasicAWSCredentials(configuration.accessKeyId, configuration.secretKey)
    AmazonS3ClientBuilder.standard()
      .withRegion(configuration.region)
      .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
      .build()
  }

} 

package ionroller.aws

import com.amazonaws.auth.AWSCredentialsProvider
import com.amazonaws.services.autoscaling.AmazonAutoScaling
import com.amazonaws.services.ec2.AmazonEC2Client
import com.amazonaws.services.elasticbeanstalk.AWSElasticBeanstalk
import com.amazonaws.services.elasticloadbalancing.AmazonElasticLoadBalancing
import com.amazonaws.services.route53.AmazonRoute53
import com.amazonaws.services.s3.AmazonS3

import scalaz.concurrent.Task
import scalaz.{Kleisli, Nondeterminism}

class AWSClientCache(
  val role: String,
  val credentialsProvider: AWSCredentialsProvider,
  val route53: AmazonRoute53,
  val elasticBeanstalk: AWSElasticBeanstalk,
  val s3: AmazonS3,
  val asg: AmazonAutoScaling,
  val elb: AmazonElasticLoadBalancing
)

object AWSClientCache {
  private[ionroller] val cache: java.util.concurrent.ConcurrentHashMap[String, AWSClientCache] = new java.util.concurrent.ConcurrentHashMap

  val getCache: Kleisli[Task, String, AWSClientCache] = {
    Kleisli { role =>
      Option(cache.get(role)) match {
        case None =>
          for {
            credentials <- CredentialsProvider(role)
            route53Client = Route53.client(credentials)
            elasticBeanstalkClient = ElasticBeanstalk.client(credentials)
            s3Client = S3.client(credentials)
            asgClient = AutoScaling.client(credentials)
            elbClient = ElasticLoadBalancing.client(credentials)
            newItem <- Nondeterminism[Task].apply5(route53Client, elasticBeanstalkClient, s3Client, asgClient, elbClient) {
              case (r53, eb, s3, asg, elb) =>
                val newEntry = new AWSClientCache(role, credentials, r53, eb, s3, asg, elb)
                cache.put(role, newEntry)
                newEntry
            }
          } yield newItem

        case Some(e) => Task.now(e)
      }
    }
  }
} 

package jp.ne.opt.redshiftfake

import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
import java.nio.charset.StandardCharsets
import java.util.zip.GZIPOutputStream

import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model.{ObjectMetadata, PutObjectRequest}
import jp.ne.opt.redshiftfake.util.Loan.using
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream

object S3Util {

   def loadGzippedDataToS3(s3Client: AmazonS3, data: String, bucket: String, key: String): Unit = {
    val arrayOutputStream = new ByteArrayOutputStream()
    using(new GZIPOutputStream(arrayOutputStream)) (gzipOutStream => {
      gzipOutStream.write(data.getBytes(StandardCharsets.UTF_8))
    })
    val buf = arrayOutputStream.toByteArray
    val metadata = new ObjectMetadata
    metadata.setContentLength(buf.length)
    val request = new PutObjectRequest(bucket, key, new ByteArrayInputStream(buf), metadata)

    s3Client.putObject(request)
  }

   def loadBzipped2DataToS3(s3Client: AmazonS3, data: String, bucket: String, key: String): Unit = {
    val arrayOutputStream = new ByteArrayOutputStream()
    using(new BZip2CompressorOutputStream(arrayOutputStream)) (bzip2OutStream => {
      bzip2OutStream.write(data.getBytes(StandardCharsets.UTF_8))
    })
    val buf = arrayOutputStream.toByteArray
    val metadata = new ObjectMetadata
    metadata.setContentLength(buf.length)
    val request = new PutObjectRequest(bucket, key, new ByteArrayInputStream(buf), metadata)

    s3Client.putObject(request)
  }

   def loadDataToS3(s3Client: AmazonS3, data: String, bucket: String, key: String): Unit = {
    val buf = data.getBytes
    val metadata = new ObjectMetadata
    metadata.setContentLength(buf.length)
    val request = new PutObjectRequest(bucket, key, new ByteArrayInputStream(buf), metadata)

    s3Client.putObject(request)
  }
} 

package sbtlighter

import scala.collection.JavaConverters._
import scala.io.Source

import com.amazonaws.services.elasticmapreduce.model.Configuration
import com.amazonaws.services.s3.AmazonS3
import io.circe.generic.auto._
import io.circe.parser._

case class EmrConfig(
    Classification: String,
    Properties: Option[Map[String, String]],
    Configurations: Option[Seq[EmrConfig]]
) {
  def withProperties(props: (String, String)*) = {
    this.copy(Properties = Some(props.toMap))
  }

  def withEmrConfigs(configs: EmrConfig*) = {
    this.copy(Configurations = Some(configs))
  }

  def toAwsEmrConfig(): Configuration = {
    Some(new Configuration().withClassification(Classification))
      .map { c =>
        Properties.map(props => c.withProperties(props.asJava)).getOrElse(c)
      }
      .map { c =>
        Configurations
          .map { configs =>
            c.withConfigurations(configs.map(_.toAwsEmrConfig): _*)
          }
          .getOrElse(c)
      }
      .get
  }
}

object EmrConfig {
  def apply(classification: String): EmrConfig =
    EmrConfig(classification, None, None)

  def parseJson(jsonString: String) = decode[List[EmrConfig]](jsonString)
  def parseJsonFromS3(s3Url: String)(implicit s3: AmazonS3) = {
    val s3JsonUrl = new S3Url(s3Url)
    val jsonString = s3.getObjectAsString(s3JsonUrl.bucket, s3JsonUrl.key)
    parseJson(jsonString)
  }
} 

package com.ing.wbaa.testkit

import java.io.{File, RandomAccessFile}

import com.amazonaws.services.s3.AmazonS3
import com.ing.wbaa.testkit.awssdk.S3SdkHelpers
import org.scalatest.Assertion

import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Random, Try}

trait RokkuFixtures extends S3SdkHelpers {

  
  def withHomeBucket(s3Client: AmazonS3, objects: Seq[String])(testCode: String => Future[Assertion])(implicit exCtx: ExecutionContext): Future[Assertion] = {
    val testBucket = "home"
    Try(s3Client.createBucket(testBucket))
    objects.foreach(obj => s3Client.putObject(testBucket, obj, ""))
    testCode(testBucket).andThen {
      case _ =>
      cleanBucket(s3Client, testBucket)
    }
  }

  private def cleanBucket(s3Client: AmazonS3, bucketName: String) = {
    import scala.collection.JavaConverters._
    s3Client.listObjectsV2(bucketName).getObjectSummaries.asScala.toList.map(_.getKey).foreach { key =>
      s3Client.deleteObject(bucketName, key)
    }
  }
} 

package com.ing.wbaa.testkit.awssdk

import java.io.File

import akka.http.scaladsl.model.Uri.Authority
import com.amazonaws.ClientConfiguration
import com.amazonaws.auth.{AWSCredentials, AWSStaticCredentialsProvider, BasicSessionCredentials}
import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
import com.amazonaws.services.s3.transfer.TransferManagerBuilder
import com.amazonaws.services.s3.transfer.model.UploadResult
import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
import com.typesafe.config.ConfigFactory

import scala.collection.JavaConverters._


trait S3SdkHelpers {
  val awsRegion = ConfigFactory.load().getString("rokku.storage.s3.region")

  def getAmazonS3(authority: Authority,
                  credentials: AWSCredentials = new BasicSessionCredentials("accesskey", "secretkey", "token")
                 ): AmazonS3 = {
    val cliConf = new ClientConfiguration()
    cliConf.setMaxErrorRetry(1)

    AmazonS3ClientBuilder
      .standard()
      .withClientConfiguration(cliConf)
      .withCredentials(new AWSStaticCredentialsProvider(credentials))
      .withPathStyleAccessEnabled(true)
      .withEndpointConfiguration(new EndpointConfiguration(s"http://s3.localhost:${authority.port}", awsRegion))
      .build()
  }

  def getKeysInBucket(sdk: AmazonS3, bucket: String): List[String] =
    sdk
      .listObjectsV2(bucket)
      .getObjectSummaries
      .asScala.toList
      .map(_.getKey)

  def doMultiPartUpload(sdk: AmazonS3, bucket: String, file: String, key: String): UploadResult = {
    val upload = TransferManagerBuilder
      .standard()
      .withS3Client(sdk)
      .build()
      .upload(bucket, key, new File(file))

    upload.waitForUploadResult()
  }
} 

package com.ing.wbaa.rokku.proxy

import akka.http.scaladsl.model.Uri.Authority
import com.amazonaws.ClientConfiguration
import com.amazonaws.auth.{AWSCredentials, AWSStaticCredentialsProvider, BasicSessionCredentials}
import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}

class RokkuS3ProxyVirtualHostedItTest extends RokkuS3ProxyItTest {

  override def getAmazonS3(authority: Authority,
                           credentials: AWSCredentials = new BasicSessionCredentials("accesskey", "secretkey", "token")
                          ): AmazonS3 = {
    val cliConf = new ClientConfiguration()
    cliConf.setMaxErrorRetry(1)

    AmazonS3ClientBuilder
      .standard()
      .withClientConfiguration(cliConf)
      .withCredentials(new AWSStaticCredentialsProvider(credentials))
      .withPathStyleAccessEnabled(false)
      .withEndpointConfiguration(new EndpointConfiguration(s"http://s3.localhost:${authority.port}", awsRegion))
      .build()
  }
} 

package com.ing.wbaa.rokku.proxy.persistence

import akka.Done
import akka.actor.{ActorSystem, Props}
import akka.http.scaladsl.model.HttpRequest
import akka.http.scaladsl.model.Uri.{Authority, Host}
import akka.persistence.cassandra.query.scaladsl.CassandraReadJournal
import akka.persistence.query.PersistenceQuery
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.Sink
import com.amazonaws.services.s3.AmazonS3
import com.ing.wbaa.rokku.proxy.RokkuS3Proxy
import com.ing.wbaa.rokku.proxy.config.{HttpSettings, KafkaSettings, StorageS3Settings}
import com.ing.wbaa.rokku.proxy.data._
import com.ing.wbaa.rokku.proxy.handler.parsers.RequestParser
import com.ing.wbaa.rokku.proxy.handler.{FilterRecursiveListBucketHandler, RequestHandlerS3Cache}
import com.ing.wbaa.rokku.proxy.provider.{AuditLogProvider, MessageProviderKafka, SignatureProviderAws}
import com.ing.wbaa.rokku.proxy.queue.MemoryUserRequestQueue
import com.ing.wbaa.testkit.RokkuFixtures
import org.scalatest.Assertion
import org.scalatest.diagrams.Diagrams
import org.scalatest.wordspec.AsyncWordSpec

import scala.concurrent.duration._
import scala.concurrent.{Await, Future}

class HttpRequestRecorderItTest extends AsyncWordSpec with Diagrams with RokkuFixtures {
  implicit val testSystem: ActorSystem = ActorSystem.create("test-system")
  implicit val mat: ActorMaterializer = ActorMaterializer()

  val rokkuHttpSettings: HttpSettings = new HttpSettings(testSystem.settings.config) {
    override val httpPort: Int = 0
    override val httpBind: String = "127.0.0.1"
  }

  def withS3SdkToMockProxy(testCode: AmazonS3 => Assertion): Future[Assertion] = {
    val proxy: RokkuS3Proxy = new RokkuS3Proxy with RequestHandlerS3Cache with SignatureProviderAws
      with FilterRecursiveListBucketHandler with MessageProviderKafka with AuditLogProvider with MemoryUserRequestQueue with RequestParser {
      override implicit lazy val system: ActorSystem = testSystem
      override val httpSettings: HttpSettings = rokkuHttpSettings

      override def isUserAuthorizedForRequest(request: S3Request, user: User)(implicit id: RequestId): Boolean = true

      override def isUserAuthenticated(httpRequest: HttpRequest, awsSecretKey: AwsSecretKey)(implicit id: RequestId): Boolean = true

      override val storageS3Settings: StorageS3Settings = StorageS3Settings(testSystem)
      override val kafkaSettings: KafkaSettings = KafkaSettings(testSystem)

      override def areCredentialsActive(awsRequestCredential: AwsRequestCredential)(implicit id: RequestId): Future[Option[User]] =
        Future(Some(User(UserRawJson("userId", Some(Set("group")), "accesskey", "secretkey", None))))

      def createLineageFromRequest(httpRequest: HttpRequest, userSTS: User, userIPs: UserIps)(implicit id: RequestId): Future[Done] = Future.successful(Done)

      override protected def auditEnabled: Boolean = false

      override val requestPersistenceEnabled: Boolean = true
      override val configuredPersistenceId: String = "localhost-1"
    }
    proxy.startup.map { binding =>
      try testCode(getAmazonS3(
        authority = Authority(Host(binding.localAddress.getAddress), binding.localAddress.getPort)
      ))
      finally proxy.shutdown()
    }
  }

  private val CHECKER_PERSISTENCE_ID = "localhost-1"
  val requestRecorder = testSystem.actorOf(Props(classOf[HttpRequestRecorder]), CHECKER_PERSISTENCE_ID)

  val queries = PersistenceQuery(testSystem)
    .readJournalFor[CassandraReadJournal](CassandraReadJournal.Identifier)


  "S3 Proxy" should {
    s"with Request Recorder" that {
      "persists requests in Cassandra" in withS3SdkToMockProxy { sdk =>
        withBucket(sdk) { bucketName =>
          Thread.sleep(6000)
          val storedInCassandraF = queries.currentEventsByPersistenceId(CHECKER_PERSISTENCE_ID, 1L, Long.MaxValue)
            .map(_.event)
            .runWith(Sink.seq)
            .mapTo[Seq[ExecutedRequestEvt]]
          val r = Await.result(storedInCassandraF, 5.seconds).filter(_.httpRequest.getUri().toString.contains(bucketName))
          assert(r.size == 1)
          assert(r.head.userSTS.userName.value == "userId")
        }
      }
    }
  }
} 

package com.ing.wbaa.rokku.proxy.provider.aws

import com.amazonaws.auth.{ AWSStaticCredentialsProvider, BasicAWSCredentials }
import com.amazonaws.client.builder.AwsClientBuilder
import com.amazonaws.regions.Regions
import com.amazonaws.services.s3.model.{ AccessControlList, BucketPolicy, GroupGrantee, Permission }
import com.amazonaws.services.s3.{ AmazonS3, AmazonS3ClientBuilder }
import com.ing.wbaa.rokku.proxy.config.StorageS3Settings
import com.ing.wbaa.rokku.proxy.data.RequestId
import com.ing.wbaa.rokku.proxy.handler.LoggerHandlerWithId

import scala.concurrent.{ ExecutionContext, Future }
import scala.util.{ Failure, Success, Try }

trait S3Client {
  protected[this] implicit def executionContext: ExecutionContext

  private val logger = new LoggerHandlerWithId
  protected[this] def storageS3Settings: StorageS3Settings

  protected[this] lazy val s3Client: AmazonS3 = {
    val credentials = new BasicAWSCredentials(
      storageS3Settings.storageS3AdminAccesskey,
      storageS3Settings.storageS3AdminSecretkey)

    val endpointConfiguration = new AwsClientBuilder.EndpointConfiguration(
      s"http://${storageS3Settings.storageS3Authority.host.address()}:${storageS3Settings.storageS3Authority.port}",
      Regions.US_EAST_1.getName)

    AmazonS3ClientBuilder.standard()
      .withPathStyleAccessEnabled(true)
      .withCredentials(new AWSStaticCredentialsProvider(credentials))
      .withEndpointConfiguration(endpointConfiguration)
      .build()
  }

  
  protected[this] def setDefaultBucketAclAndPolicy(bucketName: String)(implicit id: RequestId): Future[Unit] = Future {
    Try {
      logger.info("setting bucket acls and policies for bucket {}", bucketName)
      val acl = s3Client.getBucketAcl(bucketName)
      acl.revokeAllPermissions(GroupGrantee.AuthenticatedUsers)
      acl.grantPermission(GroupGrantee.AuthenticatedUsers, Permission.Read)
      acl.grantPermission(GroupGrantee.AuthenticatedUsers, Permission.Write)
      s3Client.setBucketAcl(bucketName, acl)
      s3Client.setBucketPolicy(bucketName, """{"Statement": [{"Action": ["s3:GetObject"],"Effect": "Allow","Principal": "*","Resource": ["arn:aws:s3:::*"]}],"Version": "2012-10-17"}""")
    } match {
      case Failure(exception) => logger.error("setting bucket acls and policies ex={}", exception.getMessage)
      case Success(_)         => logger.info("acls and policies for bucket {} done", bucketName)
    }
  }

  def getBucketAcl(bucketName: String): Future[AccessControlList] = Future {
    s3Client.getBucketAcl(bucketName)
  }

  def getBucketPolicy(bucketName: String): Future[BucketPolicy] = Future {
    s3Client.getBucketPolicy(bucketName)
  }

  def listBucket: String = {
    s3Client.listObjects(storageS3Settings.bucketName).getBucketName
  }
} 

package com.ing.wbaa.rokku.proxy.handler.radosgw

import akka.actor.ActorSystem
import com.amazonaws.services.s3.AmazonS3
import com.ing.wbaa.rokku.proxy.config.StorageS3Settings
import com.ing.wbaa.rokku.proxy.data.{ AwsAccessKey, AwsSecretKey, RequestId, User, UserName }
import com.ing.wbaa.rokku.proxy.handler.LoggerHandlerWithId
import org.twonote.rgwadmin4j.{ RgwAdmin, RgwAdminBuilder }

import scala.collection.JavaConverters._
import scala.util.{ Failure, Success, Try }

trait RadosGatewayHandler {

  private val logger = new LoggerHandlerWithId

  protected[this] implicit def system: ActorSystem

  protected[this] def storageS3Settings: StorageS3Settings

  protected[this] val s3Client: AmazonS3

  private[this] case class CredentialsOnCeph(awsAccessKey: AwsAccessKey, awsSecretKey: AwsSecretKey)

  private[this] case class UserOnCeph(userName: UserName, credentials: List[CredentialsOnCeph])

  private[this] lazy val rgwAdmin: RgwAdmin = new RgwAdminBuilder()
    .accessKey(storageS3Settings.storageS3AdminAccesskey)
    .secretKey(storageS3Settings.storageS3AdminSecretkey)
    .endpoint(s"http://${storageS3Settings.storageS3Authority.host.address()}:${storageS3Settings.storageS3Authority.port}/admin")
    .build

  private[this] def createCredentialsOnCeph(userName: UserName, awsAccessKey: AwsAccessKey, awsSecretKey: AwsSecretKey)(implicit id: RequestId): Boolean = {

    Try {
      rgwAdmin.createUser(
        userName.value,
        Map(
          "display-name" -> userName.value,
          "access-key" -> awsAccessKey.value,
          "secret-key" -> awsSecretKey.value
        ).asJava
      )
    } match {
      case Success(user) =>
        logger.info(s"Created on CEPH: " +
          s"UID=${user.getUserId}, " +
          s"AccessKey=${user.getS3Credentials.get(0).getAccessKey}," +
          s"SecretKey=${user.getS3Credentials.get(0).getSecretKey}," +
          s"DisplayName=${user.getDisplayName}")
        true

      case Failure(exc) =>
        logger.error("Unexpected exception during user creation", exc)
        false
    }
  }

  private[this] def updateCredentialsOnCeph(userName: UserName, oldAccessKey: AwsAccessKey, newAccessKey: AwsAccessKey, newSecretKey: AwsSecretKey)(implicit id: RequestId): Boolean = {
    Try {
      rgwAdmin.removeS3Credential(userName.value, oldAccessKey.value)
      rgwAdmin.createS3Credential(userName.value, newAccessKey.value, newSecretKey.value)
    } match {
      case Success(creds) =>
        logger.info(s"Updated on CEPH: " +
          s"UID=${creds.get(0).getUserId}, " +
          s"AccessKey=${creds.get(0).getAccessKey}," +
          s"SecretKey=${creds.get(0).getSecretKey}")
        true

      case Failure(exc) =>
        logger.error("Unexpected exception during user update", exc)
        false
    }
  }

  private[this] def getUserOnCeph(userName: UserName): Option[UserOnCeph] = {

    Try(rgwAdmin.getUserInfo(userName.value)).toOption.flatMap(cuo =>
      if (cuo.isPresent) {
        val cephUser = cuo.get
        Some(UserOnCeph(
          UserName(cephUser.getUserId),
          cephUser.getS3Credentials.asScala.toList.map(c =>
            CredentialsOnCeph(AwsAccessKey(c.getAccessKey), AwsSecretKey(c.getSecretKey))
          )
        ))
      } else None
    )
  }

  
  protected[this] def listAllBuckets: Seq[String] = {
    rgwAdmin.listBucket("").asScala
  }
} 

package blobstore
package s3

import cats.effect.IO
import com.amazonaws.ClientConfiguration
import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials}
import com.amazonaws.client.builder.AwsClientBuilder
import com.amazonaws.regions.Regions
import com.amazonaws.services.s3.transfer.{TransferManager, TransferManagerBuilder}
import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}

class S3StoreTest extends AbstractStoreTest {

  val credentials = new BasicAWSCredentials("my_access_key", "my_secret_key")
  val clientConfiguration = new ClientConfiguration()
  clientConfiguration.setSignerOverride("AWSS3V4SignerType")
  val minioHost: String = Option(System.getenv("BLOBSTORE_MINIO_HOST")).getOrElse("minio-container")
  val minioPort: String = Option(System.getenv("BLOBSTORE_MINIO_PORT")).getOrElse("9000")
  private val client: AmazonS3 = AmazonS3ClientBuilder.standard()
    .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
      s"http://$minioHost:$minioPort", Regions.US_EAST_1.name()))
    .withPathStyleAccessEnabled(true)
    .withClientConfiguration(clientConfiguration)
    .withCredentials(new AWSStaticCredentialsProvider(credentials))
    .build()
  private val transferManager: TransferManager = TransferManagerBuilder.standard()
    .withS3Client(client)
    .build()

  override val store: Store[IO] = new S3Store[IO](transferManager, blocker = blocker)
  override val root: String = "blobstore-test-bucket"

  override def beforeAll(): Unit = {
    super.beforeAll()
    try {
      client.createBucket(root)
    } catch {
      case e: com.amazonaws.services.s3.model.AmazonS3Exception if e.getMessage.contains("BucketAlreadyOwnedByYou") =>
        // noop
    }
    ()
  }


  override def afterAll(): Unit = {
    super.afterAll()

    try {
      client.shutdown()
    } catch {
      case _: Throwable =>
    }
  }
} 

package com.gilt.gfc.aws.s3.akka

import java.util.Date

import com.amazonaws.services.s3.AmazonS3
import com.amazonaws.services.s3.model.{S3Object, S3ObjectSummary}

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future

object AmazonS3Extensions {

  implicit class S3Extensions(val amazonS3: AmazonS3) extends AnyVal {

    import scala.concurrent.blocking

    def mostRecentObject(bucketName: String, prefix: String): Future[Option[S3Object]] = {
      Future {
        mostRecentObjectSummary(bucketName, prefix)
      }.map { objectSummaryOpt =>
        objectSummaryOpt.map { summary =>
          val key = summary.getKey
          amazonS3.getObject(bucketName, key)
        }
      }
    }

    private def mostRecentObjectSummary(bucketName: String, prefix: String): Option[S3ObjectSummary] = {
      import scala.collection.JavaConversions._
      blocking {
        amazonS3.listObjects(bucketName, prefix).getObjectSummaries.toList
      }.sortBy(_.getLastModified)(Ordering[Date].reverse).headOption
    }
  }

}