java.util.TimeZone Scala Examples
The following examples show how to use java.util.TimeZone.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
Example 1
| Source File: KustoCsvSerializationUtils.scala From azure-kusto-spark with Apache License 2.0 | 6 votes |
|
package com.microsoft.kusto.spark.datasink
import java.util.TimeZone
import com.microsoft.kusto.spark.utils.DataTypeMapping
import org.apache.commons.lang3.time.FastDateFormat
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.types.DataTypes._
import org.apache.spark.sql.types.StructType
private[kusto] class KustoCsvSerializationUtils (val schema: StructType, timeZone: String){
private[kusto] val dateFormat = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.SSSXXX", TimeZone.getTimeZone(timeZone))
private[kusto] def convertRow(row: InternalRow) = {
val values = new Array[String](row.numFields)
for (i <- 0 until row.numFields if !row.isNullAt(i))
{
val dataType = schema.fields(i).dataType
values(i) = dataType match {
case DateType => DateTimeUtils.toJavaDate(row.getInt(i)).toString
case TimestampType => dateFormat.format(DateTimeUtils.toJavaTimestamp(row.getLong(i)))
case _ => row.get(i, dataType).toString
}
}
values
}
}
private[kusto] object KustoCsvMapper {
import org.apache.spark.sql.types.StructType
import org.json
def createCsvMapping(schema: StructType): String = {
val csvMapping = new json.JSONArray()
for (i <- 0 until schema.length)
{
val field = schema.apply(i)
val dataType = field.dataType
val mapping = new json.JSONObject()
mapping.put("Name", field.name)
mapping.put("Ordinal", i)
mapping.put("DataType", DataTypeMapping.sparkTypeToKustoTypeMap.getOrElse(dataType, StringType))
csvMapping.put(mapping)
}
csvMapping.toString
}
}
Example 2
| Source File: UIUtilsSuite.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.ui
import java.util.TimeZone
import java.util.concurrent.TimeUnit
import org.scalatest.Matchers
import org.apache.spark.SparkFunSuite
class UIUtilsSuite extends SparkFunSuite with Matchers{
test("shortTimeUnitString") {
assert("ns" === UIUtils.shortTimeUnitString(TimeUnit.NANOSECONDS))
assert("us" === UIUtils.shortTimeUnitString(TimeUnit.MICROSECONDS))
assert("ms" === UIUtils.shortTimeUnitString(TimeUnit.MILLISECONDS))
assert("sec" === UIUtils.shortTimeUnitString(TimeUnit.SECONDS))
assert("min" === UIUtils.shortTimeUnitString(TimeUnit.MINUTES))
assert("hrs" === UIUtils.shortTimeUnitString(TimeUnit.HOURS))
assert("days" === UIUtils.shortTimeUnitString(TimeUnit.DAYS))
}
test("normalizeDuration") {
verifyNormalizedTime(900, TimeUnit.MILLISECONDS, 900)
verifyNormalizedTime(1.0, TimeUnit.SECONDS, 1000)
verifyNormalizedTime(1.0, TimeUnit.MINUTES, 60 * 1000)
verifyNormalizedTime(1.0, TimeUnit.HOURS, 60 * 60 * 1000)
verifyNormalizedTime(1.0, TimeUnit.DAYS, 24 * 60 * 60 * 1000)
}
private def verifyNormalizedTime(
expectedTime: Double, expectedUnit: TimeUnit, input: Long): Unit = {
val (time, unit) = UIUtils.normalizeDuration(input)
time should be (expectedTime +- 1E-6)
unit should be (expectedUnit)
}
test("convertToTimeUnit") {
verifyConvertToTimeUnit(60.0 * 1000 * 1000 * 1000, 60 * 1000, TimeUnit.NANOSECONDS)
verifyConvertToTimeUnit(60.0 * 1000 * 1000, 60 * 1000, TimeUnit.MICROSECONDS)
verifyConvertToTimeUnit(60 * 1000, 60 * 1000, TimeUnit.MILLISECONDS)
verifyConvertToTimeUnit(60, 60 * 1000, TimeUnit.SECONDS)
verifyConvertToTimeUnit(1, 60 * 1000, TimeUnit.MINUTES)
verifyConvertToTimeUnit(1.0 / 60, 60 * 1000, TimeUnit.HOURS)
verifyConvertToTimeUnit(1.0 / 60 / 24, 60 * 1000, TimeUnit.DAYS)
}
private def verifyConvertToTimeUnit(
expectedTime: Double, milliseconds: Long, unit: TimeUnit): Unit = {
val convertedTime = UIUtils.convertToTimeUnit(milliseconds, unit)
convertedTime should be (expectedTime +- 1E-6)
}
test("formatBatchTime") {
val tzForTest = TimeZone.getTimeZone("America/Los_Angeles")
val batchTime = 1431637480452L // Thu May 14 14:04:40 PDT 2015
assert("2015/05/14 14:04:40" === UIUtils.formatBatchTime(batchTime, 1000, timezone = tzForTest))
assert("2015/05/14 14:04:40.452" ===
UIUtils.formatBatchTime(batchTime, 999, timezone = tzForTest))
assert("14:04:40" === UIUtils.formatBatchTime(batchTime, 1000, false, timezone = tzForTest))
assert("14:04:40.452" === UIUtils.formatBatchTime(batchTime, 999, false, timezone = tzForTest))
}
}
Example 3
| Source File: PlayGlobalSettings.scala From play-silhouette-postgres-async-seed with Apache License 2.0 | 5 votes |
|
package utils
import java.util.TimeZone
import jdub.async.Database
import org.joda.time.DateTimeZone
import play.api.{ Application, GlobalSettings }
import services.database.Schema
object PlayGlobalSettings extends GlobalSettings {
override def onStart(app: Application) = {
DateTimeZone.setDefault(DateTimeZone.UTC)
TimeZone.setDefault(TimeZone.getTimeZone("UTC"))
val cnf = play.api.Play.current.configuration
val host = cnf.getString("db.host").getOrElse("127.0.0.1")
val port = 5432
val database = cnf.getString("db.database")
val username = cnf.getString("db.username").getOrElse("silhouette")
val password = cnf.getString("db.password")
Database.open(username, host, port, password, database)
Schema.update()
super.onStart(app)
}
override def onStop(app: Application) = {
Database.close()
super.onStop(app)
}
}
Example 4
| Source File: SimpleDateParam.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.status.api.v1
import java.text.{ParseException, SimpleDateFormat}
import java.util.{Locale, TimeZone}
import javax.ws.rs.WebApplicationException
import javax.ws.rs.core.Response
import javax.ws.rs.core.Response.Status
private[v1] class SimpleDateParam(val originalValue: String) {
val timestamp: Long = {
val format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz", Locale.US)
try {
format.parse(originalValue).getTime()
} catch {
case _: ParseException =>
val gmtDay = new SimpleDateFormat("yyyy-MM-dd", Locale.US)
gmtDay.setTimeZone(TimeZone.getTimeZone("GMT"))
try {
gmtDay.parse(originalValue).getTime()
} catch {
case _: ParseException =>
throw new WebApplicationException(
Response
.status(Status.BAD_REQUEST)
.entity("Couldn't parse date: " + originalValue)
.build()
)
}
}
}
}
Example 5
| Source File: DateUtils.scala From iolap with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.catalyst.util
import java.sql.Date
import java.text.SimpleDateFormat
import java.util.{Calendar, TimeZone}
import org.apache.spark.sql.catalyst.expressions.Cast
object DateUtils {
private val MILLIS_PER_DAY = 86400000
// Java TimeZone has no mention of thread safety. Use thread local instance to be safe.
private val LOCAL_TIMEZONE = new ThreadLocal[TimeZone] {
override protected def initialValue: TimeZone = {
Calendar.getInstance.getTimeZone
}
}
private def javaDateToDays(d: Date): Int = {
millisToDays(d.getTime)
}
// we should use the exact day as Int, for example, (year, month, day) -> day
def millisToDays(millisLocal: Long): Int = {
((millisLocal + LOCAL_TIMEZONE.get().getOffset(millisLocal)) / MILLIS_PER_DAY).toInt
}
private def toMillisSinceEpoch(days: Int): Long = {
val millisUtc = days.toLong * MILLIS_PER_DAY
millisUtc - LOCAL_TIMEZONE.get().getOffset(millisUtc)
}
def fromJavaDate(date: java.sql.Date): Int = {
javaDateToDays(date)
}
def toJavaDate(daysSinceEpoch: Int): java.sql.Date = {
new java.sql.Date(toMillisSinceEpoch(daysSinceEpoch))
}
def toString(days: Int): String = Cast.threadLocalDateFormat.get.format(toJavaDate(days))
def stringToTime(s: String): java.util.Date = {
if (!s.contains('T')) {
// JDBC escape string
if (s.contains(' ')) {
java.sql.Timestamp.valueOf(s)
} else {
java.sql.Date.valueOf(s)
}
} else if (s.endsWith("Z")) {
// this is zero timezone of ISO8601
stringToTime(s.substring(0, s.length - 1) + "GMT-00:00")
} else if (s.indexOf("GMT") == -1) {
// timezone with ISO8601
val inset = "+00.00".length
val s0 = s.substring(0, s.length - inset)
val s1 = s.substring(s.length - inset, s.length)
if (s0.substring(s0.lastIndexOf(':')).contains('.')) {
stringToTime(s0 + "GMT" + s1)
} else {
stringToTime(s0 + ".0GMT" + s1)
}
} else {
// ISO8601 with GMT insert
val ISO8601GMT: SimpleDateFormat = new SimpleDateFormat( "yyyy-MM-dd'T'HH:mm:ss.SSSz" )
ISO8601GMT.parse(s)
}
}
}
Example 6
| Source File: UIUtilsSuite.scala From iolap with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.ui
import java.util.TimeZone
import java.util.concurrent.TimeUnit
import org.scalatest.Matchers
import org.apache.spark.SparkFunSuite
class UIUtilsSuite extends SparkFunSuite with Matchers{
test("shortTimeUnitString") {
assert("ns" === UIUtils.shortTimeUnitString(TimeUnit.NANOSECONDS))
assert("us" === UIUtils.shortTimeUnitString(TimeUnit.MICROSECONDS))
assert("ms" === UIUtils.shortTimeUnitString(TimeUnit.MILLISECONDS))
assert("sec" === UIUtils.shortTimeUnitString(TimeUnit.SECONDS))
assert("min" === UIUtils.shortTimeUnitString(TimeUnit.MINUTES))
assert("hrs" === UIUtils.shortTimeUnitString(TimeUnit.HOURS))
assert("days" === UIUtils.shortTimeUnitString(TimeUnit.DAYS))
}
test("normalizeDuration") {
verifyNormalizedTime(900, TimeUnit.MILLISECONDS, 900)
verifyNormalizedTime(1.0, TimeUnit.SECONDS, 1000)
verifyNormalizedTime(1.0, TimeUnit.MINUTES, 60 * 1000)
verifyNormalizedTime(1.0, TimeUnit.HOURS, 60 * 60 * 1000)
verifyNormalizedTime(1.0, TimeUnit.DAYS, 24 * 60 * 60 * 1000)
}
private def verifyNormalizedTime(
expectedTime: Double, expectedUnit: TimeUnit, input: Long): Unit = {
val (time, unit) = UIUtils.normalizeDuration(input)
time should be (expectedTime +- 1E-6)
unit should be (expectedUnit)
}
test("convertToTimeUnit") {
verifyConvertToTimeUnit(60.0 * 1000 * 1000 * 1000, 60 * 1000, TimeUnit.NANOSECONDS)
verifyConvertToTimeUnit(60.0 * 1000 * 1000, 60 * 1000, TimeUnit.MICROSECONDS)
verifyConvertToTimeUnit(60 * 1000, 60 * 1000, TimeUnit.MILLISECONDS)
verifyConvertToTimeUnit(60, 60 * 1000, TimeUnit.SECONDS)
verifyConvertToTimeUnit(1, 60 * 1000, TimeUnit.MINUTES)
verifyConvertToTimeUnit(1.0 / 60, 60 * 1000, TimeUnit.HOURS)
verifyConvertToTimeUnit(1.0 / 60 / 24, 60 * 1000, TimeUnit.DAYS)
}
private def verifyConvertToTimeUnit(
expectedTime: Double, milliseconds: Long, unit: TimeUnit): Unit = {
val convertedTime = UIUtils.convertToTimeUnit(milliseconds, unit)
convertedTime should be (expectedTime +- 1E-6)
}
test("formatBatchTime") {
val tzForTest = TimeZone.getTimeZone("America/Los_Angeles")
val batchTime = 1431637480452L // Thu May 14 14:04:40 PDT 2015
assert("2015/05/14 14:04:40" === UIUtils.formatBatchTime(batchTime, 1000, timezone = tzForTest))
assert("2015/05/14 14:04:40.452" ===
UIUtils.formatBatchTime(batchTime, 999, timezone = tzForTest))
assert("14:04:40" === UIUtils.formatBatchTime(batchTime, 1000, false, timezone = tzForTest))
assert("14:04:40.452" === UIUtils.formatBatchTime(batchTime, 999, false, timezone = tzForTest))
}
}
Example 7
| Source File: SimpleDateParam.scala From iolap with Apache License 2.0 | 5 votes |
|
package org.apache.spark.status.api.v1
import java.text.SimpleDateFormat
import java.util.TimeZone
import javax.ws.rs.WebApplicationException
import javax.ws.rs.core.Response
import javax.ws.rs.core.Response.Status
import scala.util.Try
private[v1] class SimpleDateParam(val originalValue: String) {
val timestamp: Long = {
SimpleDateParam.formats.collectFirst {
case fmt if Try(fmt.parse(originalValue)).isSuccess =>
fmt.parse(originalValue).getTime()
}.getOrElse(
throw new WebApplicationException(
Response
.status(Status.BAD_REQUEST)
.entity("Couldn't parse date: " + originalValue)
.build()
)
)
}
}
private[v1] object SimpleDateParam {
val formats: Seq[SimpleDateFormat] = {
val gmtDay = new SimpleDateFormat("yyyy-MM-dd")
gmtDay.setTimeZone(TimeZone.getTimeZone("GMT"))
Seq(
new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz"),
gmtDay
)
}
}
Example 8
| Source File: FinagleMysqlContextConfig.scala From quill with Apache License 2.0 | 5 votes |
|
package io.getquill
import java.util.TimeZone
import com.twitter.conversions.DurationOps._
import com.twitter.finagle.Mysql
import com.twitter.finagle.client.DefaultPool
import com.twitter.util.Try
import com.typesafe.config.Config
case class FinagleMysqlContextConfig(config: Config) {
def injectionTimeZone = Try(TimeZone.getTimeZone(config.getString("timezone.injection"))).getOrElse(TimeZone.getDefault)
def extractionTimeZone = Try(TimeZone.getTimeZone(config.getString("timezone.extraction"))).getOrElse(TimeZone.getDefault)
def user = config.getString("user")
def password = Try(config.getString("password")).getOrElse(null)
def database = config.getString("database")
def dest = config.getString("dest")
def lowWatermark = Try(config.getInt("pool.watermark.low")).getOrElse(0)
def highWatermark = Try(config.getInt("pool.watermark.high")).getOrElse(10)
def idleTime = Try(config.getLong("pool.idleTime")).getOrElse(5L)
def bufferSize = Try(config.getInt("pool.bufferSize")).getOrElse(0)
def maxWaiters = Try(config.getInt("pool.maxWaiters")).getOrElse(Int.MaxValue)
def maxPrepareStatements = Try(config.getInt("maxPrepareStatements")).getOrElse(20)
def connectTimeout = Try(config.getLong("connectTimeout")).getOrElse(1L)
def noFailFast = Try(config.getBoolean("noFailFast")).getOrElse(false)
def client = {
var client = Mysql.client
.withCredentials(user, password)
.withDatabase(database)
.withMaxConcurrentPrepareStatements(maxPrepareStatements)
.withTransport
.connectTimeout(connectTimeout.seconds)
.configured(DefaultPool.Param(
low = lowWatermark, high = highWatermark,
idleTime = idleTime.seconds,
bufferSize = bufferSize,
maxWaiters = maxWaiters
))
if (noFailFast) {
client = client.withSessionQualifier.noFailFast
}
client.newRichClient(dest)
}
}
Example 9
| Source File: Encoders.scala From quill with Apache License 2.0 | 5 votes |
|
package io.getquill.context.jdbc
import java.sql.{ Date, Timestamp, Types }
import java.time.{ LocalDate, LocalDateTime }
import java.util.{ Calendar, TimeZone }
import java.{ sql, util }
trait Encoders {
this: JdbcContextBase[_, _] =>
type Encoder[T] = JdbcEncoder[T]
protected val dateTimeZone = TimeZone.getDefault
case class JdbcEncoder[T](sqlType: Int, encoder: BaseEncoder[T]) extends BaseEncoder[T] {
override def apply(index: Index, value: T, row: PrepareRow) =
encoder(index + 1, value, row)
}
def encoder[T](sqlType: Int, f: (Index, T, PrepareRow) => Unit): Encoder[T] =
JdbcEncoder(sqlType, (index: Index, value: T, row: PrepareRow) => {
f(index, value, row)
row
})
def encoder[T](sqlType: Int, f: PrepareRow => (Index, T) => Unit): Encoder[T] =
encoder(sqlType, (index: Index, value: T, row: PrepareRow) => f(row)(index, value))
implicit def mappedEncoder[I, O](implicit mapped: MappedEncoding[I, O], e: Encoder[O]): Encoder[I] =
JdbcEncoder(e.sqlType, mappedBaseEncoder(mapped, e.encoder))
private[this] val nullEncoder: Encoder[Int] = encoder(Types.INTEGER, _.setNull)
implicit def optionEncoder[T](implicit d: Encoder[T]): Encoder[Option[T]] =
JdbcEncoder(
d.sqlType,
(index, value, row) =>
value match {
case Some(v) => d.encoder(index, v, row)
case None => nullEncoder.encoder(index, d.sqlType, row)
}
)
implicit val stringEncoder: Encoder[String] = encoder(Types.VARCHAR, _.setString)
implicit val bigDecimalEncoder: Encoder[BigDecimal] =
encoder(Types.NUMERIC, (index, value, row) => row.setBigDecimal(index, value.bigDecimal))
implicit val byteEncoder: Encoder[Byte] = encoder(Types.TINYINT, _.setByte)
implicit val shortEncoder: Encoder[Short] = encoder(Types.SMALLINT, _.setShort)
implicit val intEncoder: Encoder[Int] = encoder(Types.INTEGER, _.setInt)
implicit val longEncoder: Encoder[Long] = encoder(Types.BIGINT, _.setLong)
implicit val floatEncoder: Encoder[Float] = encoder(Types.FLOAT, _.setFloat)
implicit val doubleEncoder: Encoder[Double] = encoder(Types.DOUBLE, _.setDouble)
implicit val byteArrayEncoder: Encoder[Array[Byte]] = encoder(Types.VARBINARY, _.setBytes)
implicit val dateEncoder: Encoder[util.Date] =
encoder(Types.TIMESTAMP, (index, value, row) =>
row.setTimestamp(index, new sql.Timestamp(value.getTime), Calendar.getInstance(dateTimeZone)))
implicit val localDateEncoder: Encoder[LocalDate] =
encoder(Types.DATE, (index, value, row) =>
row.setDate(index, Date.valueOf(value), Calendar.getInstance(dateTimeZone)))
implicit val localDateTimeEncoder: Encoder[LocalDateTime] =
encoder(Types.TIMESTAMP, (index, value, row) =>
row.setTimestamp(index, Timestamp.valueOf(value), Calendar.getInstance(dateTimeZone)))
}
Example 10
| Source File: DataFrameReportPerformanceSpec.scala From seahorse with Apache License 2.0 | 5 votes |
|
package ai.deepsense.deeplang.doperables.dataframe
import java.sql.Timestamp
import java.text.{DateFormat, SimpleDateFormat}
import java.util.TimeZone
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DoubleType, StructField, StructType, TimestampType}
import org.scalatest.{BeforeAndAfter, Ignore}
import ai.deepsense.commons.utils.{DoubleUtils, Logging}
import ai.deepsense.deeplang.{TestFiles, DeeplangIntegTestSupport}
// It's ignored because it does not have got assertions, it only prints report generation time.
@Ignore
class DataFrameReportPerformanceSpec
extends DeeplangIntegTestSupport
with BeforeAndAfter
with TestFiles
with Logging {
val testFile = absoluteTestsDirPath.pathWithoutScheme + "/demand_without_header.csv"
"DataFrame" should {
"generate report" when {
"DataFrame has 17K of rows" in {
val numberOfTries = 10
var results: Seq[Double] = Seq()
for (i <- 1 to numberOfTries) {
val dataFrame: DataFrame = demandDataFrame()
val start = System.nanoTime()
val report = dataFrame.report()
val end = System.nanoTime()
val time1: Double = (end - start).toDouble / 1000000000.0
results = results :+ time1
logger.debug("Report generation time: {}", DoubleUtils.double2String(time1))
}
logger.debug(
"Mean report generation time: {}",
DoubleUtils.double2String(results.fold(0D)(_ + _) / numberOfTries.toDouble))
}
}
}
private def demandDataFrame(): DataFrame = {
val rddString: RDD[String] = executionContext.sparkContext.textFile(testFile)
val data: RDD[Row] = rddString.map(DataFrameHelpers.demandString2Row)
executionContext.dataFrameBuilder.buildDataFrame(demandSchema, data)
}
private def demandSchema: StructType = StructType(Seq(
StructField("datetime", TimestampType),
StructField("log_count", DoubleType),
StructField("workingday", DoubleType),
StructField("holiday", DoubleType),
StructField("season2", DoubleType),
StructField("season3", DoubleType),
StructField("season4", DoubleType)))
private def timestamp(s: String): Timestamp = {
val format: DateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
format.setTimeZone(TimeZone.getTimeZone("UTC"))
new Timestamp(format.parse(s).getTime)
}
}
private object DataFrameHelpers {
def demandString2Row(s: String): Row = {
val split = s.split(",")
Row(
timestamp(split(0)),
split(1).toDouble,
split(2).toDouble,
split(3).toDouble,
split(4).toDouble,
split(5).toDouble,
split(6).toDouble
)
}
private def timestamp(s: String): Timestamp = {
val format: DateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
format.setTimeZone(TimeZone.getTimeZone("UTC"))
new Timestamp(format.parse(s).getTime)
}
}
Example 11
| Source File: SimpleDateParam.scala From spark1.52 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.status.api.v1
import java.text.{ParseException, SimpleDateFormat}
import java.util.TimeZone
import javax.ws.rs.WebApplicationException
import javax.ws.rs.core.Response
import javax.ws.rs.core.Response.Status
private[v1] class SimpleDateParam(val originalValue: String) {
val timestamp: Long = {
val format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz")
try {
format.parse(originalValue).getTime()
} catch {
case _: ParseException =>
val gmtDay = new SimpleDateFormat("yyyy-MM-dd")
gmtDay.setTimeZone(TimeZone.getTimeZone("GMT"))
try {
gmtDay.parse(originalValue).getTime()
} catch {
case _: ParseException =>
throw new WebApplicationException(
Response
.status(Status.BAD_REQUEST)
.entity("Couldn't parse date: " + originalValue)
.build()
)
}
}
}
}
Example 12
| Source File: JSONOptions.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.catalyst.json
import java.util.{Locale, TimeZone}
import com.fasterxml.jackson.core.{JsonFactory, JsonParser}
import org.apache.commons.lang3.time.FastDateFormat
import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.util._
def setJacksonOptions(factory: JsonFactory): Unit = {
factory.configure(JsonParser.Feature.ALLOW_COMMENTS, allowComments)
factory.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, allowUnquotedFieldNames)
factory.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, allowSingleQuotes)
factory.configure(JsonParser.Feature.ALLOW_NUMERIC_LEADING_ZEROS, allowNumericLeadingZeros)
factory.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS, allowNonNumericNumbers)
factory.configure(JsonParser.Feature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER,
allowBackslashEscapingAnyCharacter)
factory.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, allowUnquotedControlChars)
}
}
Example 13
| Source File: DateTimeTestUtils.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.catalyst.util
import java.util.TimeZone
object DateTimeTestUtils {
val ALL_TIMEZONES: Seq[TimeZone] = TimeZone.getAvailableIDs.toSeq.map(TimeZone.getTimeZone)
def withDefaultTimeZone[T](newDefaultTimeZone: TimeZone)(block: => T): T = {
val originalDefaultTimeZone = TimeZone.getDefault
try {
DateTimeUtils.resetThreadLocals()
TimeZone.setDefault(newDefaultTimeZone)
block
} finally {
TimeZone.setDefault(originalDefaultTimeZone)
DateTimeUtils.resetThreadLocals()
}
}
}
Example 14
| Source File: UIUtilsSuite.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.ui
import java.util.TimeZone
import java.util.concurrent.TimeUnit
import org.scalatest.Matchers
import org.apache.spark.SparkFunSuite
class UIUtilsSuite extends SparkFunSuite with Matchers{
test("shortTimeUnitString") {
assert("ns" === UIUtils.shortTimeUnitString(TimeUnit.NANOSECONDS))
assert("us" === UIUtils.shortTimeUnitString(TimeUnit.MICROSECONDS))
assert("ms" === UIUtils.shortTimeUnitString(TimeUnit.MILLISECONDS))
assert("sec" === UIUtils.shortTimeUnitString(TimeUnit.SECONDS))
assert("min" === UIUtils.shortTimeUnitString(TimeUnit.MINUTES))
assert("hrs" === UIUtils.shortTimeUnitString(TimeUnit.HOURS))
assert("days" === UIUtils.shortTimeUnitString(TimeUnit.DAYS))
}
test("normalizeDuration") {
verifyNormalizedTime(900, TimeUnit.MILLISECONDS, 900)
verifyNormalizedTime(1.0, TimeUnit.SECONDS, 1000)
verifyNormalizedTime(1.0, TimeUnit.MINUTES, 60 * 1000)
verifyNormalizedTime(1.0, TimeUnit.HOURS, 60 * 60 * 1000)
verifyNormalizedTime(1.0, TimeUnit.DAYS, 24 * 60 * 60 * 1000)
}
private def verifyNormalizedTime(
expectedTime: Double, expectedUnit: TimeUnit, input: Long): Unit = {
val (time, unit) = UIUtils.normalizeDuration(input)
time should be (expectedTime +- 1E-6)
unit should be (expectedUnit)
}
test("convertToTimeUnit") {
verifyConvertToTimeUnit(60.0 * 1000 * 1000 * 1000, 60 * 1000, TimeUnit.NANOSECONDS)
verifyConvertToTimeUnit(60.0 * 1000 * 1000, 60 * 1000, TimeUnit.MICROSECONDS)
verifyConvertToTimeUnit(60 * 1000, 60 * 1000, TimeUnit.MILLISECONDS)
verifyConvertToTimeUnit(60, 60 * 1000, TimeUnit.SECONDS)
verifyConvertToTimeUnit(1, 60 * 1000, TimeUnit.MINUTES)
verifyConvertToTimeUnit(1.0 / 60, 60 * 1000, TimeUnit.HOURS)
verifyConvertToTimeUnit(1.0 / 60 / 24, 60 * 1000, TimeUnit.DAYS)
}
private def verifyConvertToTimeUnit(
expectedTime: Double, milliseconds: Long, unit: TimeUnit): Unit = {
val convertedTime = UIUtils.convertToTimeUnit(milliseconds, unit)
convertedTime should be (expectedTime +- 1E-6)
}
test("formatBatchTime") {
val tzForTest = TimeZone.getTimeZone("America/Los_Angeles")
val batchTime = 1431637480452L // Thu May 14 14:04:40 PDT 2015
assert("2015/05/14 14:04:40" === UIUtils.formatBatchTime(batchTime, 1000, timezone = tzForTest))
assert("2015/05/14 14:04:40.452" ===
UIUtils.formatBatchTime(batchTime, 999, timezone = tzForTest))
assert("14:04:40" === UIUtils.formatBatchTime(batchTime, 1000, false, timezone = tzForTest))
assert("14:04:40.452" === UIUtils.formatBatchTime(batchTime, 999, false, timezone = tzForTest))
}
}
Example 15
| Source File: SimpleDateParam.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.status.api.v1
import java.text.{ParseException, SimpleDateFormat}
import java.util.{Locale, TimeZone}
import javax.ws.rs.WebApplicationException
import javax.ws.rs.core.Response
import javax.ws.rs.core.Response.Status
private[v1] class SimpleDateParam(val originalValue: String) {
val timestamp: Long = {
val format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz", Locale.US)
try {
format.parse(originalValue).getTime()
} catch {
case _: ParseException =>
val gmtDay = new SimpleDateFormat("yyyy-MM-dd", Locale.US)
gmtDay.setTimeZone(TimeZone.getTimeZone("GMT"))
try {
gmtDay.parse(originalValue).getTime()
} catch {
case _: ParseException =>
throw new WebApplicationException(
Response
.status(Status.BAD_REQUEST)
.entity("Couldn't parse date: " + originalValue)
.build()
)
}
}
}
}
Example 16
| Source File: UIUtilsSuite.scala From BigDatalog with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.ui
import java.util.TimeZone
import java.util.concurrent.TimeUnit
import org.scalatest.Matchers
import org.apache.spark.SparkFunSuite
class UIUtilsSuite extends SparkFunSuite with Matchers{
test("shortTimeUnitString") {
assert("ns" === UIUtils.shortTimeUnitString(TimeUnit.NANOSECONDS))
assert("us" === UIUtils.shortTimeUnitString(TimeUnit.MICROSECONDS))
assert("ms" === UIUtils.shortTimeUnitString(TimeUnit.MILLISECONDS))
assert("sec" === UIUtils.shortTimeUnitString(TimeUnit.SECONDS))
assert("min" === UIUtils.shortTimeUnitString(TimeUnit.MINUTES))
assert("hrs" === UIUtils.shortTimeUnitString(TimeUnit.HOURS))
assert("days" === UIUtils.shortTimeUnitString(TimeUnit.DAYS))
}
test("normalizeDuration") {
verifyNormalizedTime(900, TimeUnit.MILLISECONDS, 900)
verifyNormalizedTime(1.0, TimeUnit.SECONDS, 1000)
verifyNormalizedTime(1.0, TimeUnit.MINUTES, 60 * 1000)
verifyNormalizedTime(1.0, TimeUnit.HOURS, 60 * 60 * 1000)
verifyNormalizedTime(1.0, TimeUnit.DAYS, 24 * 60 * 60 * 1000)
}
private def verifyNormalizedTime(
expectedTime: Double, expectedUnit: TimeUnit, input: Long): Unit = {
val (time, unit) = UIUtils.normalizeDuration(input)
time should be (expectedTime +- 1E-6)
unit should be (expectedUnit)
}
test("convertToTimeUnit") {
verifyConvertToTimeUnit(60.0 * 1000 * 1000 * 1000, 60 * 1000, TimeUnit.NANOSECONDS)
verifyConvertToTimeUnit(60.0 * 1000 * 1000, 60 * 1000, TimeUnit.MICROSECONDS)
verifyConvertToTimeUnit(60 * 1000, 60 * 1000, TimeUnit.MILLISECONDS)
verifyConvertToTimeUnit(60, 60 * 1000, TimeUnit.SECONDS)
verifyConvertToTimeUnit(1, 60 * 1000, TimeUnit.MINUTES)
verifyConvertToTimeUnit(1.0 / 60, 60 * 1000, TimeUnit.HOURS)
verifyConvertToTimeUnit(1.0 / 60 / 24, 60 * 1000, TimeUnit.DAYS)
}
private def verifyConvertToTimeUnit(
expectedTime: Double, milliseconds: Long, unit: TimeUnit): Unit = {
val convertedTime = UIUtils.convertToTimeUnit(milliseconds, unit)
convertedTime should be (expectedTime +- 1E-6)
}
test("formatBatchTime") {
val tzForTest = TimeZone.getTimeZone("America/Los_Angeles")
val batchTime = 1431637480452L // Thu May 14 14:04:40 PDT 2015
assert("2015/05/14 14:04:40" === UIUtils.formatBatchTime(batchTime, 1000, timezone = tzForTest))
assert("2015/05/14 14:04:40.452" ===
UIUtils.formatBatchTime(batchTime, 999, timezone = tzForTest))
assert("14:04:40" === UIUtils.formatBatchTime(batchTime, 1000, false, timezone = tzForTest))
assert("14:04:40.452" === UIUtils.formatBatchTime(batchTime, 999, false, timezone = tzForTest))
}
}
Example 17
| Source File: SimpleDateParam.scala From BigDatalog with Apache License 2.0 | 5 votes |
|
package org.apache.spark.status.api.v1
import java.text.{ParseException, SimpleDateFormat}
import java.util.TimeZone
import javax.ws.rs.WebApplicationException
import javax.ws.rs.core.Response
import javax.ws.rs.core.Response.Status
private[v1] class SimpleDateParam(val originalValue: String) {
val timestamp: Long = {
val format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz")
try {
format.parse(originalValue).getTime()
} catch {
case _: ParseException =>
val gmtDay = new SimpleDateFormat("yyyy-MM-dd")
gmtDay.setTimeZone(TimeZone.getTimeZone("GMT"))
try {
gmtDay.parse(originalValue).getTime()
} catch {
case _: ParseException =>
throw new WebApplicationException(
Response
.status(Status.BAD_REQUEST)
.entity("Couldn't parse date: " + originalValue)
.build()
)
}
}
}
}
Example 18
| Source File: JVMCrossDate.scala From perfolation with MIT License | 5 votes |
|
package perfolation
import java.util.{Calendar, TimeZone}
class JVMCrossDate(val calendar: java.util.Calendar) extends AnyVal with CrossDate {
override def milliseconds: Long = calendar.getTimeInMillis
override def hour24: Int = calendar.get(Calendar.HOUR_OF_DAY)
override def minuteOfHour: Int = calendar.get(Calendar.MINUTE)
override def secondOfMinute: Int = calendar.get(Calendar.SECOND)
override def milliOfSecond: Int = calendar.get(Calendar.MILLISECOND)
override def isAM: Boolean = calendar.get(Calendar.AM_PM) == 0
override def timeZoneOffsetMillis: Int = calendar.get(Calendar.ZONE_OFFSET) + calendar.get(Calendar.DST_OFFSET)
override def year: Int = calendar.get(Calendar.YEAR)
override def month: Int = calendar.get(Calendar.MONTH)
override def dayOfWeek: Int = calendar.get(Calendar.DAY_OF_WEEK)
override def dayOfMonth: Int = calendar.get(Calendar.DAY_OF_MONTH)
override def dayOfYear: Int = calendar.get(Calendar.DAY_OF_YEAR) - 1
override def timeZone: String = calendar.getTimeZone.getDisplayName(calendar.get(Calendar.DST_OFFSET) != 0, TimeZone.SHORT)
}
Example 19
| Source File: KeysExtractor.scala From stream-reactor with Apache License 2.0 | 5 votes |
|
package com.datamountaineer.streamreactor.connect.azure.documentdb.sink
import java.text.SimpleDateFormat
import java.util.TimeZone
import org.apache.kafka.common.config.ConfigException
import org.apache.kafka.connect.data._
import org.json4s.JsonAST._
object KeysExtractor {
private val ISO_DATE_FORMAT: SimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
private val TIME_FORMAT: SimpleDateFormat = new SimpleDateFormat("HH:mm:ss.SSSZ")
ISO_DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("UTC"))
def fromStruct(struct: Struct, keys: Set[String]): Set[(String, Any)] = {
keys.map { key =>
val schema = struct.schema().field(key).schema()
val value = struct.get(key)
val v = schema.`type`() match {
case Schema.Type.INT32 =>
if (schema != null && Date.LOGICAL_NAME == schema.name) ISO_DATE_FORMAT.format(Date.toLogical(schema, value.asInstanceOf[Int]))
else if (schema != null && Time.LOGICAL_NAME == schema.name) TIME_FORMAT.format(Time.toLogical(schema, value.asInstanceOf[Int]))
else value
case Schema.Type.INT64 =>
if (Timestamp.LOGICAL_NAME == schema.name) Timestamp.fromLogical(schema, value.asInstanceOf[(java.util.Date)])
else value
case Schema.Type.STRING => value.asInstanceOf[CharSequence].toString
case Schema.Type.BYTES =>
if (Decimal.LOGICAL_NAME == schema.name) value.asInstanceOf[BigDecimal].toDouble
else throw new ConfigException(s"Schema.Type.BYTES is not supported for $key.")
case Schema.Type.ARRAY =>
throw new ConfigException(s"Schema.Type.ARRAY is not supported for $key.")
case Schema.Type.MAP => throw new ConfigException(s"Schema.Type.MAP is not supported for $key.")
case Schema.Type.STRUCT => throw new ConfigException(s"Schema.Type.STRUCT is not supported for $key.")
case other => throw new ConfigException(s"$other is not supported for $key.")
}
key -> v
}
}
def fromMap(map: java.util.Map[String, Any], keys: Set[String]): Set[(String, Any)] = {
keys.map { key =>
if (!map.containsKey(key)) throw new ConfigException(s"The key $key can't be found")
val value = map.get(key) match {
case t: String => t
case t: Boolean => t
case t: Int => t
case t: Long => t
case t: Double => t
case t: BigInt => t.toLong
case t: BigDecimal => t.toDouble
case other => throw new ConfigException(s"The key $key is not supported for type ${Option(other).map(_.getClass.getName).getOrElse("NULL")}")
}
key -> value
}
}
def fromJson(jvalue: JValue, keys: Set[String]): List[(String, Any)] = {
jvalue match {
case JObject(children) =>
children.collect {
case JField(name, value) if keys.contains(name) =>
val v = value match {
case JBool(b) => b
case JDecimal(d) => d.toDouble
case JDouble(d) => d
case JInt(i) => i.toLong
case JLong(l) => l
case JString(s) => s
case other => throw new ConfigException(s"Field $name is not handled as a key (${other.getClass}). it needs to be a int, long, string, double or decimal")
}
name -> v
}
case other => throw new ConfigException(s"${other.getClass} is not supported")
}
}
}
Example 20
| Source File: StructFieldsExtractorBytes.scala From stream-reactor with Apache License 2.0 | 5 votes |
|
package com.datamountaineer.streamreactor.connect.hbase
import java.text.SimpleDateFormat
import java.util.TimeZone
import com.datamountaineer.streamreactor.connect.hbase.BytesHelper._
import com.typesafe.scalalogging.StrictLogging
import org.apache.kafka.connect.data._
import scala.collection.JavaConverters._
trait FieldsValuesExtractor {
def get(struct: Struct): Seq[(String, Array[Byte])]
}
case class StructFieldsExtractorBytes(includeAllFields: Boolean, fieldsAliasMap: Map[String, String]) extends FieldsValuesExtractor with StrictLogging {
def get(struct: Struct): Seq[(String, Array[Byte])] = {
val schema = struct.schema()
val fields: Seq[Field] = if (includeAllFields) {
schema.fields().asScala
}
else {
val selectedFields = schema.fields().asScala.filter(f => fieldsAliasMap.contains(f.name()))
val diffSet = fieldsAliasMap.keySet.diff(selectedFields.map(_.name()).toSet)
if (diffSet.nonEmpty) {
val errMsg = s"Following columns ${diffSet.mkString(",")} have not been found. Available columns:${fieldsAliasMap.keys.mkString(",")}"
logger.error(errMsg)
sys.error(errMsg)
}
selectedFields
}
val fieldsAndValues = fields.flatMap(field =>
getFieldBytes(field, struct).map(bytes => fieldsAliasMap.getOrElse(field.name(), field.name()) -> bytes))
fieldsAndValues
}
private def getFieldBytes(field: Field, struct: Struct): Option[Array[Byte]] = {
Option(struct.get(field))
.map { value =>
Option(field.schema().name()).collect {
case Decimal.LOGICAL_NAME =>
value.asInstanceOf[Any] match {
case _:java.math.BigDecimal => value.fromBigDecimal()
case arr: Array[Byte] => Decimal.toLogical(field.schema, arr).asInstanceOf[Any].fromBigDecimal()
case _ => throw new IllegalArgumentException(s"${field.name()} is not handled for value:$value")
}
case Time.LOGICAL_NAME =>
value.asInstanceOf[Any] match {
case i: Int => StructFieldsExtractorBytes.TimeFormat.format(Time.toLogical(field.schema, i)).asInstanceOf[Any].fromString()
case d:java.util.Date => StructFieldsExtractorBytes.TimeFormat.format(d).asInstanceOf[Any].fromString()
case _ => throw new IllegalArgumentException(s"${field.name()} is not handled for value:$value")
}
case Timestamp.LOGICAL_NAME =>
value.asInstanceOf[Any] match {
case d:java.util.Date => StructFieldsExtractorBytes.DateFormat.format(d).asInstanceOf[Any].fromString()
case l: Long => StructFieldsExtractorBytes.DateFormat.format(Timestamp.toLogical(field.schema, l)).asInstanceOf[Any].fromString()
case _ => throw new IllegalArgumentException(s"${field.name()} is not handled for value:$value")
}
}.getOrElse {
field.schema().`type`() match {
case Schema.Type.BOOLEAN => value.fromBoolean()
case Schema.Type.BYTES => value.fromBytes()
case Schema.Type.FLOAT32 => value.fromFloat()
case Schema.Type.FLOAT64 => value.fromDouble()
case Schema.Type.INT8 => value.fromByte()
case Schema.Type.INT16 => value.fromShort()
case Schema.Type.INT32 => value.fromInt()
case Schema.Type.INT64 => value.fromLong()
case Schema.Type.STRING => value.fromString()
case other => sys.error(s"$other is not a recognized schema!")
}
}
}
}
}
object StructFieldsExtractorBytes {
val DateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
val TimeFormat = new SimpleDateFormat("HH:mm:ss.SSSZ")
DateFormat.setTimeZone(TimeZone.getTimeZone("UTC"))
}
Example 21
| Source File: StructFieldsExtractor.scala From stream-reactor with Apache License 2.0 | 5 votes |
|
package com.datamountaineer.streamreactor.connect.voltdb
import java.text.SimpleDateFormat
import java.util.TimeZone
import com.typesafe.scalalogging.StrictLogging
import org.apache.kafka.connect.data.{Field, Struct, _}
import scala.collection.JavaConverters._
trait FieldsValuesExtractor {
def get(struct: Struct): Map[String, Any]
}
case class StructFieldsExtractor(targetTable: String,
includeAllFields: Boolean,
fieldsAliasMap: Map[String, String],
isUpsert: Boolean = false) extends FieldsValuesExtractor with StrictLogging {
require(targetTable != null && targetTable.trim.length > 0)
def get(struct: Struct): Map[String, Any] = {
val schema = struct.schema()
val fields: Seq[Field] = {
if (includeAllFields) {
schema.fields().asScala
} else {
val selectedFields = schema.fields().asScala.filter(f => fieldsAliasMap.contains(f.name()))
val diffSet = fieldsAliasMap.keySet.diff(selectedFields.map(_.name()).toSet)
if (diffSet.nonEmpty) {
val errMsg = s"Following columns ${diffSet.mkString(",")} have not been found. Available columns:${fieldsAliasMap.keys.mkString(",")}"
logger.error(errMsg)
sys.error(errMsg)
}
selectedFields
}
}
//need to select all fields including null. the stored proc needs a fixed set of params
fields.map { field =>
val schema = field.schema()
val value = Option(struct.get(field))
.map { value =>
//handle specific schema
schema.name() match {
case Decimal.LOGICAL_NAME =>
value.asInstanceOf[Any] match {
case _:java.math.BigDecimal => value
case arr: Array[Byte] => Decimal.toLogical(schema, arr)
case _ => throw new IllegalArgumentException(s"${field.name()} is not handled for value:$value")
}
case Time.LOGICAL_NAME =>
value.asInstanceOf[Any] match {
case i: Int => StructFieldsExtractor.TimeFormat.format(Time.toLogical(schema, i))
case d:java.util.Date => StructFieldsExtractor.TimeFormat.format(d)
case _ => throw new IllegalArgumentException(s"${field.name()} is not handled for value:$value")
}
case Timestamp.LOGICAL_NAME =>
value.asInstanceOf[Any] match {
case d:java.util.Date => StructFieldsExtractor.DateFormat.format(d)
case l: Long => StructFieldsExtractor.DateFormat.format(Timestamp.toLogical(schema, l))
case _ => throw new IllegalArgumentException(s"${field.name()} is not handled for value:$value")
}
case _ => value
}
}.orNull
fieldsAliasMap.getOrElse(field.name(), field.name()) -> value
}.toMap
}
}
object StructFieldsExtractor {
val DateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
val TimeFormat: SimpleDateFormat = new SimpleDateFormat("HH:mm:ss.SSSZ")
DateFormat.setTimeZone(TimeZone.getTimeZone("UTC"))
}
Example 22
| Source File: TimeHelper.scala From ez-framework with Apache License 2.0 | 5 votes |
|
package com.ecfront.ez.framework.core.helper
import java.text.SimpleDateFormat
import java.time.ZonedDateTime
import java.util.{Calendar, Date, TimeZone}
object TimeHelper {
val msf = new SimpleDateFormat("yyyyMMddHHmmssSSS")
val sf = new SimpleDateFormat("yyyyMMddHHmmss")
val mf = new SimpleDateFormat("yyyyMMddHHmm")
val hf = new SimpleDateFormat("yyyyMMddHH")
val df = new SimpleDateFormat("yyyyMMdd")
val Mf = new SimpleDateFormat("yyyyMM")
val yf = new SimpleDateFormat("yyyy")
val yyyy_MM_dd_HH_mm_ss_SSS = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS")
val yyyy_MM_dd_HH_mm_ss = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
val yyyy_MM_dd = new SimpleDateFormat("yyyy-MM-dd")
def dateOffset(offsetValue: Int, offsetUnit: Int, currentTime: Long): Long = {
val format = currentTime.toString.length match {
case 8 => df
case 10 => hf
case 12 => mf
case 14 => sf
case 17 => msf
}
val calendar = Calendar.getInstance()
calendar.setTime(format.parse(currentTime + ""))
calendar.add(offsetUnit, offsetValue)
format.format(calendar.getTime).toLong
}
def dateOffset(offsetValue: Int, offsetUnit: Int, currentDate: Date): Date = {
val calendar = Calendar.getInstance()
calendar.setTime(currentDate)
calendar.add(offsetUnit, offsetValue)
calendar.getTime
}
def utc2Local(utcTime: String, localTimePatten: String = "yyyy-MM-dd'T'HH:mm:ss"): String = {
val utcDate = Date.from(ZonedDateTime.parse(utcTime).toInstant)
val localF = new SimpleDateFormat(localTimePatten)
localF.setTimeZone(TimeZone.getDefault)
localF.format(utcDate.getTime)
}
}
Example 23
| Source File: Util.scala From aardpfark with Apache License 2.0 | 5 votes |
|
package com.ibm.aardpfark.pfa.utils
import java.text.SimpleDateFormat
import java.util.{Date, TimeZone}
object Utils {
def getCurrentDate = {
val fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'")
fmt.setTimeZone(TimeZone.getTimeZone("UTC"))
fmt.format(new Date())
}
def getCurrentTs = {
new Date().getTime
}
}
Example 24
| Source File: NetflixPrizeUtils.scala From zen with Apache License 2.0 | 5 votes |
|
package com.github.cloudml.zen.examples.ml
import java.text.SimpleDateFormat
import java.util.{Locale, TimeZone}
import breeze.linalg.{SparseVector => BSV}
import org.apache.spark.SparkContext
import org.apache.spark.mllib.linalg.{SparseVector => SSV}
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel
import scala.collection.mutable.ArrayBuffer
object NetflixPrizeUtils {
def genSamplesWithTime(
sc: SparkContext,
input: String,
numPartitions: Int = -1,
newLevel: StorageLevel = StorageLevel.MEMORY_AND_DISK):
(RDD[(Long, LabeledPoint)], RDD[(Long, LabeledPoint)], Array[Long]) = {
val probeFile = s"$input/probe.txt"
val dataSetFile = s"$input/training_set
val views = Array(maxUserId, maxMovieId + maxUserId, numFeatures).map(_.toLong)
(trainSet, testSet, views)
}
}
Example 25
| Source File: TimeColumnBuffer.scala From spark-vector with Apache License 2.0 | 5 votes |
|
package com.actian.spark_vector.colbuffer.time
import java.nio.ByteBuffer
import java.sql.Timestamp
import java.util.{ Calendar, TimeZone }
import org.apache.spark.sql.catalyst.util.DateTimeUtils
import com.actian.spark_vector.ComposePartial
import com.actian.spark_vector.colbuffer._
import com.actian.spark_vector.colbuffer.util._
import com.actian.spark_vector.vector.VectorDataType
private case class TimeColumnBufferParams(cbParams: ColumnBufferBuildParams,
converter: TimeConversion.TimeConverter,
adjustToUTC: Boolean = false)
private[colbuffer] abstract class TimeColumnBuffer(p: TimeColumnBufferParams, valueWidth: Int)
extends ColumnBuffer[Timestamp, Long](p.cbParams.name, p.cbParams.maxValueCount, valueWidth, valueWidth, p.cbParams.nullable) {
private val ts = new Timestamp(System.currentTimeMillis())
private val cal = Calendar.getInstance
override def put(source: Timestamp, buffer: ByteBuffer): Unit = {
if (p.adjustToUTC) {
TimeConversion.convertLocalTimestampToUTC(source, cal)
}
val convertedSource = p.converter.convert(TimeConversion.normalizeTime(source), p.cbParams.scale)
putConverted(convertedSource, buffer)
}
protected def putConverted(converted: Long, buffer: ByteBuffer): Unit
override def get(buffer: ByteBuffer): Long = {
val deconvertedSource = p.converter.deconvert(getConverted(buffer), p.cbParams.scale)
ts.setTime(TimeConversion.scaleNanos(deconvertedSource, MillisecondsScale))
ts.setNanos((deconvertedSource % PowersOfTen(NanosecondsScale)).toInt)
if (p.adjustToUTC) {
TimeConversion.convertUTCToLocalTimestamp(ts, cal)
}
DateTimeUtils.fromJavaTimestamp(ts)
}
protected def getConverted(buffer: ByteBuffer): Long
}
private class TimeIntColumnBuffer(p: TimeColumnBufferParams) extends TimeColumnBuffer(p, IntSize) {
override protected def putConverted(converted: Long, buffer: ByteBuffer): Unit = buffer.putInt(converted.toInt)
override protected def getConverted(buffer: ByteBuffer): Long = buffer.getInt()
}
private class TimeLongColumnBuffer(p: TimeColumnBufferParams) extends TimeColumnBuffer(p, LongSize) {
override protected def putConverted(converted: Long, buffer: ByteBuffer): Unit = buffer.putLong(converted)
override protected def getConverted(buffer: ByteBuffer): Long = buffer.getLong()
}
private class TimeNZLZConverter extends TimeConversion.TimeConverter {
override def convert(unscaledNanos: Long, scale: Int): Long = TimeConversion.scaleNanos(unscaledNanos, scale)
override def deconvert(scaledNanos: Long, scale: Int): Long = TimeConversion.unscaleNanos(scaledNanos, scale)
}
private class TimeTZConverter extends TimeConversion.TimeConverter {
override def convert(unscaledNanos: Long, scale: Int): Long =
(TimeConversion.scaleNanos(unscaledNanos, scale) << TimeMaskSize)
override def deconvert(scaledNanos: Long, scale: Int): Long =
TimeConversion.unscaleNanos(scaledNanos >> TimeMaskSize, scale)
}
private[colbuffer] object TimeColumnBuffer extends ColumnBufferBuilder {
private final val (nzlzIntScaleBounds, nzlzLongScaleBounds) = ((0, 4), (5, 9))
private final val (tzIntScaleBounds, tzLongScaleBounds) = ((0, 1), (2, 9))
private val calIsNotUTC = Calendar.getInstance.getTimeZone != TimeZone.getTimeZone("UTC")
private val buildNZPartial: PartialFunction[ColumnBufferBuildParams, TimeColumnBufferParams] =
ofDataType(VectorDataType.TimeType) andThen { TimeColumnBufferParams(_, new TimeNZLZConverter(), calIsNotUTC) }
private val buildLZPartial: PartialFunction[ColumnBufferBuildParams, TimeColumnBufferParams] =
ofDataType(VectorDataType.TimeLTZType) andThen { TimeColumnBufferParams(_, new TimeNZLZConverter()) }
private val buildNZLZ: PartialFunction[ColumnBufferBuildParams, ColumnBuffer[_, _]] = (buildNZPartial orElse buildLZPartial) andThenPartial {
case nzlz if isInBounds(nzlz.cbParams.scale, nzlzIntScaleBounds) => new TimeIntColumnBuffer(nzlz)
case nzlz if isInBounds(nzlz.cbParams.scale, nzlzLongScaleBounds) => new TimeLongColumnBuffer(nzlz)
}
private val buildTZPartial: PartialFunction[ColumnBufferBuildParams, TimeColumnBufferParams] =
ofDataType(VectorDataType.TimeTZType) andThen { TimeColumnBufferParams(_, new TimeTZConverter()) }
private val buildTZ: PartialFunction[ColumnBufferBuildParams, ColumnBuffer[_, _]] = buildTZPartial andThenPartial {
case tz if isInBounds(tz.cbParams.scale, tzIntScaleBounds) => new TimeIntColumnBuffer(tz)
case tz if isInBounds(tz.cbParams.scale, tzLongScaleBounds) => new TimeLongColumnBuffer(tz)
}
override private[colbuffer] val build: PartialFunction[ColumnBufferBuildParams, ColumnBuffer[_, _]] = buildNZLZ orElse buildTZ
}
Example 26
| Source File: DateTimeDataFixture.scala From spark-vector with Apache License 2.0 | 5 votes |
|
package com.actian.spark_vector.vector
import java.util.TimeZone
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.types.{DateType, StructField, StructType, TimestampType}
import com.actian.spark_vector.test.util.DateHelper.{ansiDateFor, timestampFor}
object DateTimeDataFixture {
def timeRDD(sparkContext: SparkContext): (RDD[Seq[Any]], StructType) = createTimeRDD(sparkContext, timeData)
private[vector] val tz = TimeZone.getTimeZone("GMT-06:00")
private[vector] val utc = TimeZone.getTimeZone("UTC")
private[vector] val timeData = Seq(
Seq[Any](
timestampFor(1995, 1, 22, 18, 3, 29, 234, tz),
timestampFor(1996, 2, 22, 18, 3, 29, 234),
timestampFor(1997, 2, 22, 18, 3, 29, 234),
timestampFor(1998, 1, 22, 18, 3, 29, 234, tz),
timestampFor(1999, 2, 22, 18, 3, 29, 234),
timestampFor(2000, 2, 22, 18, 3, 29, 234),
timestampFor(2015, 11, 23, 18, 3, 29, 123, tz),
timestampFor(2015, 11, 23, 18, 3, 29, 123),
ansiDateFor(1995, 2, 22)),
Seq[Any](
timestampFor(2015, 3, 2, 17, 52, 12, 678, tz),
timestampFor(2015, 4, 2, 17, 52, 12, 678),
timestampFor(2015, 4, 2, 17, 52, 12, 678),
timestampFor(2015, 3, 2, 17, 52, 12, 678, tz),
timestampFor(2015, 4, 2, 17, 52, 12, 678),
timestampFor(2015, 4, 2, 17, 52, 12, 678),
timestampFor(2015, 11, 13, 17, 52, 12, 123, tz),
ansiDateFor(2015, 4, 2)))
private def createTimeRDD(sparkContext: SparkContext, data: Seq[Seq[Any]]): (RDD[Seq[Any]], StructType) = {
val schema = StructType(Seq(
StructField("tswtz", TimestampType),
StructField("tsntz", TimestampType),
StructField("tsltz", TimestampType),
StructField("tswtz4", TimestampType),
StructField("tsntz4", TimestampType),
StructField("tsltz4", TimestampType),
StructField("tmwtz", TimestampType),
StructField("tmntz", TimestampType),
StructField("tmltz", TimestampType),
StructField("tmwtz3", TimestampType),
StructField("tmntz3", TimestampType),
StructField("tmltz3", TimestampType),
StructField("date", DateType)))
(sparkContext.parallelize(data, 2), schema)
}
def createTimeTable(connectionProps: VectorConnectionProperties)(tableName: String): Unit = {
VectorJDBC.withJDBC(connectionProps) { cxn =>
cxn.dropTable(tableName)
cxn.executeStatement(
s"""|create table ${tableName} (
| tswtz timestamp with time zone,
| tsntz timestamp without time zone,
| tsltz timestamp with local time zone,
| tswtz4 timestamp(4) with time zone,
| tsntz4 timestamp(4) without time zone,
| tsltz4 timestamp(4) with local time zone,
| tmwtz time with time zone,
| tmntz time without time zone,
| tmltz time with local time zone,
| tmwtz3 time(3) with time zone,
| tmntz3 time(3) without time zone,
| tmltz3 time(3) with local time zone,
| dt date
|)""".stripMargin)
}
}
}
Example 27
| Source File: DateHelper.scala From spark-vector with Apache License 2.0 | 5 votes |
|
package com.actian.spark_vector.test.util
import java.sql.{ Date => sqlDate, Timestamp }
import java.util.{ Calendar, Date, GregorianCalendar, TimeZone }
object DateHelper {
def dateFor(year: Int, month: Int, day: Int, hours: Int = 0, minutes: Int = 0, seconds: Int = 0, millis: Int = 0, tz: TimeZone = TimeZone.getDefault): Date = {
import java.util.Calendar._
val cal = new GregorianCalendar()
cal.setTimeZone(tz)
if (year > 0) cal.set(YEAR, year)
if (month >= 0) cal.set(MONTH, month)
if (day > 0) cal.set(DAY_OF_MONTH, day)
cal.set(HOUR_OF_DAY, hours)
cal.set(MINUTE, minutes)
cal.set(SECOND, seconds)
cal.set(MILLISECOND, millis)
cal.getTime
}
def timestampFor(year: Int, month: Int, day: Int, hours: Int = 0, minutes: Int = 0, seconds: Int = 0, millis: Int = 0, tz: TimeZone = TimeZone.getDefault): Timestamp =
new Timestamp(dateFor(year, month, day, hours, minutes, seconds, millis, tz).getTime)
def timeFor(hours: Int, minutes: Int, seconds: Int): Timestamp = {
import java.util.Calendar
val year = Calendar.getInstance().get(Calendar.YEAR)
val month = Calendar.getInstance().get(Calendar.MONTH)
val day = Calendar.getInstance().get(Calendar.DATE)
timestampFor(year, month, day, hours, minutes, seconds)
}
def ansiDateFor(year: Int, month: Int, day: Int): sqlDate = new sqlDate(dateFor(year, month, day).getTime())
}
Example 28
| Source File: ReceiptRenderer.scala From apple-of-my-iap with MIT License | 5 votes |
|
package com.meetup.iap.receipt
import java.text.SimpleDateFormat
import com.meetup.iap.AppleApi
import AppleApi.{ReceiptResponse, ReceiptInfo}
import java.util.{Date, TimeZone}
import org.json4s.JsonDSL._
import org.json4s.native.JsonMethods._
import org.json4s.JsonAST.JValue
import org.slf4j.LoggerFactory
object ReceiptRenderer {
val log = LoggerFactory.getLogger(ReceiptRenderer.getClass)
private def appleDateFormat(date: Date): String = {
val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss 'Etc/GMT'")
sdf.setTimeZone(TimeZone.getTimeZone("UTC"))
sdf.format(date)
}
def apply(response: ReceiptResponse): String = {
pretty(render(
("status" -> response.statusCode) ~
("latest_receipt_info" -> response.latestReceiptInfo.reverse.map(renderReceipt)) ~
("latest_receipt" -> response.latestReceipt)))
}
private def renderReceipt(receiptInfo: ReceiptInfo): JValue = {
val origPurchaseDate = receiptInfo.originalPurchaseDate
val origPurchaseDateStr = appleDateFormat(origPurchaseDate)
val origPurchaseDateMs = origPurchaseDate.getTime
val purchaseDate = receiptInfo.purchaseDate
val purchaseDateStr = appleDateFormat(purchaseDate)
val purchaseDateMs = purchaseDate.getTime
val expiresDate = receiptInfo.expiresDate
val expiresDateStr = appleDateFormat(expiresDate)
val expiresDateMs = expiresDate.getTime
val cancellationDate = receiptInfo.cancellationDate.map { date =>
appleDateFormat(date)
}
("quantity" -> "1") ~
("product_id" -> receiptInfo.productId) ~
("transaction_id" -> receiptInfo.transactionId) ~
("original_transaction_id" -> receiptInfo.originalTransactionId) ~
("purchase_date" -> purchaseDateStr) ~
("purchase_date_ms" -> purchaseDateMs.toString) ~
("original_purchase_date" -> origPurchaseDateStr) ~
("original_purchase_date_ms" -> origPurchaseDateMs.toString) ~
("expires_date" -> expiresDateStr) ~
("expires_date_ms" -> expiresDateMs.toString) ~
("is_trial_period" -> receiptInfo.isTrialPeriod.toString) ~ //We mimic Apple's weird json here by converting the boolean type to a string
("is_in_intro_offer_period" -> receiptInfo.isInIntroOfferPeriod.map(_.toString)) ~
("cancellation_date" -> cancellationDate)
}
}
Example 29
| Source File: DataFrameReportPerformanceSpec.scala From seahorse-workflow-executor with Apache License 2.0 | 5 votes |
|
package io.deepsense.deeplang.doperables.dataframe
import java.sql.Timestamp
import java.text.{DateFormat, SimpleDateFormat}
import java.util.TimeZone
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DoubleType, StructField, StructType, TimestampType}
import org.scalatest.{BeforeAndAfter, Ignore}
import io.deepsense.commons.utils.{DoubleUtils, Logging}
import io.deepsense.deeplang.{TestFiles, DeeplangIntegTestSupport}
// It's ignored because it does not have got assertions, it only prints report generation time.
@Ignore
class DataFrameReportPerformanceSpec
extends DeeplangIntegTestSupport
with BeforeAndAfter
with TestFiles
with Logging {
val testFile = absoluteTestsDirPath.pathWithoutScheme + "/demand_without_header.csv"
"DataFrame" should {
"generate report" when {
"DataFrame has 17K of rows" in {
val numberOfTries = 10
var results: Seq[Double] = Seq()
for (i <- 1 to numberOfTries) {
val dataFrame: DataFrame = demandDataFrame()
val start = System.nanoTime()
val report = dataFrame.report
val end = System.nanoTime()
val time1: Double = (end - start).toDouble / 1000000000.0
results = results :+ time1
logger.debug("Report generation time: {}", DoubleUtils.double2String(time1))
}
logger.debug(
"Mean report generation time: {}",
DoubleUtils.double2String(results.fold(0D)(_ + _) / numberOfTries.toDouble))
}
}
}
private def demandDataFrame(): DataFrame = {
val rddString: RDD[String] = executionContext.sparkContext.textFile(testFile)
val data: RDD[Row] = rddString.map(DataFrameHelpers.demandString2Row)
executionContext.dataFrameBuilder.buildDataFrame(demandSchema, data)
}
private def demandSchema: StructType = StructType(Seq(
StructField("datetime", TimestampType),
StructField("log_count", DoubleType),
StructField("workingday", DoubleType),
StructField("holiday", DoubleType),
StructField("season2", DoubleType),
StructField("season3", DoubleType),
StructField("season4", DoubleType)))
private def timestamp(s: String): Timestamp = {
val format: DateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
format.setTimeZone(TimeZone.getTimeZone("UTC"))
new Timestamp(format.parse(s).getTime)
}
}
private object DataFrameHelpers {
def demandString2Row(s: String): Row = {
val split = s.split(",")
Row(
timestamp(split(0)),
split(1).toDouble,
split(2).toDouble,
split(3).toDouble,
split(4).toDouble,
split(5).toDouble,
split(6).toDouble
)
}
private def timestamp(s: String): Timestamp = {
val format: DateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
format.setTimeZone(TimeZone.getTimeZone("UTC"))
new Timestamp(format.parse(s).getTime)
}
}
Example 30
| Source File: QuartzExtension.scala From sumobot with Apache License 2.0 | 5 votes |
|
package com.sumologic.sumobot.quartz
import java.util.TimeZone
import akka.actor._
import org.quartz.{JobDataMap, JobExecutionContext}
import org.quartz.impl.StdSchedulerFactory
import org.quartz.TriggerBuilder._
import org.quartz.CronScheduleBuilder._
import org.quartz.JobBuilder._
object QuartzExtension
extends ExtensionId[QuartzExtension]
with ExtensionIdProvider {
override def createExtension(system: ExtendedActorSystem): QuartzExtension = new QuartzExtension
override def lookup() = QuartzExtension
}
class QuartzExtension extends Extension {
private val scheduler = StdSchedulerFactory.getDefaultScheduler
def scheduleMessage(name: String, cronExpression: String, actor: ActorRef, message: AnyRef): Unit = {
ensureStarted()
val job = newJob(classOf[ActorSenderJob]).
withIdentity(name).
build()
val map = new JobDataMap()
map.put(ActorSenderJob.ActorRef, actor)
map.put(ActorSenderJob.Message, message)
val trigger = newTrigger().
withIdentity(name).
usingJobData(map).
withSchedule(cronSchedule(cronExpression).inTimeZone(TimeZone.getDefault)).
build()
scheduler.scheduleJob(job, trigger)
}
private def ensureStarted(): Unit = {
if (!scheduler.isStarted) {
scheduler.start()
}
}
}
object ActorSenderJob {
val ActorRef = "ActorRef"
val Message = "Message"
}
class ActorSenderJob extends org.quartz.Job {
override def execute(jobExecutionContext: JobExecutionContext): Unit = {
val actor = jobExecutionContext.getMergedJobDataMap.get(ActorSenderJob.ActorRef).asInstanceOf[ActorRef]
val message = jobExecutionContext.getMergedJobDataMap.get(ActorSenderJob.Message)
actor ! message
}
}
Example 31
| Source File: SLF4JSpec.scala From scribe with MIT License | 5 votes |
|
package spec
import java.util.TimeZone
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
import org.slf4j.{LoggerFactory, MDC}
import scribe.handler.LogHandler
import scribe.output.LogOutput
import scribe.util.Time
import scribe.writer.Writer
import scribe.{Level, LogRecord, Logger}
class SLF4JSpec extends AnyWordSpec with Matchers {
TimeZone.setDefault(TimeZone.getTimeZone("UTC"))
private var logs: List[LogRecord[_]] = Nil
private var logOutput: List[String] = Nil
private val recordHolder = LogHandler.default.withMinimumLevel(Level.Info).withWriter(new Writer {
override def write[M](record: LogRecord[M], output: LogOutput): Unit = {
logs = record :: logs
logOutput = output.plainText :: logOutput
}
})
"SLF4J" should {
"set the time to an arbitrary value" in {
Time.function = () => 1542376191920L
}
"remove existing handlers from Root" in {
Logger.root.clearHandlers().replace()
}
"add a testing handler" in {
Logger.root.withHandler(recordHolder).replace()
}
"verify not records are in the RecordHolder" in {
logs.isEmpty should be(true)
}
"log to Scribe" in {
val logger = LoggerFactory.getLogger(getClass)
logger.info("Hello World!")
}
"verify Scribe received the record" in {
logs.size should be(1)
val r = logs.head
r.level should be(Level.Info)
r.message.plainText should be("Hello World!")
r.className should be("spec.SLF4JSpec")
logs = Nil
}
"verify Scribe wrote value" in {
logOutput.size should be(1)
val s = logOutput.head
s should be("2018.11.16 13:49:51 [INFO] spec.SLF4JSpec - Hello World!")
}
"use MDC" in {
MDC.put("name", "John Doe")
val logger = LoggerFactory.getLogger(getClass)
logger.info("A generic name")
logOutput.head should be("2018.11.16 13:49:51 [INFO] spec.SLF4JSpec - A generic name (name: John Doe)")
}
"clear MDC" in {
MDC.clear()
val logger = LoggerFactory.getLogger(getClass)
logger.info("MDC cleared")
logOutput.head should be("2018.11.16 13:49:51 [INFO] spec.SLF4JSpec - MDC cleared")
}
"make sure logging nulls doesn't error" in {
val logger = LoggerFactory.getLogger(getClass)
logger.error(null)
logs.length should be(3)
logOutput.head should be("2018.11.16 13:49:51 [ERROR] spec.SLF4JSpec - null")
}
}
}
Example 32
| Source File: SimpleDateParam.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.status.api.v1
import java.text.{ParseException, SimpleDateFormat}
import java.util.TimeZone
import javax.ws.rs.WebApplicationException
import javax.ws.rs.core.Response
import javax.ws.rs.core.Response.Status
private[v1] class SimpleDateParam(val originalValue: String) {
val timestamp: Long = {
val format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz")
try {
format.parse(originalValue).getTime()
} catch {
case _: ParseException =>
val gmtDay = new SimpleDateFormat("yyyy-MM-dd")
gmtDay.setTimeZone(TimeZone.getTimeZone("GMT"))
try {
gmtDay.parse(originalValue).getTime()
} catch {
case _: ParseException =>
throw new WebApplicationException(
Response
.status(Status.BAD_REQUEST)
.entity("Couldn't parse date: " + originalValue)
.build()
)
}
}
}
}
Example 33
| Source File: Gens.scala From chronoscala with MIT License | 5 votes |
|
package jp.ne.opt.chronoscala
import java.time._
import java.util.TimeZone
import org.scalacheck.Gen
trait Gens {
def instantGen: Gen[Instant] = Gen.chooseNum(0L, Long.MaxValue).map(Instant.ofEpochMilli)
def localDateGen: Gen[LocalDate] = for {
year <- Gen.chooseNum(Year.MIN_VALUE, Year.MAX_VALUE)
month <- Gen.chooseNum(1, 12)
dayOfMonth <- Gen.chooseNum(1, Month.of(month).length(Year.isLeap(year)))
} yield LocalDate.of(year, month, dayOfMonth)
def localDateTimeGen: Gen[LocalDateTime] = for {
instant <- instantGen
zoneId <- Gen.oneOf(TimeZone.getAvailableIDs.map(TimeZone.getTimeZone(_).toZoneId).toSeq)
} yield LocalDateTime.ofInstant(instant, zoneId)
def localTimeGen: Gen[LocalTime] = for {
hour <- Gen.chooseNum(0, 23)
minute <- Gen.chooseNum(0, 59)
second <- Gen.chooseNum(0, 59)
} yield LocalTime.of(hour, minute, second)
def zonedDateTimeGen: Gen[ZonedDateTime] = for {
instant <- instantGen
zoneId <- Gen.oneOf(TimeZone.getAvailableIDs.map(TimeZone.getTimeZone(_).toZoneId).toSeq)
} yield ZonedDateTime.ofInstant(instant, zoneId)
def offsetDateTimeGen: Gen[OffsetDateTime] = for {
instant <- instantGen.map(_.atZone(ZoneOffset.UTC).toLocalDateTime)
offset <- Gen.chooseNum(ZoneOffset.MIN.getTotalSeconds, ZoneOffset.MAX.getTotalSeconds)
.map(ZoneOffset.ofTotalSeconds)
} yield OffsetDateTime.of(instant, offset)
def durationGen: Gen[Duration] = for {
start <- instantGen
end <- instantGen
} yield Duration.between(start, end)
}
Example 34
| Source File: DataConverter.scala From spark-cdm with MIT License | 5 votes |
|
package com.microsoft.cdm.utils
import java.text.SimpleDateFormat
import java.util.{Locale, TimeZone}
import java.sql.Timestamp
import org.apache.commons.lang.time.DateUtils
import org.apache.spark.sql.catalyst.util.TimestampFormatter
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
class DataConverter() extends Serializable {
val dateFormatter = new SimpleDateFormat(Constants.SINGLE_DATE_FORMAT)
val timestampFormatter = TimestampFormatter(Constants.TIMESTAMP_FORMAT, TimeZone.getTimeZone("UTC"))
val toSparkType: Map[CDMDataType.Value, DataType] = Map(
CDMDataType.int64 -> LongType,
CDMDataType.dateTime -> DateType,
CDMDataType.string -> StringType,
CDMDataType.double -> DoubleType,
CDMDataType.decimal -> DecimalType(Constants.DECIMAL_PRECISION,0),
CDMDataType.boolean -> BooleanType,
CDMDataType.dateTimeOffset -> TimestampType
)
def jsonToData(dt: DataType, value: String): Any = {
return dt match {
case LongType => value.toLong
case DoubleType => value.toDouble
case DecimalType() => Decimal(value)
case BooleanType => value.toBoolean
case DateType => dateFormatter.parse(value)
case TimestampType => timestampFormatter.parse(value)
case _ => UTF8String.fromString(value)
}
}
def toCdmType(dt: DataType): CDMDataType.Value = {
return dt match {
case IntegerType => CDMDataType.int64
case LongType => CDMDataType.int64
case DateType => CDMDataType.dateTime
case StringType => CDMDataType.string
case DoubleType => CDMDataType.double
case DecimalType() => CDMDataType.decimal
case BooleanType => CDMDataType.boolean
case TimestampType => CDMDataType.dateTimeOffset
}
}
def dataToString(data: Any, dataType: DataType): String = {
(dataType, data) match {
case (_, null) => null
case (DateType, _) => dateFormatter.format(data)
case (TimestampType, v: Number) => timestampFormatter.format(data.asInstanceOf[Long])
case _ => data.toString
}
}
}
Example 35
| Source File: WriteAndReadGenericApp.scala From parquet4s with MIT License | 5 votes |
|
package com.github.mjakubowski84.parquet4s.core
import java.time.{LocalDate, ZoneOffset}
import java.util.TimeZone
import com.github.mjakubowski84.parquet4s.{ParquetReader, ParquetWriter, RowParquetRecord, ValueCodecConfiguration}
import com.google.common.io.Files
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.{BINARY, INT32, INT64}
import org.apache.parquet.schema.Type.Repetition.{OPTIONAL, REQUIRED}
import org.apache.parquet.schema.{MessageType, OriginalType, Types}
object WriteAndReadGenericApp extends App {
val ID = "id"
val Name = "name"
val Birthday = "birthday"
val Schema = "user_schema"
val path = Files.createTempDir().getAbsolutePath
val vcc = ValueCodecConfiguration(TimeZone.getTimeZone(ZoneOffset.UTC))
val users = List(
(1L, "Alice", LocalDate.of(2000, 1, 1)),
(2L, "Bob", LocalDate.of(1980, 2, 28)),
(3L, "Cecilia", LocalDate.of(1977, 3, 15))
).map { case (id, name, birthday) =>
RowParquetRecord.empty
.add(ID, id, vcc)
.add(Name, name, vcc)
.add(Birthday, birthday, vcc)
}
// write
implicit val schema: MessageType = Types.buildMessage()
.addField(Types.primitive(INT64, REQUIRED).as(OriginalType.INT_64).named(ID))
.addField(Types.primitive(BINARY, OPTIONAL).as(OriginalType.UTF8).named(Name))
.addField(Types.primitive(INT32, OPTIONAL).as(OriginalType.DATE).named(Birthday))
.named(Schema)
ParquetWriter.writeAndClose(s"$path/users.parquet", users)
//read
val readData = ParquetReader.read[RowParquetRecord](path)
try {
readData.foreach { record =>
val id = record.get[Long](ID, vcc)
val name = record.get[String](Name, vcc)
val birthday = record.get[LocalDate](Birthday, vcc)
println(s"User[$ID=$id,$Name=$name,$Birthday=$birthday]")
}
} finally readData.close()
}
Example 36
| Source File: TimeEncodingCompatibilityItSpec.scala From parquet4s with MIT License | 5 votes |
|
package com.github.mjakubowski84.parquet4s
import java.util.TimeZone
import com.github.mjakubowski84.parquet4s.CompatibilityTestCases.TimePrimitives
import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers}
class TimeEncodingCompatibilityItSpec extends
FlatSpec
with Matchers
with BeforeAndAfter
with SparkHelper {
private val localTimeZone = TimeZone.getDefault
private val utcTimeZone = TimeZone.getTimeZone("UTC")
private lazy val newYearEveEvening = java.sql.Timestamp.valueOf(java.time.LocalDateTime.of(2018, 12, 31, 23, 0, 0))
private lazy val newYearMidnight = java.sql.Timestamp.valueOf(java.time.LocalDateTime.of(2019, 1, 1, 0, 0, 0))
private lazy val newYear = java.sql.Date.valueOf(java.time.LocalDate.of(2019, 1, 1))
override def beforeAll(): Unit = {
super.beforeAll()
TimeZone.setDefault(utcTimeZone)
}
before {
clearTemp()
}
private def writeWithSpark(data: TimePrimitives): Unit = writeToTemp(Seq(data))
private def readWithSpark: TimePrimitives = readFromTemp[TimePrimitives].head
private def writeWithParquet4S(data: TimePrimitives, timeZone: TimeZone): Unit =
ParquetWriter.writeAndClose(tempPathString, Seq(data), ParquetWriter.Options(timeZone = timeZone))
private def readWithParquet4S(timeZone: TimeZone): TimePrimitives = {
val parquetIterable = ParquetReader.read[TimePrimitives](tempPathString, ParquetReader.Options(timeZone = timeZone))
try {
parquetIterable.head
} finally {
parquetIterable.close()
}
}
"Spark" should "read properly time written with time zone one hour east" in {
val input = TimePrimitives(timestamp = newYearMidnight, date = newYear)
val expectedOutput = TimePrimitives(timestamp = newYearEveEvening, date = newYear)
writeWithParquet4S(input, TimeZone.getTimeZone("GMT+1"))
readWithSpark should be(expectedOutput)
}
it should "read properly written with time zone one hour west" in {
val input = TimePrimitives(timestamp = newYearEveEvening, date = newYear)
val expectedOutput = TimePrimitives(timestamp = newYearMidnight, date = newYear)
writeWithParquet4S(input, TimeZone.getTimeZone("GMT-1"))
readWithSpark should be(expectedOutput)
}
"Parquet4S" should "read properly time written with time zone one hour east" in {
val input = TimePrimitives(timestamp = newYearMidnight, date = newYear)
val expectedOutput = TimePrimitives(timestamp = newYearEveEvening, date = newYear)
writeWithSpark(input)
readWithParquet4S(TimeZone.getTimeZone("GMT-1")) should be(expectedOutput)
}
it should "read properly time written with time zone one hour west" in {
val input = TimePrimitives(timestamp = newYearEveEvening, date = newYear)
val expectedOutput = TimePrimitives(timestamp = newYearMidnight, date = newYear)
writeWithSpark(input)
readWithParquet4S(TimeZone.getTimeZone("GMT+1")) should be(expectedOutput)
}
override def afterAll(): Unit = {
TimeZone.setDefault(localTimeZone)
super.afterAll()
}
}
Example 37
| Source File: TimeEncodingCompatibilityItSpec.scala From parquet4s with MIT License | 5 votes |
|
package com.github.mjakubowski84.parquet4s
import java.util.TimeZone
import com.github.mjakubowski84.parquet4s.CompatibilityTestCases.TimePrimitives
import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers}
class TimeEncodingCompatibilityItSpec extends
FlatSpec
with Matchers
with BeforeAndAfter
with SparkHelper {
private val localTimeZone = TimeZone.getDefault
private val utcTimeZone = TimeZone.getTimeZone("UTC")
private lazy val newYearEveEvening = java.sql.Timestamp.valueOf(java.time.LocalDateTime.of(2018, 12, 31, 23, 0, 0))
private lazy val newYearMidnight = java.sql.Timestamp.valueOf(java.time.LocalDateTime.of(2019, 1, 1, 0, 0, 0))
private lazy val newYear = java.sql.Date.valueOf(java.time.LocalDate.of(2019, 1, 1))
override def beforeAll(): Unit = {
super.beforeAll()
TimeZone.setDefault(utcTimeZone)
}
before {
clearTemp()
}
private def writeWithSpark(data: TimePrimitives): Unit = writeToTemp(Seq(data))
private def readWithSpark: TimePrimitives = readFromTemp[TimePrimitives].head
private def writeWithParquet4S(data: TimePrimitives, timeZone: TimeZone): Unit =
ParquetWriter.writeAndClose(tempPathString, Seq(data), ParquetWriter.Options(timeZone = timeZone))
private def readWithParquet4S(timeZone: TimeZone): TimePrimitives = {
val parquetIterable = ParquetReader.read[TimePrimitives](tempPathString, ParquetReader.Options(timeZone = timeZone))
try {
parquetIterable.head
} finally {
parquetIterable.close()
}
}
"Spark" should "read properly time written with time zone one hour east" in {
val input = TimePrimitives(timestamp = newYearMidnight, date = newYear)
val expectedOutput = TimePrimitives(timestamp = newYearEveEvening, date = newYear)
writeWithParquet4S(input, TimeZone.getTimeZone("GMT+1"))
readWithSpark should be(expectedOutput)
}
it should "read properly written with time zone one hour west" in {
val input = TimePrimitives(timestamp = newYearEveEvening, date = newYear)
val expectedOutput = TimePrimitives(timestamp = newYearMidnight, date = newYear)
writeWithParquet4S(input, TimeZone.getTimeZone("GMT-1"))
readWithSpark should be(expectedOutput)
}
"Parquet4S" should "read properly time written with time zone one hour east" in {
val input = TimePrimitives(timestamp = newYearMidnight, date = newYear)
val expectedOutput = TimePrimitives(timestamp = newYearEveEvening, date = newYear)
writeWithSpark(input)
readWithParquet4S(TimeZone.getTimeZone("GMT-1")) should be(expectedOutput)
}
it should "read properly time written with time zone one hour west" in {
val input = TimePrimitives(timestamp = newYearEveEvening, date = newYear)
val expectedOutput = TimePrimitives(timestamp = newYearMidnight, date = newYear)
writeWithSpark(input)
readWithParquet4S(TimeZone.getTimeZone("GMT+1")) should be(expectedOutput)
}
override def afterAll(): Unit = {
TimeZone.setDefault(localTimeZone)
super.afterAll()
}
}
Example 38
| Source File: Total.scala From akka_streams_tutorial with MIT License | 5 votes |
|
package sample.stream_actor
import java.text.SimpleDateFormat
import java.util.{Date, TimeZone}
import akka.Done
import akka.actor.Actor
import sample.stream_actor.Total.Increment
object Total {
case class Increment(value: Long, avg: Double, id: String)
}
class Total extends Actor {
var total: Long = 0
override def receive: Receive = {
case Increment(value, avg, id) =>
println(s"Received $value new measurements from turbine with id: $id - Avg wind speed is: $avg")
total = total + value
val date = new Date()
val df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
df.setTimeZone(TimeZone.getTimeZone("Europe/Zurich"))
println(s"${df.format(date) } - Current total of all measurements: $total")
sender ! Done
}
}
Example 39
| Source File: CSVSpec.scala From flint with Apache License 2.0 | 5 votes |
|
package com.twosigma.flint.timeseries
import java.util.TimeZone
import com.twosigma.flint.timeseries.row.Schema
import org.apache.spark.sql.types._
class CSVSpec extends TimeSeriesSuite {
"CSV" should "read a CSV file without header." in {
withResource("/timeseries/csv/Price.csv") { source =>
val expectedSchema = Schema("C1" -> IntegerType, "C2" -> DoubleType)
val timeseriesRdd = CSV.from(sqlContext, "file://" + source, sorted = true, schema = expectedSchema)
val ts = timeseriesRdd.collect()
assert(timeseriesRdd.schema == expectedSchema)
assert(ts(0).getAs[Long](TimeSeriesRDD.timeColumnName) == 1000L)
assert(ts(0).getAs[Integer]("C1") == 7)
assert(ts(0).getAs[Double]("C2") == 0.5)
assert(ts.length == 12)
}
}
it should "read a CSV file with header." in {
withResource("/timeseries/csv/PriceWithHeader.csv") { source =>
val expectedSchema = Schema("id" -> IntegerType, "price" -> DoubleType, "info" -> StringType)
val timeseriesRdd = CSV.from(
sqlContext,
"file://" + source, header = true, sorted = true
)
val ts = timeseriesRdd.collect()
assert(timeseriesRdd.schema == expectedSchema)
assert(ts(0).getAs[Long](TimeSeriesRDD.timeColumnName) == 1000L)
assert(ts(0).getAs[Integer]("id") == 7)
assert(ts(0).getAs[Double]("price") == 0.5)
assert(ts.length == 12)
}
}
it should "read a CSV file with header and keep origin time column." in {
withResource("/timeseries/csv/PriceWithHeader.csv") { source =>
val expectedSchema = Schema(
"time_" -> IntegerType,
"id" -> IntegerType,
"price" -> DoubleType,
"info" -> StringType
)
val timeseriesRdd = CSV.from(sqlContext, "file://" + source,
header = true, keepOriginTimeColumn = true, sorted = true)
val ts = timeseriesRdd.collect()
// we want to keep the time column first, but the order isn't guaranteed
assert(timeseriesRdd.schema.fieldIndex(TimeSeriesRDD.timeColumnName) == 0)
assert(timeseriesRdd.schema.fields.toSet == expectedSchema.fields.toSet)
assert(ts(0).getAs[Long](TimeSeriesRDD.timeColumnName) == 1000L)
assert(ts(0).getAs[Integer]("id") == 7)
assert(ts(0).getAs[Double]("price") == 0.5)
assert(ts.forall(_.getAs[String]("info") == "test"))
assert(ts.length == 12)
}
}
it should "read an unsorted CSV file with header" in {
val ts1 = withResource("/timeseries/csv/PriceWithHeaderUnsorted.csv") { source =>
val timeseriesRdd = CSV.from(sqlContext, "file://" + source, header = true, sorted = false)
timeseriesRdd.collect()
}
val ts2 = withResource("/timeseries/csv/PriceWithHeader.csv") { source =>
val timeseriesRdd = CSV.from(sqlContext, "file://" + source, header = true, sorted = true)
timeseriesRdd.collect()
}
assert(ts1.length == ts2.length)
assert(ts1.deep == ts2.deep)
}
it should "correctly convert SQL TimestampType with default format" in {
withResource("/timeseries/csv/TimeStampsWithHeader.csv") { source =>
val timeseriesRdd = CSV.from(sqlContext, "file://" + source,
header = true, sorted = false)
val first = timeseriesRdd.first()
val format = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S")
format.setTimeZone(TimeZone.getDefault)
assert(first.getAs[Long]("time") == format.parse("2008-01-02 00:00:00.000").getTime * 1000000L)
}
}
it should "correctly convert SQL TimestampType with specified format" in {
withResource("/timeseries/csv/TimeStampsWithHeader2.csv") { source =>
val timeseriesRdd = CSV.from(sqlContext, "file://" + source,
header = true, sorted = false, dateFormat = "yyyyMMdd'T'HH:mm:ssZ")
val first = timeseriesRdd.first()
val format = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S")
format.setTimeZone(TimeZone.getTimeZone("UTC"))
assert(first.getAs[Long]("time") == format.parse("2008-01-02 00:00:00.000").getTime * 1000000L)
}
}
}
Example 40
| Source File: TimestampFormatter.scala From XSQL with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.catalyst.util
import java.text.ParseException
import java.time._
import java.time.format.DateTimeParseException
import java.time.temporal.TemporalQueries
import java.util.{Locale, TimeZone}
import org.apache.spark.sql.catalyst.util.DateTimeUtils.instantToMicros
sealed trait TimestampFormatter extends Serializable {
@throws(classOf[ParseException])
@throws(classOf[DateTimeParseException])
@throws(classOf[DateTimeException])
def parse(s: String): Long
def format(us: Long): String
}
class Iso8601TimestampFormatter(
pattern: String,
timeZone: TimeZone,
locale: Locale) extends TimestampFormatter with DateTimeFormatterHelper {
@transient
private lazy val formatter = getOrCreateFormatter(pattern, locale)
private def toInstant(s: String): Instant = {
val temporalAccessor = formatter.parse(s)
if (temporalAccessor.query(TemporalQueries.offset()) == null) {
toInstantWithZoneId(temporalAccessor, timeZone.toZoneId)
} else {
Instant.from(temporalAccessor)
}
}
override def parse(s: String): Long = instantToMicros(toInstant(s))
override def format(us: Long): String = {
val secs = Math.floorDiv(us, DateTimeUtils.MICROS_PER_SECOND)
val mos = Math.floorMod(us, DateTimeUtils.MICROS_PER_SECOND)
val instant = Instant.ofEpochSecond(secs, mos * DateTimeUtils.NANOS_PER_MICROS)
formatter.withZone(timeZone.toZoneId).format(instant)
}
}
object TimestampFormatter {
val defaultPattern: String = "yyyy-MM-dd HH:mm:ss"
val defaultLocale: Locale = Locale.US
def apply(format: String, timeZone: TimeZone, locale: Locale): TimestampFormatter = {
new Iso8601TimestampFormatter(format, timeZone, locale)
}
def apply(format: String, timeZone: TimeZone): TimestampFormatter = {
apply(format, timeZone, defaultLocale)
}
def apply(timeZone: TimeZone): TimestampFormatter = {
apply(defaultPattern, timeZone, defaultLocale)
}
}
Example 41
| Source File: SortOrderExpressionsSuite.scala From XSQL with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.catalyst.expressions
import java.sql.{Date, Timestamp}
import java.util.TimeZone
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.collection.unsafe.sort.PrefixComparators._
class SortOrderExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
test("SortPrefix") {
val b1 = Literal.create(false, BooleanType)
val b2 = Literal.create(true, BooleanType)
val i1 = Literal.create(20132983, IntegerType)
val i2 = Literal.create(-20132983, IntegerType)
val l1 = Literal.create(20132983, LongType)
val l2 = Literal.create(-20132983, LongType)
val millis = 1524954911000L;
// Explicitly choose a time zone, since Date objects can create different values depending on
// local time zone of the machine on which the test is running
val oldDefaultTZ = TimeZone.getDefault
val d1 = try {
TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
Literal.create(new java.sql.Date(millis), DateType)
} finally {
TimeZone.setDefault(oldDefaultTZ)
}
val t1 = Literal.create(new Timestamp(millis), TimestampType)
val f1 = Literal.create(0.7788229f, FloatType)
val f2 = Literal.create(-0.7788229f, FloatType)
val db1 = Literal.create(0.7788229d, DoubleType)
val db2 = Literal.create(-0.7788229d, DoubleType)
val s1 = Literal.create("T", StringType)
val s2 = Literal.create("This is longer than 8 characters", StringType)
val bin1 = Literal.create(Array[Byte](12), BinaryType)
val bin2 = Literal.create(Array[Byte](12, 17, 99, 0, 0, 0, 2, 3, 0xf4.asInstanceOf[Byte]),
BinaryType)
val dec1 = Literal(Decimal(20132983L, 10, 2))
val dec2 = Literal(Decimal(20132983L, 19, 2))
val dec3 = Literal(Decimal(20132983L, 21, 2))
val list1 = Literal(List(1, 2), ArrayType(IntegerType))
val nullVal = Literal.create(null, IntegerType)
checkEvaluation(SortPrefix(SortOrder(b1, Ascending)), 0L)
checkEvaluation(SortPrefix(SortOrder(b2, Ascending)), 1L)
checkEvaluation(SortPrefix(SortOrder(i1, Ascending)), 20132983L)
checkEvaluation(SortPrefix(SortOrder(i2, Ascending)), -20132983L)
checkEvaluation(SortPrefix(SortOrder(l1, Ascending)), 20132983L)
checkEvaluation(SortPrefix(SortOrder(l2, Ascending)), -20132983L)
// For some reason, the Literal.create code gives us the number of days since the epoch
checkEvaluation(SortPrefix(SortOrder(d1, Ascending)), 17649L)
checkEvaluation(SortPrefix(SortOrder(t1, Ascending)), millis * 1000)
checkEvaluation(SortPrefix(SortOrder(f1, Ascending)),
DoublePrefixComparator.computePrefix(f1.value.asInstanceOf[Float].toDouble))
checkEvaluation(SortPrefix(SortOrder(f2, Ascending)),
DoublePrefixComparator.computePrefix(f2.value.asInstanceOf[Float].toDouble))
checkEvaluation(SortPrefix(SortOrder(db1, Ascending)),
DoublePrefixComparator.computePrefix(db1.value.asInstanceOf[Double]))
checkEvaluation(SortPrefix(SortOrder(db2, Ascending)),
DoublePrefixComparator.computePrefix(db2.value.asInstanceOf[Double]))
checkEvaluation(SortPrefix(SortOrder(s1, Ascending)),
StringPrefixComparator.computePrefix(s1.value.asInstanceOf[UTF8String]))
checkEvaluation(SortPrefix(SortOrder(s2, Ascending)),
StringPrefixComparator.computePrefix(s2.value.asInstanceOf[UTF8String]))
checkEvaluation(SortPrefix(SortOrder(bin1, Ascending)),
BinaryPrefixComparator.computePrefix(bin1.value.asInstanceOf[Array[Byte]]))
checkEvaluation(SortPrefix(SortOrder(bin2, Ascending)),
BinaryPrefixComparator.computePrefix(bin2.value.asInstanceOf[Array[Byte]]))
checkEvaluation(SortPrefix(SortOrder(dec1, Ascending)), 20132983L)
checkEvaluation(SortPrefix(SortOrder(dec2, Ascending)), 2013298L)
checkEvaluation(SortPrefix(SortOrder(dec3, Ascending)),
DoublePrefixComparator.computePrefix(201329.83d))
checkEvaluation(SortPrefix(SortOrder(list1, Ascending)), 0L)
checkEvaluation(SortPrefix(SortOrder(nullVal, Ascending)), null)
}
}
Example 42
| Source File: DateTimeTestUtils.scala From XSQL with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.catalyst.util
import java.util.TimeZone
object DateTimeTestUtils {
val ALL_TIMEZONES: Seq[TimeZone] = TimeZone.getAvailableIDs.toSeq.map(TimeZone.getTimeZone)
val outstandingTimezonesIds: Seq[String] = Seq(
"UTC",
"PST",
"CET",
"Africa/Dakar",
"America/Los_Angeles",
"Antarctica/Vostok",
"Asia/Hong_Kong",
"Europe/Amsterdam")
val outstandingTimezones: Seq[TimeZone] = outstandingTimezonesIds.map(TimeZone.getTimeZone)
def withDefaultTimeZone[T](newDefaultTimeZone: TimeZone)(block: => T): T = {
val originalDefaultTimeZone = TimeZone.getDefault
try {
DateTimeUtils.resetThreadLocals()
TimeZone.setDefault(newDefaultTimeZone)
block
} finally {
TimeZone.setDefault(originalDefaultTimeZone)
DateTimeUtils.resetThreadLocals()
}
}
}
Example 43
| Source File: DerivedFunctionTest.scala From maha with Apache License 2.0 | 5 votes |
|
package com.yahoo.maha.core
import java.util.TimeZone
import com.yahoo.maha.core.DruidDerivedFunction._
import org.joda.time.DateTimeZone
import org.json4s.JObject
import org.scalatest.{FunSuiteLike, Matchers}
class DerivedFunctionTest extends FunSuiteLike with Matchers {
test("Create a DECODE_DIM failure cases") {
val minLengthCatch = intercept[IllegalArgumentException] {
new DECODE_DIM("fieldName", "tooFewArgs")
}
assert(minLengthCatch.getMessage.contains("Usage: DECODE( expression , search , result [, search , result]... [, default] )"))
}
test("Create value DECODE_DIM") {
val newDecode = new DECODE_DIM("fieldName", "arg1", "decodeVal1", "arg2", "decodeVal2", "default")
val newDecodeWithoutDefault = new DECODE_DIM("fieldName", "arg1", "decodeVal1", "arg2", "decodeVal2")
assert(newDecode.apply("arg1") == Some("decodeVal1"))
assert(newDecode.apply("arg3") == Some("default"))
assert(newDecodeWithoutDefault.apply("arg3") == None)
assert(newDecode.apply.isDefinedAt("arg20"))
}
test("Attempt LOOKUP_WITH_DECODE fail") {
val minLengthCatch = intercept[IllegalArgumentException] {
new LOOKUP_WITH_DECODE("fieldNameSpace", "valueCol", dimensionOverrideMap = Map.empty, "tooFewArgs")
}
assert(minLengthCatch.getMessage.contains("Usage: DECODE( expression , search , result [, search , result]... [, default] )"))
}
test("Failure to get interval date with blank format") {
val thrown = intercept[IllegalArgumentException]{
GET_INTERVAL_DATE.checkFormat("")
}
assert(thrown.getMessage.contains("Format for get_interval_date must be d|w|m|day|yr not"))
}
test("All Derived Functions should generate proper JSON Strings.") {
val gid = GET_INTERVAL_DATE("fieldName", "yyyyMMdd")
val dow = DAY_OF_WEEK("fieldName")
val dtf = DATETIME_FORMATTER("fieldName", 0, 10)
val dd = DECODE_DIM("fieldName", "arg1", "decodeVal1", "arg2", "decodeVal2", "default")
val js = JAVASCRIPT("fieldName", "function(x) { return x > 0; }")
val rgx = REGEX("fieldName", "blah", 0, true, "t")
val lu = LOOKUP("namespace", "val", Map("a" -> "b"))
val lwd = LOOKUP_WITH_DECODE("namespace", "valCol", Map("b" -> "a"), "arg1", "decodeVal1", "arg2", "decodeVal2", "default")
val lwe = LOOKUP_WITH_EMPTY_VALUE_OVERRIDE("namespace", "valCol", "ovr", Map("c" -> "d"))
val lwo = LOOKUP_WITH_DECODE_ON_OTHER_COLUMN("namespace", "valCol", "valToCheck", "valIfMatched", "valIfNot", Map("2" -> "4", "b" -> "a"))
val ltf = LOOKUP_WITH_TIMEFORMATTER("namespace", "valCol", "yyyyMMdd", "yyyy", Map("do" -> "dont"), Some("override"))
val ldr = LOOKUP_WITH_DECODE_RETAIN_MISSING_VALUE("namespace", "valCol", true, true, Map("rtn" -> "not"), "arg1", "decodeVal1", "arg2", "decodeVal2", "default")
val dtz = DRUID_TIME_FORMAT("format", DateTimeZone.forID("Asia/Jakarta"))
val dpg = DRUID_TIME_FORMAT_WITH_PERIOD_GRANULARITY("format", "P1D", DateTimeZone.forID("Asia/Jakarta"))
val rc = TIME_FORMAT_WITH_REQUEST_CONTEXT("yyyy")
val lwt = LOOKUP_WITH_TIMESTAMP("namespace", "val", "fmt", Map.empty, Some("ovrVal"), asMillis = false)
val resultArray = List(gid, dow, dtf, dd, js, rgx, lu, lwd, lwe, lwo, ltf, ldr, dtz, dpg, rc, lwt)
val expectedJSONs = List(
"""{"function_type":"GET_INTERVAL_DATE","fieldName":"fieldName","format":"yyyyMMdd"}""",
"""{"function_type":"DAY_OF_WEEK","fieldName":"fieldName"}""",
"""{"function_type":"DATETIME_FORMATTER","fieldName":"fieldName","index":0,"length":10}""",
"""{"function_type":"DECODE_DIM","fieldName":"fieldName","args":"arg1,decodeVal1,arg2,decodeVal2,default"}""",
"""{"function_type":"JAVASCRIPT","fieldName":"fieldName","function":"function(x) { return x > 0; }"}""",
"""{"function_type":"REGEX","fieldName":"fieldName","expr":"blah","index":0,"replaceMissingValue":true,"replaceMissingValueWith":"t"}""",
"""{"function_type":"LOOKUP","lookupNamespace":"namespace","valueColumn":"val","dimensionOverrideMap":{"a":"b"}}""",
"""{"function_type":"LOOKUP_WITH_DECODE","lookupNamespace":"namespace","valueColumn":"valCol","dimensionOverrideMap":{"b":"a"},"args":"arg1,decodeVal1,arg2,decodeVal2,default"}""",
"""{"function_type":"LOOKUP_WITH_EMPTY_VALUE_OVERRIDE","lookupNamespace":"namespace","valueColumn":"valCol","overrideValue":"ovr","dimensionOverrideMap":{"c":"d"}}""",
"""{"function_type":"LOOKUP_WITH_DECODE_ON_OTHER_COLUMN","lookupNamespace":"namespace","columnToCheck":"valCol","valueToCheck":"valToCheck","columnIfValueMatched":"valIfMatched","columnIfValueNotMatched":"valIfNot","dimensionOverrideMap":{"2":"4","b":"a"}}""",
"""{"function_type":"LOOKUP_WITH_TIMEFORMATTER","lookupNamespace":"namespace","valueColumn":"valCol","inputFormat":"yyyyMMdd","resultFormat":"yyyy","dimensionOverrideMap":{"do":"dont"}}""",
"""{"function_type":"LOOKUP_WITH_DECODE_RETAIN_MISSING_VALUE","lookupNamespace":"namespace","valueColumn":"valCol","retainMissingValue":true,"injective":true,"dimensionOverrideMap":{"rtn":"not"},"args":"arg1,decodeVal1,arg2,decodeVal2,default"}""",
"""{"function_type":"DRUID_TIME_FORMAT","format":"format","zone":"Asia/Jakarta"}""",
"""{"function_type":"DRUID_TIME_FORMAT_WITH_PERIOD_GRANULARITY","format":"format","period":"P1D","zone":"Asia/Jakarta"}""",
"""{"function_type":"TIME_FORMAT_WITH_REQUEST_CONTEXT","format":"yyyy"}""",
"""{"function_type":"LOOKUP_WITH_TIMESTAMP","lookupNamespace":"namespace","valueColumn":"val","resultFormat":"fmt","dimensionOverrideMap":{},"overrideValue":"ovrVal","asMillis":false}"""
)
import org.json4s._
import org.json4s.jackson.JsonMethods._
implicit val formats = DefaultFormats
val allJSONs: List[JObject] = resultArray.map(expn => expn.asJSON)
val allJsonStrings: List[String] = allJSONs.map(json => compact(json))
assert(allJsonStrings.forall(str => expectedJSONs.contains(str)))
}
}
Example 44
| Source File: OapQuerySuite.scala From OAP with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive.execution
import java.util.{Locale, TimeZone}
import org.scalatest.{BeforeAndAfter, Ignore}
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.hive.HiveUtils
import org.apache.spark.sql.hive.test.TestHive
import org.apache.spark.sql.internal.SQLConf
// Ignore because in separate package will encounter problem with shaded spark source.
@Ignore
class OapQuerySuite extends HiveComparisonTest with BeforeAndAfter {
private lazy val originalTimeZone = TimeZone.getDefault
private lazy val originalLocale = Locale.getDefault
import org.apache.spark.sql.hive.test.TestHive._
// Note: invoke TestHive will create a SparkContext which can't be configured by us.
// So be careful this may affect current using SparkContext and cause strange problem.
private lazy val originalCrossJoinEnabled = TestHive.conf.crossJoinEnabled
override def beforeAll() {
super.beforeAll()
TestHive.setCacheTables(true)
// Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
// Add Locale setting
Locale.setDefault(Locale.US)
// Ensures that cross joins are enabled so that we can test them
TestHive.setConf(SQLConf.CROSS_JOINS_ENABLED, true)
TestHive.setConf(HiveUtils.CONVERT_METASTORE_PARQUET, true)
}
override def afterAll() {
try {
TestHive.setCacheTables(false)
TimeZone.setDefault(originalTimeZone)
Locale.setDefault(originalLocale)
sql("DROP TEMPORARY FUNCTION IF EXISTS udtf_count2")
TestHive.setConf(SQLConf.CROSS_JOINS_ENABLED, originalCrossJoinEnabled)
} finally {
super.afterAll()
}
}
private def assertDupIndex(body: => Unit): Unit = {
val e = intercept[AnalysisException] { body }
assert(e.getMessage.toLowerCase.contains("exists"))
}
test("create hive table in parquet format") {
try {
sql("create table p_table (key int, val string) stored as parquet")
sql("insert overwrite table p_table select * from src")
sql("create oindex if not exists p_index on p_table(key)")
assert(sql("select val from p_table where key = 238")
.collect().head.getString(0) == "val_238")
} finally {
sql("drop oindex p_index on p_table")
sql("drop table p_table")
}
}
test("create duplicate hive table in parquet format") {
try {
sql("create table p_table1 (key int, val string) stored as parquet")
sql("insert overwrite table p_table1 select * from src")
sql("create oindex p_index on p_table1(key)")
assertDupIndex { sql("create oindex p_index on p_table1(key)") }
} finally {
sql("drop oindex p_index on p_table1")
}
}
}
Example 45
| Source File: SLF4JSpec.scala From scribe with MIT License | 5 votes |
|
package spec
import java.util.TimeZone
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AnyWordSpec
import org.slf4j.{LoggerFactory, MDC}
import scribe.handler.LogHandler
import scribe.output.LogOutput
import scribe.util.Time
import scribe.writer.Writer
import scribe.{Level, LogRecord, Logger}
class SLF4JSpec extends AnyWordSpec with Matchers {
TimeZone.setDefault(TimeZone.getTimeZone("UTC"))
private var logs: List[LogRecord[_]] = Nil
private var logOutput: List[String] = Nil
private val recordHolder = LogHandler.default.withMinimumLevel(Level.Info).withWriter(new Writer {
override def write[M](record: LogRecord[M], output: LogOutput): Unit = {
logs = record :: logs
logOutput = output.plainText :: logOutput
}
})
"SLF4J" should {
"set the time to an arbitrary value" in {
Time.function = () => 1542376191920L
}
"remove existing handlers from Root" in {
Logger.root.clearHandlers().replace()
}
"add a testing handler" in {
Logger.root.withHandler(recordHolder).replace()
}
"verify not records are in the RecordHolder" in {
logs.isEmpty should be(true)
}
"log to Scribe" in {
val logger = LoggerFactory.getLogger(getClass)
logger.info("Hello World!")
}
"verify Scribe received the record" in {
logs.size should be(1)
val r = logs.head
r.level should be(Level.Info)
r.message.plainText should be("Hello World!")
r.className should be("spec.SLF4JSpec")
logs = Nil
}
"verify Scribe wrote value" in {
logOutput.size should be(1)
val s = logOutput.head
s should be("2018.11.16 13:49:51 [INFO] spec.SLF4JSpec - Hello World!")
}
"use MDC" in {
MDC.put("name", "John Doe")
val logger = LoggerFactory.getLogger(getClass)
logger.info("A generic name")
logOutput.head should be("2018.11.16 13:49:51 [INFO] spec.SLF4JSpec - A generic name (name: John Doe)")
}
"clear MDC" in {
MDC.clear()
val logger = LoggerFactory.getLogger(getClass)
logger.info("MDC cleared")
logOutput.head should be("2018.11.16 13:49:51 [INFO] spec.SLF4JSpec - MDC cleared")
}
"make sure logging nulls doesn't error" in {
val logger = LoggerFactory.getLogger(getClass)
logger.error(null)
logs.length should be(3)
logOutput.head should be("2018.11.16 13:49:51 [ERROR] spec.SLF4JSpec - null")
}
}
}
Example 46
| Source File: DateTimeTestUtils.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.catalyst.util
import java.util.TimeZone
object DateTimeTestUtils {
val ALL_TIMEZONES: Seq[TimeZone] = TimeZone.getAvailableIDs.toSeq.map(TimeZone.getTimeZone)
def withDefaultTimeZone[T](newDefaultTimeZone: TimeZone)(block: => T): T = {
val originalDefaultTimeZone = TimeZone.getDefault
try {
DateTimeUtils.resetThreadLocals()
TimeZone.setDefault(newDefaultTimeZone)
block
} finally {
TimeZone.setDefault(originalDefaultTimeZone)
DateTimeUtils.resetThreadLocals()
}
}
}
Example 47
| Source File: UIUtilsSuite.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.ui
import java.util.TimeZone
import java.util.concurrent.TimeUnit
import org.scalatest.Matchers
import org.apache.spark.SparkFunSuite
class UIUtilsSuite extends SparkFunSuite with Matchers{
test("shortTimeUnitString") {
assert("ns" === UIUtils.shortTimeUnitString(TimeUnit.NANOSECONDS))
assert("us" === UIUtils.shortTimeUnitString(TimeUnit.MICROSECONDS))
assert("ms" === UIUtils.shortTimeUnitString(TimeUnit.MILLISECONDS))
assert("sec" === UIUtils.shortTimeUnitString(TimeUnit.SECONDS))
assert("min" === UIUtils.shortTimeUnitString(TimeUnit.MINUTES))
assert("hrs" === UIUtils.shortTimeUnitString(TimeUnit.HOURS))
assert("days" === UIUtils.shortTimeUnitString(TimeUnit.DAYS))
}
test("normalizeDuration") {
verifyNormalizedTime(900, TimeUnit.MILLISECONDS, 900)
verifyNormalizedTime(1.0, TimeUnit.SECONDS, 1000)
verifyNormalizedTime(1.0, TimeUnit.MINUTES, 60 * 1000)
verifyNormalizedTime(1.0, TimeUnit.HOURS, 60 * 60 * 1000)
verifyNormalizedTime(1.0, TimeUnit.DAYS, 24 * 60 * 60 * 1000)
}
private def verifyNormalizedTime(
expectedTime: Double, expectedUnit: TimeUnit, input: Long): Unit = {
val (time, unit) = UIUtils.normalizeDuration(input)
time should be (expectedTime +- 1E-6)
unit should be (expectedUnit)
}
test("convertToTimeUnit") {
verifyConvertToTimeUnit(60.0 * 1000 * 1000 * 1000, 60 * 1000, TimeUnit.NANOSECONDS)
verifyConvertToTimeUnit(60.0 * 1000 * 1000, 60 * 1000, TimeUnit.MICROSECONDS)
verifyConvertToTimeUnit(60 * 1000, 60 * 1000, TimeUnit.MILLISECONDS)
verifyConvertToTimeUnit(60, 60 * 1000, TimeUnit.SECONDS)
verifyConvertToTimeUnit(1, 60 * 1000, TimeUnit.MINUTES)
verifyConvertToTimeUnit(1.0 / 60, 60 * 1000, TimeUnit.HOURS)
verifyConvertToTimeUnit(1.0 / 60 / 24, 60 * 1000, TimeUnit.DAYS)
}
private def verifyConvertToTimeUnit(
expectedTime: Double, milliseconds: Long, unit: TimeUnit): Unit = {
val convertedTime = UIUtils.convertToTimeUnit(milliseconds, unit)
convertedTime should be (expectedTime +- 1E-6)
}
test("formatBatchTime") {
val tzForTest = TimeZone.getTimeZone("America/Los_Angeles")
val batchTime = 1431637480452L // Thu May 14 14:04:40 PDT 2015
assert("2015/05/14 14:04:40" === UIUtils.formatBatchTime(batchTime, 1000, timezone = tzForTest))
assert("2015/05/14 14:04:40.452" ===
UIUtils.formatBatchTime(batchTime, 999, timezone = tzForTest))
assert("14:04:40" === UIUtils.formatBatchTime(batchTime, 1000, false, timezone = tzForTest))
assert("14:04:40.452" === UIUtils.formatBatchTime(batchTime, 999, false, timezone = tzForTest))
}
}
Example 48
| Source File: SimpleDateParam.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.status.api.v1
import java.text.{ParseException, SimpleDateFormat}
import java.util.{Locale, TimeZone}
import javax.ws.rs.WebApplicationException
import javax.ws.rs.core.Response
import javax.ws.rs.core.Response.Status
private[v1] class SimpleDateParam(val originalValue: String) {
val timestamp: Long = {
val format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz", Locale.US)
try {
format.parse(originalValue).getTime()
} catch {
case _: ParseException =>
val gmtDay = new SimpleDateFormat("yyyy-MM-dd", Locale.US)
gmtDay.setTimeZone(TimeZone.getTimeZone("GMT"))
try {
gmtDay.parse(originalValue).getTime()
} catch {
case _: ParseException =>
throw new WebApplicationException(
Response
.status(Status.BAD_REQUEST)
.entity("Couldn't parse date: " + originalValue)
.build()
)
}
}
}
}
Example 49
| Source File: PlainText.scala From eidos with Apache License 2.0 | 5 votes |
|
package org.clulab.wm.eidos.utils.meta
import java.text.SimpleDateFormat
import java.util.Calendar
import java.util.TimeZone
import org.clulab.wm.eidos.utils.EidosException
import org.clulab.timenorm.scate.SimpleInterval
import org.clulab.wm.eidos.context.DCT
import org.clulab.wm.eidos.document.Metadata
class PlainText(text: String,
titleOpt: Option[String] = None,
idOpt: Option[String] = None,
dateOpt: Option[String] = None,
locationOpt: Option[String] = None
) extends EidosText {
protected val metadata = {
val dctOpt: Option[DCT] = {
dateOpt.map { date =>
val calendar = try {
val parsed = PlainText.dateFormat.parse(date)
val calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"))
calendar.setTime(parsed)
calendar
}
catch {
case throwable: Throwable =>
throw new EidosException(s"""Could not decipher "${date}" as a date""", throwable)
}
val simpleInterval = SimpleInterval.of(calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH) + 1, calendar.get(Calendar.DAY_OF_MONTH))
DCT(simpleInterval, date)
}
}
new Metadata(dctOpt, idOpt, titleOpt, locationOpt)
}
def getText: String = text
def getMetadata: Metadata = metadata
}
object PlainText {
protected val dateFormat: SimpleDateFormat = {
val dateFormat = new SimpleDateFormat("yyyy-MM-dd")
val timeZone = TimeZone.getTimeZone("UTC")
dateFormat.setTimeZone(timeZone)
dateFormat
}
}
Example 50
| Source File: httpserverplugin_staticfile.scala From scalabpe with Apache License 2.0 | 5 votes |
|
package scalabpe.plugin.http
import java.io.File
import java.net.URLEncoder
import java.text.SimpleDateFormat
import java.util.Calendar
import java.util.GregorianCalendar
import java.util.Locale
import java.util.TimeZone
import scala.collection.mutable.HashMap
import org.jboss.netty.handler.codec.http.HttpHeaders
import scalabpe.core.HashMapStringAny
class StaticFilePlugin extends HttpServerPlugin with HttpServerStaticFilePlugin {
val ETAG_TAG = "etag"
val EXPIRE_TAG = "expire"
val ATTACHMENT = "attachment"
val FILENAME = "filename"
val HTTP_DATE_FORMAT = "EEE, dd MMM yyyy HH:mm:ss zzz";
val HTTP_DATE_GMT_TIMEZONE = "GMT";
val df_tl = new ThreadLocal[SimpleDateFormat]() {
override def initialValue(): SimpleDateFormat = {
val df = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US)
df.setTimeZone(TimeZone.getTimeZone(HTTP_DATE_GMT_TIMEZONE));
df
}
}
def generateStaticFile(serviceId: Int, msgId: Int, errorCode: Int, errorMessage: String, body: HashMapStringAny, pluginParam: String, headers: HashMap[String, String]): String = {
if (body.ns(FILENAME) == "") {
return null
}
val filename = body.ns(FILENAME)
if (!new File(filename).exists()) {
return null
}
if (body.ns(ETAG_TAG) != "") {
headers.put("ETag", body.ns(ETAG_TAG))
}
if (body.ns(EXPIRE_TAG) != "") {
body.i(EXPIRE_TAG) match {
case 0 | -1 =>
headers.put(HttpHeaders.Names.CACHE_CONTROL, "no-cache")
case n => // seconds
val time = new GregorianCalendar();
time.add(Calendar.SECOND, n);
headers.put(HttpHeaders.Names.EXPIRES, df_tl.get.format(time.getTime()));
headers.put(HttpHeaders.Names.CACHE_CONTROL, "max-age=" + n);
}
}
val ext = parseExt(filename)
if (ext != "")
body.put("__file_ext__", ext)
if (body.ns(ATTACHMENT, "1") == "1") {
val filename = body.ns(FILENAME)
val v = "attachment; filename=\"%s\"".format(URLEncoder.encode(parseFilename(filename), "UTF-8"))
headers.put("Content-Disposition", v)
}
filename
}
def parseFilename(name: String): String = {
val p = name.lastIndexOf("/")
if (p < 0) return name
name.substring(p + 1)
}
def parseExt(name: String): String = {
val p = name.lastIndexOf(".")
if (p < 0) return ""
name.substring(p + 1).toLowerCase()
}
}
Example 51
| Source File: StringToTimestampParser.scala From bandar-log with Apache License 2.0 | 5 votes |
|
package com.aol.one.dwh.infra.parser
import java.text.{DateFormat, SimpleDateFormat}
import java.util.TimeZone
import com.aol.one.dwh.infra.util.{ExceptionPrinter, LogTrait}
import scala.util.control.NonFatal
import scala.util.{Failure, Try}
object StringToTimestampParser extends LogTrait with ExceptionPrinter {
def parse(value: String, format: String): Option[Long] = {
Try {
val dateFormat: DateFormat = new SimpleDateFormat(format)
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"))
dateFormat.parse(value).getTime
}.recoverWith {
case NonFatal(e) =>
logger.error(s"Could not parse value:[$value] using format:[$format]. Catching exception {}", e.getStringStackTrace)
Failure(e)
}.toOption
}
}
Example 52
| Source File: AnnotatorParam.scala From spark-nlp with Apache License 2.0 | 5 votes |
|
package com.johnsnowlabs.nlp.annotators.param
import java.util.{Date, TimeZone}
import org.apache.spark.ml.param.Param
import org.apache.spark.ml.util.Identifiable
import org.json4s._
import org.json4s.jackson.JsonMethods._
import org.json4s.jackson.Serialization.write
object SerializableFormat extends Formats with Serializable {
class SerializableDateFormat extends DateFormat {
def timezone: TimeZone = throw new Exception("SerializableFormat does not implement dateformat")
override def format(d: Date): String = throw new Exception("SerializableFormat does not implement dateformat")
override def parse(s: String): Option[Date] = throw new Exception("SerializableFormat does not implement dateformat")
}
override def dateFormat: DateFormat = new SerializableDateFormat
}
implicit val formats = SerializableFormat
override def jsonEncode(value: A): String = write(value.serialize)
override def jsonDecode(json: String): A = parse(json).extract[B].deserialize
}
Example 53
| Source File: TimeBasedDataService.scala From kafka-jdbc-connector with Apache License 2.0 | 5 votes |
|
package com.agoda.kafka.connector.jdbc.services
import java.sql.{Connection, PreparedStatement, ResultSet, Timestamp}
import java.util.{Date, GregorianCalendar, TimeZone}
import com.agoda.kafka.connector.jdbc.JdbcSourceConnectorConstants
import com.agoda.kafka.connector.jdbc.models.DatabaseProduct
import com.agoda.kafka.connector.jdbc.models.DatabaseProduct.{MsSQL, MySQL}
import com.agoda.kafka.connector.jdbc.models.Mode.TimestampMode
import com.agoda.kafka.connector.jdbc.utils.DataConverter
import org.apache.kafka.connect.data.Schema
import org.apache.kafka.connect.source.SourceRecord
import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
import scala.util.Try
case class TimeBasedDataService(databaseProduct: DatabaseProduct,
storedProcedureName: String,
batchSize: Int,
batchSizeVariableName: String,
timestampVariableName: String,
var timestampOffset: Long,
timestampFieldName: String,
topic: String,
keyFieldOpt: Option[String],
dataConverter: DataConverter,
calendar: GregorianCalendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"))
) extends DataService {
override def createPreparedStatement(connection: Connection): Try[PreparedStatement] = Try {
val preparedStatement = databaseProduct match {
case MsSQL => connection.prepareStatement(s"EXECUTE $storedProcedureName @$timestampVariableName = ?, @$batchSizeVariableName = ?")
case MySQL => connection.prepareStatement(s"CALL $storedProcedureName (@$timestampVariableName := ?, @$batchSizeVariableName := ?)")
}
preparedStatement.setTimestamp(1, new Timestamp(timestampOffset), calendar)
preparedStatement.setObject(2, batchSize)
preparedStatement
}
override def extractRecords(resultSet: ResultSet, schema: Schema): Try[Seq[SourceRecord]] = Try {
val sourceRecords = ListBuffer.empty[SourceRecord]
var max = timestampOffset
while (resultSet.next()) {
dataConverter.convertRecord(schema, resultSet) map { record =>
val time = record.get(timestampFieldName).asInstanceOf[Date].getTime
max = if(time > max) {
keyFieldOpt match {
case Some(keyField) =>
sourceRecords += new SourceRecord(
Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> storedProcedureName).asJava,
Map(TimestampMode.entryName -> time).asJava, topic, null, schema, record.get(keyField), schema, record
)
case None =>
sourceRecords += new SourceRecord(
Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> storedProcedureName).asJava,
Map(TimestampMode.entryName -> time).asJava, topic, schema, record
)
}
time
} else max
}
}
timestampOffset = max
sourceRecords
}
override def toString: String = {
s"""
|{
| "name" : "${this.getClass.getSimpleName}"
| "mode" : "${TimestampMode.entryName}"
| "stored-procedure.name" : "$storedProcedureName"
|}
""".stripMargin
}
}
Example 54
| Source File: PlanningTest.scala From spark-druid-olap with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.sources.druid.test
import java.util.TimeZone
import com.github.nscala_time.time.Imports._
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.catalyst.expressions.{Expression, PredicateHelper}
import org.apache.spark.sql.catalyst.plans.logical.Filter
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.hive.test.sparklinedata.TestHive
import org.apache.spark.sql.sources.druid.DruidPlanner
import org.scalatest.BeforeAndAfterEach
import org.sparklinedata.druid._
import org.sparklinedata.druid.client.test.BaseTest
import org.sparklinedata.druid.metadata.DruidRelationInfo
trait PlanningTestHelper extends PredicateHelper {
System.setProperty("user.timezone", "UTC")
TimeZone.setDefault(TimeZone.getTimeZone("UTC"))
override def splitConjunctivePredicates(condition: Expression): Seq[Expression] = {
super.splitConjunctivePredicates(condition)
}
}
abstract class PlanningTest extends BaseTest with BeforeAndAfterEach with PlanningTestHelper {
val dPlanner = new DruidPlanner(TestHive)
var tab: DataFrame = _
var drInfo: DruidRelationInfo = _
var dqb: DruidQueryBuilder = _
var iCE: IntervalConditionExtractor = _
var iCE2: SparkIntervalConditionExtractor = _
override def beforeAll() = {
super.beforeAll()
tab = TestHive.table("orderLineItemPartSupplier")
drInfo = tab.queryExecution.optimizedPlan.
asInstanceOf[LogicalRelation].relation.asInstanceOf[DruidRelation].info
}
override protected def beforeEach(): Unit = {
dqb = DruidQueryBuilder(drInfo)
iCE = new IntervalConditionExtractor(dqb)
iCE2 = new SparkIntervalConditionExtractor(dqb)
}
def validateFilter(filterStr: String,
pushedToDruid: Boolean = true,
filSpec: Option[FilterSpec] = None,
intervals: List[Interval] = List()
): Unit = {
val q = tab.where(filterStr)
val filter = q.queryExecution.optimizedPlan.asInstanceOf[Filter]
val dqbs = dPlanner.translateProjectFilter(
Some(dqb),
Seq(),
splitConjunctivePredicates(filter.condition),
true
)
if (pushedToDruid) {
assert(dqbs.size == 1)
val odqb = dqbs(0)
assert(odqb.filterSpec == filSpec)
assert(odqb.queryIntervals.intervals == intervals)
}
}
}
Example 55
| Source File: UIUtilsSuite.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.streaming.ui
import java.util.TimeZone
import java.util.concurrent.TimeUnit
import org.scalatest.Matchers
import org.apache.spark.SparkFunSuite
class UIUtilsSuite extends SparkFunSuite with Matchers{
test("shortTimeUnitString") {
assert("ns" === UIUtils.shortTimeUnitString(TimeUnit.NANOSECONDS))
assert("us" === UIUtils.shortTimeUnitString(TimeUnit.MICROSECONDS))
assert("ms" === UIUtils.shortTimeUnitString(TimeUnit.MILLISECONDS))
assert("sec" === UIUtils.shortTimeUnitString(TimeUnit.SECONDS))
assert("min" === UIUtils.shortTimeUnitString(TimeUnit.MINUTES))
assert("hrs" === UIUtils.shortTimeUnitString(TimeUnit.HOURS))
assert("days" === UIUtils.shortTimeUnitString(TimeUnit.DAYS))
}
test("normalizeDuration") {
verifyNormalizedTime(900, TimeUnit.MILLISECONDS, 900)
verifyNormalizedTime(1.0, TimeUnit.SECONDS, 1000)
verifyNormalizedTime(1.0, TimeUnit.MINUTES, 60 * 1000)
verifyNormalizedTime(1.0, TimeUnit.HOURS, 60 * 60 * 1000)
verifyNormalizedTime(1.0, TimeUnit.DAYS, 24 * 60 * 60 * 1000)
}
private def verifyNormalizedTime(
expectedTime: Double, expectedUnit: TimeUnit, input: Long): Unit = {
val (time, unit) = UIUtils.normalizeDuration(input)
time should be (expectedTime +- 1E-6)
unit should be (expectedUnit)
}
test("convertToTimeUnit") {
verifyConvertToTimeUnit(60.0 * 1000 * 1000 * 1000, 60 * 1000, TimeUnit.NANOSECONDS)
verifyConvertToTimeUnit(60.0 * 1000 * 1000, 60 * 1000, TimeUnit.MICROSECONDS)
verifyConvertToTimeUnit(60 * 1000, 60 * 1000, TimeUnit.MILLISECONDS)
verifyConvertToTimeUnit(60, 60 * 1000, TimeUnit.SECONDS)
verifyConvertToTimeUnit(1, 60 * 1000, TimeUnit.MINUTES)
verifyConvertToTimeUnit(1.0 / 60, 60 * 1000, TimeUnit.HOURS)
verifyConvertToTimeUnit(1.0 / 60 / 24, 60 * 1000, TimeUnit.DAYS)
}
private def verifyConvertToTimeUnit(
expectedTime: Double, milliseconds: Long, unit: TimeUnit): Unit = {
val convertedTime = UIUtils.convertToTimeUnit(milliseconds, unit)
convertedTime should be (expectedTime +- 1E-6)
}
test("formatBatchTime") {
val tzForTest = TimeZone.getTimeZone("America/Los_Angeles")
val batchTime = 1431637480452L // Thu May 14 14:04:40 PDT 2015
assert("2015/05/14 14:04:40" === UIUtils.formatBatchTime(batchTime, 1000, timezone = tzForTest))
assert("2015/05/14 14:04:40.452" ===
UIUtils.formatBatchTime(batchTime, 999, timezone = tzForTest))
assert("14:04:40" === UIUtils.formatBatchTime(batchTime, 1000, false, timezone = tzForTest))
assert("14:04:40.452" === UIUtils.formatBatchTime(batchTime, 999, false, timezone = tzForTest))
}
}
Example 56
| Source File: DateTimeTestUtils.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.catalyst.util
import java.util.TimeZone
object DateTimeTestUtils {
val ALL_TIMEZONES: Seq[TimeZone] = TimeZone.getAvailableIDs.toSeq.map(TimeZone.getTimeZone)
def withDefaultTimeZone[T](newDefaultTimeZone: TimeZone)(block: => T): T = {
val originalDefaultTimeZone = TimeZone.getDefault
try {
DateTimeUtils.resetThreadLocals()
TimeZone.setDefault(newDefaultTimeZone)
block
} finally {
TimeZone.setDefault(originalDefaultTimeZone)
DateTimeUtils.resetThreadLocals()
}
}
}
Example 57
| Source File: DateTimeTestUtils.scala From drizzle-spark with Apache License 2.0 | 4 votes |
|
package org.apache.spark.sql.catalyst.util
import java.util.TimeZone
object DateTimeTestUtils {
val ALL_TIMEZONES: Seq[TimeZone] = TimeZone.getAvailableIDs.toSeq.map(TimeZone.getTimeZone)
def withDefaultTimeZone[T](newDefaultTimeZone: TimeZone)(block: => T): T = {
val originalDefaultTimeZone = TimeZone.getDefault
try {
DateTimeUtils.resetThreadLocals()
TimeZone.setDefault(newDefaultTimeZone)
block
} finally {
TimeZone.setDefault(originalDefaultTimeZone)
DateTimeUtils.resetThreadLocals()
}
}
}
Example 58
| Source File: RollingFileLogger.scala From odin with Apache License 2.0 | 4 votes |
|
package io.odin.loggers
import java.nio.file.{Files, Path, Paths}
import java.time.{Instant, LocalDateTime}
import java.time.format.DateTimeFormatter
import java.util.TimeZone
import java.util.concurrent.TimeUnit
import cats.Monad
import cats.effect.concurrent.Ref
import cats.effect.{Concurrent, ContextShift, Fiber, Resource, Timer}
import cats.syntax.all._
import io.odin.formatter.Formatter
import io.odin.{Level, Logger, LoggerMessage}
import scala.concurrent.duration.{FiniteDuration, _}
object RollingFileLogger {
def apply[F[_]](
fileNamePattern: LocalDateTime => String,
maxFileSizeInBytes: Option[Long],
rolloverInterval: Option[FiniteDuration],
formatter: Formatter,
minLevel: Level
)(implicit F: Concurrent[F], timer: Timer[F], cs: ContextShift[F]): Resource[F, Logger[F]] = {
new RollingFileLoggerFactory(
fileNamePattern,
maxFileSizeInBytes,
rolloverInterval,
formatter,
minLevel,
FileLogger.apply[F]
).mk
}
private[odin] class RefLogger[F[_]: Timer: Monad](
current: Ref[F, Logger[F]],
override val minLevel: Level
) extends DefaultLogger[F](minLevel) {
def log(msg: LoggerMessage): F[Unit] = current.get.flatMap(_.log(msg))
override def log(msgs: List[LoggerMessage]): F[Unit] = current.get.flatMap(_.log(msgs))
}
private[odin] class RollingFileLoggerFactory[F[_]](
fileNamePattern: LocalDateTime => String,
maxFileSizeInBytes: Option[Long],
rolloverInterval: Option[FiniteDuration],
formatter: Formatter,
minLevel: Level,
underlyingLogger: (String, Formatter, Level) => Resource[F, Logger[F]],
fileSizeCheck: Path => Long = Files.size
)(implicit F: Concurrent[F], timer: Timer[F], cs: ContextShift[F]) {
val df: DateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss")
def mk: Resource[F, Logger[F]] = {
val logger = for {
((logger, watcherFiber), release) <- allocate.allocated
refLogger <- Ref.of(logger)
refRelease <- Ref.of(release)
_ <- F.start(rollingLoop(watcherFiber, refLogger, refRelease))
} yield {
(new RefLogger(refLogger, minLevel), refRelease)
}
Resource.make(logger)(_._2.get.flatten).map {
case (logger, _) => logger
}
}
def now: F[Long] = timer.clock.realTime(TimeUnit.MILLISECONDS)
def rollingLoop(watcher: Fiber[F, Unit], logger: Ref[F, Logger[F]], release: Ref[F, F[Unit]]): F[Unit] =
for {
_ <- watcher.join
oldRelease <- release.get
((newLogger, newWatcher), newRelease) <- allocate.allocated
_ <- logger.set(newLogger)
_ <- release.set(newRelease)
_ <- oldRelease
_ <- rollingLoop(newWatcher, logger, release)
} yield ()
}
}
Example 59
| Source File: StreamingProducer.scala From Scala-Programming-Projects with MIT License | 4 votes |
|
package coinyser
import java.sql.Timestamp
import java.text.SimpleDateFormat
import java.util.TimeZone
import cats.effect.IO
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.pusher.client.Client
import com.pusher.client.channel.SubscriptionEventListener
import com.typesafe.scalalogging.StrictLogging
object StreamingProducer extends StrictLogging {
def subscribe(pusher: Client)(onTradeReceived: String => Unit): IO[Unit] =
for {
_ <- IO(pusher.connect())
channel <- IO(pusher.subscribe("live_trades"))
_ <- IO(channel.bind("trade", new SubscriptionEventListener() {
override def onEvent(channel: String, event: String, data: String): Unit = {
logger.info(s"Received event: $event with data: $data")
onTradeReceived(data)
}
}))
} yield ()
val mapper: ObjectMapper = {
val m = new ObjectMapper()
m.registerModule(DefaultScalaModule)
val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
// Very important: the storage must be in UTC
sdf.setTimeZone(TimeZone.getTimeZone("UTC"))
m.setDateFormat(sdf)
}
def deserializeWebsocketTransaction(s: String): WebsocketTransaction =
mapper.readValue(s, classOf[WebsocketTransaction])
def convertWsTransaction(wsTx: WebsocketTransaction): Transaction =
Transaction(
timestamp = new Timestamp(wsTx.timestamp.toLong * 1000), tid = wsTx.id,
price = wsTx.price, sell = wsTx.`type` == 1, amount = wsTx.amount)
def serializeTransaction(tx: Transaction): String =
mapper.writeValueAsString(tx)
}
