org.apache.spark.sql.test.SQLTestUtils Scala Examples
The following examples show how to use org.apache.spark.sql.test.SQLTestUtils.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
Example 1
| Source File: ParquetTest.scala From spark1.52 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.execution.datasources.parquet
import java.io.File
import scala.reflect.ClassTag
import scala.reflect.runtime.universe.TypeTag
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.{DataFrame, SaveMode, SQLContext}
protected def withParquetTable[T <: Product: ClassTag: TypeTag]
(data: Seq[T], tableName: String)
(f: => Unit): Unit = {
withParquetDataFrame(data) { df =>
//注册数据集Seq及临时表名
_sqlContext.registerDataFrameAsTable(df, tableName)
withTempTable(tableName)(f)
}
}
//产生Parquet文件
protected def makeParquetFile[T <: Product: ClassTag: TypeTag](
data: Seq[T], path: File): Unit = {
//当数据输出的位置已存在时,覆盖重写
_sqlContext.createDataFrame(data).write.mode(SaveMode.Overwrite).parquet(path.getCanonicalPath)
}
//产生Parquet文件
protected def makeParquetFile[T <: Product: ClassTag: TypeTag](
df: DataFrame, path: File): Unit = {
//当数据输出的位置已存在时,重写
df.write.mode(SaveMode.Overwrite).parquet(path.getCanonicalPath)
}
//产生分区目录
protected def makePartitionDir(
basePath: File,
defaultPartitionName: String,
partitionCols: (String, Any)*): File = {
val partNames = partitionCols.map { case (k, v) =>
val valueString = if (v == null || v == "") defaultPartitionName else v.toString
s"$k=$valueString"
}
val partDir = partNames.foldLeft(basePath) { (parent, child) =>
new File(parent, child)
}
assert(partDir.mkdirs(), s"Couldn't create directory $partDir")
partDir
}
}
Example 2
| Source File: OrcTest.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive.orc
import java.io.File
import scala.reflect.ClassTag
import scala.reflect.runtime.universe.TypeTag
import org.apache.spark.sql._
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
private[sql] trait OrcTest extends SQLTestUtils with TestHiveSingleton {
import testImplicits._
protected def withOrcTable[T <: Product: ClassTag: TypeTag]
(data: Seq[T], tableName: String)
(f: => Unit): Unit = {
withOrcDataFrame(data) { df =>
df.createOrReplaceTempView(tableName)
withTempView(tableName)(f)
}
}
protected def makeOrcFile[T <: Product: ClassTag: TypeTag](
data: Seq[T], path: File): Unit = {
data.toDF().write.mode(SaveMode.Overwrite).orc(path.getCanonicalPath)
}
protected def makeOrcFile[T <: Product: ClassTag: TypeTag](
df: DataFrame, path: File): Unit = {
df.write.mode(SaveMode.Overwrite).orc(path.getCanonicalPath)
}
}
Example 3
| Source File: MetastoreRelationSuite.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive
import org.apache.spark.sql.{QueryTest, Row}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType}
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
class MetastoreRelationSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
test("makeCopy and toJSON should work") {
val table = CatalogTable(
identifier = TableIdentifier("test", Some("db")),
tableType = CatalogTableType.VIEW,
storage = CatalogStorageFormat.empty,
schema = StructType(StructField("a", IntegerType, true) :: Nil))
val relation = MetastoreRelation("db", "test")(table, null)
// No exception should be thrown
relation.makeCopy(Array("db", "test"))
// No exception should be thrown
relation.toJSON
}
test("SPARK-17409: Do Not Optimize Query in CTAS (Hive Serde Table) More Than Once") {
withTable("bar") {
withTempView("foo") {
sql("select 0 as id").createOrReplaceTempView("foo")
// If we optimize the query in CTAS more than once, the following saveAsTable will fail
// with the error: `GROUP BY position 0 is not in select list (valid range is [1, 1])`
sql("CREATE TABLE bar AS SELECT * FROM foo group by id")
checkAnswer(spark.table("bar"), Row(0) :: Nil)
val tableMetadata = spark.sessionState.catalog.getTableMetadata(TableIdentifier("bar"))
assert(tableMetadata.provider == Some("hive"), "the expected table is a Hive serde table")
}
}
}
}
Example 4
| Source File: PruneFileSourcePartitionsSuite.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive.execution
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, LogicalRelation, PruneFileSourcePartitions}
import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.types.StructType
class PruneFileSourcePartitionsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
object Optimize extends RuleExecutor[LogicalPlan] {
val batches = Batch("PruneFileSourcePartitions", Once, PruneFileSourcePartitions) :: Nil
}
test("PruneFileSourcePartitions should not change the output of LogicalRelation") {
withTable("test") {
withTempDir { dir =>
sql(
s"""
|CREATE EXTERNAL TABLE test(i int)
|PARTITIONED BY (p int)
|STORED AS parquet
|LOCATION '${dir.getAbsolutePath}'""".stripMargin)
val tableMeta = spark.sharedState.externalCatalog.getTable("default", "test")
val catalogFileIndex = new CatalogFileIndex(spark, tableMeta, 0)
val dataSchema = StructType(tableMeta.schema.filterNot { f =>
tableMeta.partitionColumnNames.contains(f.name)
})
val relation = HadoopFsRelation(
location = catalogFileIndex,
partitionSchema = tableMeta.partitionSchema,
dataSchema = dataSchema,
bucketSpec = None,
fileFormat = new ParquetFileFormat(),
options = Map.empty)(sparkSession = spark)
val logicalRelation = LogicalRelation(relation, catalogTable = Some(tableMeta))
val query = Project(Seq('i, 'p), Filter('p === 1, logicalRelation)).analyze
val optimized = Optimize.execute(query)
assert(optimized.missingInput.isEmpty)
}
}
}
}
Example 5
| Source File: HiveUtilsSuite.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive
import org.apache.hadoop.hive.conf.HiveConf.ConfVars
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.QueryTest
class HiveUtilsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
test("newTemporaryConfiguration overwrites listener configurations") {
Seq(true, false).foreach { useInMemoryDerby =>
val conf = HiveUtils.newTemporaryConfiguration(useInMemoryDerby)
assert(conf(ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname) === "")
assert(conf(ConfVars.METASTORE_EVENT_LISTENERS.varname) === "")
assert(conf(ConfVars.METASTORE_END_FUNCTION_LISTENERS.varname) === "")
}
}
}
Example 6
| Source File: CommitFailureTestRelationSuite.scala From multi-tenancy-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.sources
import org.apache.hadoop.fs.Path
import org.apache.spark.SparkException
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.functions._
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
class CommitFailureTestRelationSuite extends SQLTestUtils with TestHiveSingleton {
// When committing a task, `CommitFailureTestSource` throws an exception for testing purpose.
val dataSourceName: String = classOf[CommitFailureTestSource].getCanonicalName
test("SPARK-7684: commitTask() failure should fallback to abortTask()") {
withTempPath { file =>
// Here we coalesce partition number to 1 to ensure that only a single task is issued. This
// prevents race condition happened when FileOutputCommitter tries to remove the `_temporary`
// directory while committing/aborting the job. See SPARK-8513 for more details.
val df = spark.range(0, 10).coalesce(1)
intercept[SparkException] {
df.write.format(dataSourceName).save(file.getCanonicalPath)
}
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
test("call failure callbacks before close writer - default") {
SimpleTextRelation.failCommitter = false
withTempPath { file =>
// fail the job in the middle of writing
val divideByZero = udf((x: Int) => { x / (x - 1)})
val df = spark.range(0, 10).coalesce(1).select(divideByZero(col("id")))
SimpleTextRelation.callbackCalled = false
intercept[SparkException] {
df.write.format(dataSourceName).save(file.getCanonicalPath)
}
assert(SimpleTextRelation.callbackCalled, "failure callback should be called")
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
test("call failure callbacks before close writer - partitioned") {
SimpleTextRelation.failCommitter = false
withTempPath { file =>
// fail the job in the middle of writing
val df = spark.range(0, 10).coalesce(1).select(col("id").mod(2).as("key"), col("id"))
SimpleTextRelation.callbackCalled = false
SimpleTextRelation.failWriter = true
intercept[SparkException] {
df.write.format(dataSourceName).partitionBy("key").save(file.getCanonicalPath)
}
assert(SimpleTextRelation.callbackCalled, "failure callback should be called")
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
}
Example 7
| Source File: OrcTest.scala From iolap with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive.orc
import java.io.File
import scala.reflect.ClassTag
import scala.reflect.runtime.universe.TypeTag
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.sql.hive.test.TestHive
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql._
private[sql] trait OrcTest extends SQLTestUtils {
protected def hiveContext = sqlContext.asInstanceOf[HiveContext]
import sqlContext.sparkContext
import sqlContext.implicits._
protected def withOrcTable[T <: Product: ClassTag: TypeTag]
(data: Seq[T], tableName: String)
(f: => Unit): Unit = {
withOrcDataFrame(data) { df =>
hiveContext.registerDataFrameAsTable(df, tableName)
withTempTable(tableName)(f)
}
}
protected def makeOrcFile[T <: Product: ClassTag: TypeTag](
data: Seq[T], path: File): Unit = {
data.toDF().write.format("orc").mode(SaveMode.Overwrite).save(path.getCanonicalPath)
}
protected def makeOrcFile[T <: Product: ClassTag: TypeTag](
df: DataFrame, path: File): Unit = {
df.write.format("orc").mode(SaveMode.Overwrite).save(path.getCanonicalPath)
}
}
Example 8
| Source File: ParquetTest.scala From iolap with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.parquet
import java.io.File
import scala.reflect.ClassTag
import scala.reflect.runtime.universe.TypeTag
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.{DataFrame, SaveMode}
protected def withParquetTable[T <: Product: ClassTag: TypeTag]
(data: Seq[T], tableName: String)
(f: => Unit): Unit = {
withParquetDataFrame(data) { df =>
sqlContext.registerDataFrameAsTable(df, tableName)
withTempTable(tableName)(f)
}
}
protected def makeParquetFile[T <: Product: ClassTag: TypeTag](
data: Seq[T], path: File): Unit = {
data.toDF().write.mode(SaveMode.Overwrite).parquet(path.getCanonicalPath)
}
protected def makeParquetFile[T <: Product: ClassTag: TypeTag](
df: DataFrame, path: File): Unit = {
df.write.mode(SaveMode.Overwrite).parquet(path.getCanonicalPath)
}
protected def makePartitionDir(
basePath: File,
defaultPartitionName: String,
partitionCols: (String, Any)*): File = {
val partNames = partitionCols.map { case (k, v) =>
val valueString = if (v == null || v == "") defaultPartitionName else v.toString
s"$k=$valueString"
}
val partDir = partNames.foldLeft(basePath) { (parent, child) =>
new File(parent, child)
}
assert(partDir.mkdirs(), s"Couldn't create directory $partDir")
partDir
}
}
Example 9
| Source File: QueryPartitionSuite.scala From spark1.52 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive
import com.google.common.io.Files
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.{QueryTest, _}
import org.apache.spark.util.Utils
//查询分区套件
class QueryPartitionSuite extends QueryTest with SQLTestUtils {
private lazy val ctx = org.apache.spark.sql.hive.test.TestHive
import ctx.implicits._
protected def _sqlContext = ctx
//查询数据当路径不存在时
test("SPARK-5068: query data when path doesn't exist"){
withSQLConf((SQLConf.HIVE_VERIFY_PARTITION_PATH.key, "true")) {
val testData = ctx.sparkContext.parallelize(
(1 to 10).map(i => TestData(i, i.toString))).toDF()
testData.registerTempTable("testData")
val tmpDir = Files.createTempDir()
// create the table for test 创建表进行测试
sql(s"CREATE TABLE table_with_partition(key int,value string) " +
s"PARTITIONED by (ds string) location '${tmpDir.toURI.toString}' ")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='1') " +
"SELECT key,value FROM testData")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='2') " +
"SELECT key,value FROM testData")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='3') " +
"SELECT key,value FROM testData")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='4') " +
"SELECT key,value FROM testData")
// test for the exist path 测试存在的路径
checkAnswer(sql("select key,value from table_with_partition"),
testData.toDF.collect ++ testData.toDF.collect
++ testData.toDF.collect ++ testData.toDF.collect)
// delete the path of one partition 删除一个分区的路径
tmpDir.listFiles
.find { f => f.isDirectory && f.getName().startsWith("ds=") }
.foreach { f => Utils.deleteRecursively(f) }
// test for after delete the path 测试后删除路径
checkAnswer(sql("select key,value from table_with_partition"),
testData.toDF.collect ++ testData.toDF.collect ++ testData.toDF.collect)
sql("DROP TABLE table_with_partition")
sql("DROP TABLE createAndInsertTest")
}
}
}
Example 10
| Source File: OrcTest.scala From spark1.52 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive.orc
import java.io.File
import scala.reflect.ClassTag
import scala.reflect.runtime.universe.TypeTag
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql._
import org.apache.spark.sql.test.SQLTestUtils
private[sql] trait OrcTest extends SQLTestUtils { this: SparkFunSuite =>
protected override def _sqlContext: SQLContext = org.apache.spark.sql.hive.test.TestHive
protected val sqlContext = _sqlContext
import sqlContext.implicits._
import sqlContext.sparkContext
protected def withOrcTable[T <: Product: ClassTag: TypeTag]
(data: Seq[T], tableName: String)
(f: => Unit): Unit = {
withOrcDataFrame(data) { df =>
sqlContext.registerDataFrameAsTable(df, tableName)
withTempTable(tableName)(f)
}
}
protected def makeOrcFile[T <: Product: ClassTag: TypeTag](
data: Seq[T], path: File): Unit = {
data.toDF().write.mode(SaveMode.Overwrite).orc(path.getCanonicalPath)
}
protected def makeOrcFile[T <: Product: ClassTag: TypeTag](
df: DataFrame, path: File): Unit = {
df.write.mode(SaveMode.Overwrite).orc(path.getCanonicalPath)
}
}
Example 11
| Source File: CommitFailureTestRelationSuite.scala From spark1.52 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.sources
import org.apache.hadoop.fs.Path
import org.apache.spark.{SparkException, SparkFunSuite}
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.hive.test.TestHive
import org.apache.spark.sql.test.SQLTestUtils
class CommitFailureTestRelationSuite extends SparkFunSuite with SQLTestUtils {
override def _sqlContext: SQLContext = TestHive
private val sqlContext = _sqlContext
// When committing a task, `CommitFailureTestSource` throws an exception for testing purpose.
//提交任务时，“CommitFailureTestSource”会为测试目的引发异常
val dataSourceName: String = classOf[CommitFailureTestSource].getCanonicalName
//commitTask（）失败应该回退到abortTask（）
test("SPARK-7684: commitTask() failure should fallback to abortTask()") {
withTempPath { file =>
// Here we coalesce partition number to 1 to ensure that only a single task is issued. This
// prevents race condition happened when FileOutputCommitter tries to remove the `_temporary`
// directory while committing/aborting the job. See SPARK-8513 for more details.
//这里我们将分区号合并为1，以确保只发出一个任务, 这个防止当FileOutputCommitter尝试删除`_temporary`时发生竞争条件
//目录提交/中止作业, 有关详细信息，请参阅SPARK-8513
val df = sqlContext.range(0, 10).coalesce(1)
intercept[SparkException] {
df.write.format(dataSourceName).save(file.getCanonicalPath)
}
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
}
Example 12
| Source File: DeltaHiveTest.scala From delta with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.delta.test
import org.apache.spark.sql.delta.catalog.DeltaCatalog
import io.delta.sql.DeltaSparkSessionExtension
import org.scalatest.BeforeAndAfterAll
import org.apache.spark.{SparkContext, SparkFunSuite}
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.hive.test.{TestHive, TestHiveContext}
import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
import org.apache.spark.sql.test.SQLTestUtils
trait DeltaHiveTest extends SparkFunSuite with BeforeAndAfterAll { self: SQLTestUtils =>
private var _session: SparkSession = _
private var _hiveContext: TestHiveContext = _
private var _sc: SparkContext = _
override def beforeAll(): Unit = {
val conf = TestHive.sparkSession.sparkContext.getConf.clone()
TestHive.sparkSession.stop()
conf.set(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[DeltaCatalog].getName)
conf.set(StaticSQLConf.SPARK_SESSION_EXTENSIONS.key,
classOf[DeltaSparkSessionExtension].getName)
_sc = new SparkContext("local", this.getClass.getName, conf)
_hiveContext = new TestHiveContext(_sc)
_session = _hiveContext.sparkSession
SparkSession.setActiveSession(_session)
super.beforeAll()
}
override protected def spark: SparkSession = _session
override def afterAll(): Unit = {
try {
_hiveContext.reset()
} finally {
_sc.stop()
}
}
}
Example 13
| Source File: QueryPartitionSuite.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive
import java.io.File
import java.sql.Timestamp
import com.google.common.io.Files
import org.apache.hadoop.fs.FileSystem
import org.apache.spark.sql._
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.util.Utils
class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
import spark.implicits._
test("SPARK-5068: query data when path doesn't exist") {
withSQLConf((SQLConf.HIVE_VERIFY_PARTITION_PATH.key, "true")) {
val testData = sparkContext.parallelize(
(1 to 10).map(i => TestData(i, i.toString))).toDF()
testData.createOrReplaceTempView("testData")
val tmpDir = Files.createTempDir()
// create the table for test
sql(s"CREATE TABLE table_with_partition(key int,value string) " +
s"PARTITIONED by (ds string) location '${tmpDir.toURI}' ")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='1') " +
"SELECT key,value FROM testData")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='2') " +
"SELECT key,value FROM testData")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='3') " +
"SELECT key,value FROM testData")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='4') " +
"SELECT key,value FROM testData")
// test for the exist path
checkAnswer(sql("select key,value from table_with_partition"),
testData.toDF.collect ++ testData.toDF.collect
++ testData.toDF.collect ++ testData.toDF.collect)
// delete the path of one partition
tmpDir.listFiles
.find { f => f.isDirectory && f.getName().startsWith("ds=") }
.foreach { f => Utils.deleteRecursively(f) }
// test for after delete the path
checkAnswer(sql("select key,value from table_with_partition"),
testData.toDF.collect ++ testData.toDF.collect ++ testData.toDF.collect)
sql("DROP TABLE IF EXISTS table_with_partition")
sql("DROP TABLE IF EXISTS createAndInsertTest")
}
}
test("SPARK-21739: Cast expression should initialize timezoneId") {
withTable("table_with_timestamp_partition") {
sql("CREATE TABLE table_with_timestamp_partition(value int) PARTITIONED BY (ts TIMESTAMP)")
sql("INSERT OVERWRITE TABLE table_with_timestamp_partition " +
"PARTITION (ts = '2010-01-01 00:00:00.000') VALUES (1)")
// test for Cast expression in TableReader
checkAnswer(sql("SELECT * FROM table_with_timestamp_partition"),
Seq(Row(1, Timestamp.valueOf("2010-01-01 00:00:00.000"))))
// test for Cast expression in HiveTableScanExec
checkAnswer(sql("SELECT value FROM table_with_timestamp_partition " +
"WHERE ts = '2010-01-01 00:00:00.000'"), Row(1))
}
}
}
Example 14
| Source File: TestHiveSuite.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.hive.test.{TestHiveSingleton, TestHiveSparkSession}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SQLTestUtils
class TestHiveSuite extends TestHiveSingleton with SQLTestUtils {
test("load test table based on case sensitivity") {
val testHiveSparkSession = spark.asInstanceOf[TestHiveSparkSession]
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
sql("SELECT * FROM SRC").queryExecution.analyzed
assert(testHiveSparkSession.getLoadedTables.contains("src"))
assert(testHiveSparkSession.getLoadedTables.size == 1)
}
testHiveSparkSession.reset()
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
val err = intercept[AnalysisException] {
sql("SELECT * FROM SRC").queryExecution.analyzed
}
assert(err.message.contains("Table or view not found"))
}
testHiveSparkSession.reset()
}
test("SPARK-15887: hive-site.xml should be loaded") {
assert(hiveClient.getConf("hive.in.test", "") == "true")
}
}
Example 15
| Source File: PruneFileSourcePartitionsSuite.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive.execution
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, LogicalRelation, PruneFileSourcePartitions}
import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.types.StructType
class PruneFileSourcePartitionsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
object Optimize extends RuleExecutor[LogicalPlan] {
val batches = Batch("PruneFileSourcePartitions", Once, PruneFileSourcePartitions) :: Nil
}
test("PruneFileSourcePartitions should not change the output of LogicalRelation") {
withTable("test") {
withTempDir { dir =>
sql(
s"""
|CREATE EXTERNAL TABLE test(i int)
|PARTITIONED BY (p int)
|STORED AS parquet
|LOCATION '${dir.toURI}'""".stripMargin)
val tableMeta = spark.sharedState.externalCatalog.getTable("default", "test")
val catalogFileIndex = new CatalogFileIndex(spark, tableMeta, 0)
val dataSchema = StructType(tableMeta.schema.filterNot { f =>
tableMeta.partitionColumnNames.contains(f.name)
})
val relation = HadoopFsRelation(
location = catalogFileIndex,
partitionSchema = tableMeta.partitionSchema,
dataSchema = dataSchema,
bucketSpec = None,
fileFormat = new ParquetFileFormat(),
options = Map.empty)(sparkSession = spark)
val logicalRelation = LogicalRelation(relation, tableMeta)
val query = Project(Seq('i, 'p), Filter('p === 1, logicalRelation)).analyze
val optimized = Optimize.execute(query)
assert(optimized.missingInput.isEmpty)
}
}
}
test("SPARK-20986 Reset table's statistics after PruneFileSourcePartitions rule") {
withTable("tbl") {
spark.range(10).selectExpr("id", "id % 3 as p").write.partitionBy("p").saveAsTable("tbl")
sql(s"ANALYZE TABLE tbl COMPUTE STATISTICS")
val tableStats = spark.sessionState.catalog.getTableMetadata(TableIdentifier("tbl")).stats
assert(tableStats.isDefined && tableStats.get.sizeInBytes > 0, "tableStats is lost")
val df = sql("SELECT * FROM tbl WHERE p = 1")
val sizes1 = df.queryExecution.analyzed.collect {
case relation: LogicalRelation => relation.catalogTable.get.stats.get.sizeInBytes
}
assert(sizes1.size === 1, s"Size wrong for:\n ${df.queryExecution}")
assert(sizes1(0) == tableStats.get.sizeInBytes)
val relations = df.queryExecution.optimizedPlan.collect {
case relation: LogicalRelation => relation
}
assert(relations.size === 1, s"Size wrong for:\n ${df.queryExecution}")
val size2 = relations(0).stats.sizeInBytes
assert(size2 == relations(0).catalogTable.get.stats.get.sizeInBytes)
assert(size2 < tableStats.get.sizeInBytes)
}
}
}
Example 16
| Source File: HiveUtilsSuite.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive
import java.net.URL
import org.apache.hadoop.hive.conf.HiveConf.ConfVars
import org.apache.spark.SparkConf
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SQLTestUtils}
import org.apache.spark.util.{ChildFirstURLClassLoader, MutableURLClassLoader}
class HiveUtilsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
test("newTemporaryConfiguration overwrites listener configurations") {
Seq(true, false).foreach { useInMemoryDerby =>
val conf = HiveUtils.newTemporaryConfiguration(useInMemoryDerby)
assert(conf(ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname) === "")
assert(conf(ConfVars.METASTORE_EVENT_LISTENERS.varname) === "")
assert(conf(ConfVars.METASTORE_END_FUNCTION_LISTENERS.varname) === "")
}
}
test("newTemporaryConfiguration respect spark.hadoop.foo=bar in SparkConf") {
sys.props.put("spark.hadoop.foo", "bar")
Seq(true, false) foreach { useInMemoryDerby =>
val hiveConf = HiveUtils.newTemporaryConfiguration(useInMemoryDerby)
assert(!hiveConf.contains("spark.hadoop.foo"))
assert(hiveConf("foo") === "bar")
}
}
test("ChildFirstURLClassLoader's parent is null, get spark classloader instead") {
val conf = new SparkConf
val contextClassLoader = Thread.currentThread().getContextClassLoader
val loader = new ChildFirstURLClassLoader(Array(), contextClassLoader)
try {
Thread.currentThread().setContextClassLoader(loader)
HiveUtils.newClientForMetadata(
conf,
SparkHadoopUtil.newConfiguration(conf),
HiveUtils.newTemporaryConfiguration(useInMemoryDerby = true))
} finally {
Thread.currentThread().setContextClassLoader(contextClassLoader)
}
}
test("toHiveString correctly handles UDTs") {
val point = new ExamplePoint(50.0, 50.0)
val tpe = new ExamplePointUDT()
assert(HiveUtils.toHiveString((point, tpe)) === "(50.0, 50.0)")
}
}
Example 17
| Source File: CommitFailureTestRelationSuite.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.sources
import org.apache.hadoop.fs.Path
import org.apache.spark.SparkException
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.functions._
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
class CommitFailureTestRelationSuite extends SQLTestUtils with TestHiveSingleton {
// When committing a task, `CommitFailureTestSource` throws an exception for testing purpose.
val dataSourceName: String = classOf[CommitFailureTestSource].getCanonicalName
test("SPARK-7684: commitTask() failure should fallback to abortTask()") {
withTempPath { file =>
// Here we coalesce partition number to 1 to ensure that only a single task is issued. This
// prevents race condition happened when FileOutputCommitter tries to remove the `_temporary`
// directory while committing/aborting the job. See SPARK-8513 for more details.
val df = spark.range(0, 10).coalesce(1)
intercept[SparkException] {
df.write.format(dataSourceName).save(file.getCanonicalPath)
}
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
test("call failure callbacks before close writer - default") {
SimpleTextRelation.failCommitter = false
withTempPath { file =>
// fail the job in the middle of writing
val divideByZero = udf((x: Int) => { x / (x - 1)})
val df = spark.range(0, 10).coalesce(1).select(divideByZero(col("id")))
SimpleTextRelation.callbackCalled = false
intercept[SparkException] {
df.write.format(dataSourceName).save(file.getCanonicalPath)
}
assert(SimpleTextRelation.callbackCalled, "failure callback should be called")
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
test("call failure callbacks before close writer - partitioned") {
SimpleTextRelation.failCommitter = false
withTempPath { file =>
// fail the job in the middle of writing
val df = spark.range(0, 10).coalesce(1).select(col("id").mod(2).as("key"), col("id"))
SimpleTextRelation.callbackCalled = false
SimpleTextRelation.failWriter = true
intercept[SparkException] {
df.write.format(dataSourceName).partitionBy("key").save(file.getCanonicalPath)
}
assert(SimpleTextRelation.callbackCalled, "failure callback should be called")
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
}
Example 18
| Source File: OrcTest.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.execution.datasources.orc
import java.io.File
import scala.reflect.ClassTag
import scala.reflect.runtime.universe.TypeTag
import org.scalatest.BeforeAndAfterAll
import org.apache.spark.sql._
import org.apache.spark.sql.internal.SQLConf.ORC_IMPLEMENTATION
import org.apache.spark.sql.test.SQLTestUtils
protected def withOrcTable[T <: Product: ClassTag: TypeTag]
(data: Seq[T], tableName: String)
(f: => Unit): Unit = {
withOrcDataFrame(data) { df =>
df.createOrReplaceTempView(tableName)
withTempView(tableName)(f)
}
}
protected def makeOrcFile[T <: Product: ClassTag: TypeTag](
data: Seq[T], path: File): Unit = {
data.toDF().write.mode(SaveMode.Overwrite).orc(path.getCanonicalPath)
}
protected def makeOrcFile[T <: Product: ClassTag: TypeTag](
df: DataFrame, path: File): Unit = {
df.write.mode(SaveMode.Overwrite).orc(path.getCanonicalPath)
}
}
Example 19
| Source File: QueryPartitionSuite.scala From BigDatalog with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive
import com.google.common.io.Files
import org.apache.spark.util.Utils
import org.apache.spark.sql.{QueryTest, _}
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
import hiveContext.implicits._
test("SPARK-5068: query data when path doesn't exist") {
withSQLConf((SQLConf.HIVE_VERIFY_PARTITION_PATH.key, "true")) {
val testData = sparkContext.parallelize(
(1 to 10).map(i => TestData(i, i.toString))).toDF()
testData.registerTempTable("testData")
val tmpDir = Files.createTempDir()
// create the table for test
sql(s"CREATE TABLE table_with_partition(key int,value string) " +
s"PARTITIONED by (ds string) location '${tmpDir.toURI.toString}' ")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='1') " +
"SELECT key,value FROM testData")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='2') " +
"SELECT key,value FROM testData")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='3') " +
"SELECT key,value FROM testData")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='4') " +
"SELECT key,value FROM testData")
// test for the exist path
checkAnswer(sql("select key,value from table_with_partition"),
testData.toDF.collect ++ testData.toDF.collect
++ testData.toDF.collect ++ testData.toDF.collect)
// delete the path of one partition
tmpDir.listFiles
.find { f => f.isDirectory && f.getName().startsWith("ds=") }
.foreach { f => Utils.deleteRecursively(f) }
// test for after delete the path
checkAnswer(sql("select key,value from table_with_partition"),
testData.toDF.collect ++ testData.toDF.collect ++ testData.toDF.collect)
sql("DROP TABLE table_with_partition")
sql("DROP TABLE createAndInsertTest")
}
}
}
Example 20
| Source File: OrcTest.scala From BigDatalog with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive.orc
import java.io.File
import scala.reflect.ClassTag
import scala.reflect.runtime.universe.TypeTag
import org.apache.spark.sql._
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.hive.test.TestHiveSingleton
private[sql] trait OrcTest extends SQLTestUtils with TestHiveSingleton {
import testImplicits._
protected def withOrcTable[T <: Product: ClassTag: TypeTag]
(data: Seq[T], tableName: String)
(f: => Unit): Unit = {
withOrcDataFrame(data) { df =>
sqlContext.registerDataFrameAsTable(df, tableName)
withTempTable(tableName)(f)
}
}
protected def makeOrcFile[T <: Product: ClassTag: TypeTag](
data: Seq[T], path: File): Unit = {
data.toDF().write.mode(SaveMode.Overwrite).orc(path.getCanonicalPath)
}
protected def makeOrcFile[T <: Product: ClassTag: TypeTag](
df: DataFrame, path: File): Unit = {
df.write.mode(SaveMode.Overwrite).orc(path.getCanonicalPath)
}
}
Example 21
| Source File: HiveExplainSuite.scala From BigDatalog with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive.execution
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.hive.test.TestHiveSingleton
class HiveExplainSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
test("explain extended command") {
checkExistence(sql(" explain select * from src where key=123 "), true,
"== Physical Plan ==")
checkExistence(sql(" explain select * from src where key=123 "), false,
"== Parsed Logical Plan ==",
"== Analyzed Logical Plan ==",
"== Optimized Logical Plan ==")
checkExistence(sql(" explain extended select * from src where key=123 "), true,
"== Parsed Logical Plan ==",
"== Analyzed Logical Plan ==",
"== Optimized Logical Plan ==",
"== Physical Plan ==")
}
test("explain create table command") {
checkExistence(sql("explain create table temp__b as select * from src limit 2"), true,
"== Physical Plan ==",
"InsertIntoHiveTable",
"Limit",
"src")
checkExistence(sql("explain extended create table temp__b as select * from src limit 2"), true,
"== Parsed Logical Plan ==",
"== Analyzed Logical Plan ==",
"== Optimized Logical Plan ==",
"== Physical Plan ==",
"CreateTableAsSelect",
"InsertIntoHiveTable",
"Limit",
"src")
checkExistence(sql(
"""
| EXPLAIN EXTENDED CREATE TABLE temp__b
| ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"
| WITH SERDEPROPERTIES("serde_p1"="p1","serde_p2"="p2")
| STORED AS RCFile
| TBLPROPERTIES("tbl_p1"="p11", "tbl_p2"="p22")
| AS SELECT * FROM src LIMIT 2
""".stripMargin), true,
"== Parsed Logical Plan ==",
"== Analyzed Logical Plan ==",
"== Optimized Logical Plan ==",
"== Physical Plan ==",
"CreateTableAsSelect",
"InsertIntoHiveTable",
"Limit",
"src")
}
test("SPARK-6212: The EXPLAIN output of CTAS only shows the analyzed plan") {
withTempTable("jt") {
val rdd = sparkContext.parallelize((1 to 10).map(i => s"""{"a":$i, "b":"str$i"}"""))
hiveContext.read.json(rdd).registerTempTable("jt")
val outputs = sql(
s"""
|EXPLAIN EXTENDED
|CREATE TABLE t1
|AS
|SELECT * FROM jt
""".stripMargin).collect().map(_.mkString).mkString
val shouldContain =
"== Parsed Logical Plan ==" :: "== Analyzed Logical Plan ==" :: "Subquery" ::
"== Optimized Logical Plan ==" :: "== Physical Plan ==" ::
"CreateTableAsSelect" :: "InsertIntoHiveTable" :: "jt" :: Nil
for (key <- shouldContain) {
assert(outputs.contains(key), s"$key doesn't exist in result")
}
val physicalIndex = outputs.indexOf("== Physical Plan ==")
assert(!outputs.substring(physicalIndex).contains("Subquery"),
"Physical Plan should not contain Subquery since it's eliminated by optimizer")
}
}
}
Example 22
| Source File: CommitFailureTestRelationSuite.scala From BigDatalog with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.sources
import org.apache.hadoop.fs.Path
import org.apache.spark.SparkException
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
class CommitFailureTestRelationSuite extends SQLTestUtils with TestHiveSingleton {
// When committing a task, `CommitFailureTestSource` throws an exception for testing purpose.
val dataSourceName: String = classOf[CommitFailureTestSource].getCanonicalName
test("SPARK-7684: commitTask() failure should fallback to abortTask()") {
withTempPath { file =>
// Here we coalesce partition number to 1 to ensure that only a single task is issued. This
// prevents race condition happened when FileOutputCommitter tries to remove the `_temporary`
// directory while committing/aborting the job. See SPARK-8513 for more details.
val df = sqlContext.range(0, 10).coalesce(1)
intercept[SparkException] {
df.write.format(dataSourceName).save(file.getCanonicalPath)
}
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
}
Example 23
| Source File: CommitFailureTestRelationSuite.scala From XSQL with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.sources
import org.apache.hadoop.fs.Path
import org.apache.spark.SparkException
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.functions._
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
class CommitFailureTestRelationSuite extends SQLTestUtils with TestHiveSingleton {
// When committing a task, `CommitFailureTestSource` throws an exception for testing purpose.
val dataSourceName: String = classOf[CommitFailureTestSource].getCanonicalName
test("SPARK-7684: commitTask() failure should fallback to abortTask()") {
withTempPath { file =>
// Here we coalesce partition number to 1 to ensure that only a single task is issued. This
// prevents race condition happened when FileOutputCommitter tries to remove the `_temporary`
// directory while committing/aborting the job. See SPARK-8513 for more details.
val df = spark.range(0, 10).coalesce(1)
intercept[SparkException] {
df.write.format(dataSourceName).save(file.getCanonicalPath)
}
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
test("call failure callbacks before close writer - default") {
SimpleTextRelation.failCommitter = false
withTempPath { file =>
// fail the job in the middle of writing
val divideByZero = udf((x: Int) => { x / (x - 1)})
val df = spark.range(0, 10).coalesce(1).select(divideByZero(col("id")))
SimpleTextRelation.callbackCalled = false
intercept[SparkException] {
df.write.format(dataSourceName).save(file.getCanonicalPath)
}
assert(SimpleTextRelation.callbackCalled, "failure callback should be called")
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
test("call failure callbacks before close writer - partitioned") {
SimpleTextRelation.failCommitter = false
withTempPath { file =>
// fail the job in the middle of writing
val df = spark.range(0, 10).coalesce(1).select(col("id").mod(2).as("key"), col("id"))
SimpleTextRelation.callbackCalled = false
SimpleTextRelation.failWriter = true
intercept[SparkException] {
df.write.format(dataSourceName).partitionBy("key").save(file.getCanonicalPath)
}
assert(SimpleTextRelation.callbackCalled, "failure callback should be called")
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
}
Example 24
| Source File: MetastoreRelationSuite.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive
import org.apache.spark.sql.{QueryTest, Row}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType}
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
class MetastoreRelationSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
test("makeCopy and toJSON should work") {
val table = CatalogTable(
identifier = TableIdentifier("test", Some("db")),
tableType = CatalogTableType.VIEW,
storage = CatalogStorageFormat.empty,
schema = StructType(StructField("a", IntegerType, true) :: Nil))
val relation = MetastoreRelation("db", "test")(table, null)
// No exception should be thrown
relation.makeCopy(Array("db", "test"))
// No exception should be thrown
relation.toJSON
}
test("SPARK-17409: Do Not Optimize Query in CTAS (Hive Serde Table) More Than Once") {
withTable("bar") {
withTempView("foo") {
sql("select 0 as id").createOrReplaceTempView("foo")
// If we optimize the query in CTAS more than once, the following saveAsTable will fail
// with the error: `GROUP BY position 0 is not in select list (valid range is [1, 1])`
sql("CREATE TABLE bar AS SELECT * FROM foo group by id")
checkAnswer(spark.table("bar"), Row(0) :: Nil)
val tableMetadata = spark.sessionState.catalog.getTableMetadata(TableIdentifier("bar"))
assert(tableMetadata.provider == Some("hive"), "the expected table is a Hive serde table")
}
}
}
}
Example 25
| Source File: HiveExplainSuite.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive.execution
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.catalyst.parser.ParseException
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
}"
)
checkKeywordsNotExist(sql("EXPLAIN CODEGEN SELECT 1"),
"== Physical Plan =="
)
intercept[ParseException] {
sql("EXPLAIN EXTENDED CODEGEN SELECT 1")
}
}
}
Example 26
| Source File: PruneFileSourcePartitionsSuite.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive.execution
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation, PruneFileSourcePartitions, TableFileCatalog}
import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.types.StructType
class PruneFileSourcePartitionsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
object Optimize extends RuleExecutor[LogicalPlan] {
val batches = Batch("PruneFileSourcePartitions", Once, PruneFileSourcePartitions) :: Nil
}
test("PruneFileSourcePartitions should not change the output of LogicalRelation") {
withTable("test") {
withTempDir { dir =>
sql(
s"""
|CREATE EXTERNAL TABLE test(i int)
|PARTITIONED BY (p int)
|STORED AS parquet
|LOCATION '${dir.getAbsolutePath}'""".stripMargin)
val tableMeta = spark.sharedState.externalCatalog.getTable("default", "test")
val tableFileCatalog = new TableFileCatalog(spark, tableMeta, 0)
val dataSchema = StructType(tableMeta.schema.filterNot { f =>
tableMeta.partitionColumnNames.contains(f.name)
})
val relation = HadoopFsRelation(
location = tableFileCatalog,
partitionSchema = tableMeta.partitionSchema,
dataSchema = dataSchema,
bucketSpec = None,
fileFormat = new ParquetFileFormat(),
options = Map.empty)(sparkSession = spark)
val logicalRelation = LogicalRelation(relation, catalogTable = Some(tableMeta))
val query = Project(Seq('i, 'p), Filter('p === 1, logicalRelation)).analyze
val optimized = Optimize.execute(query)
assert(optimized.missingInput.isEmpty)
}
}
}
}
Example 27
| Source File: HiveUtilsSuite.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive
import org.apache.hadoop.hive.conf.HiveConf.ConfVars
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.QueryTest
class HiveUtilsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
test("newTemporaryConfiguration overwrites listener configurations") {
Seq(true, false).foreach { useInMemoryDerby =>
val conf = HiveUtils.newTemporaryConfiguration(useInMemoryDerby)
assert(conf(ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname) === "")
assert(conf(ConfVars.METASTORE_EVENT_LISTENERS.varname) === "")
assert(conf(ConfVars.METASTORE_END_FUNCTION_LISTENERS.varname) === "")
}
}
}
Example 28
| Source File: CommitFailureTestRelationSuite.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.sources
import org.apache.hadoop.fs.Path
import org.apache.spark.SparkException
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.functions._
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
class CommitFailureTestRelationSuite extends SQLTestUtils with TestHiveSingleton {
// When committing a task, `CommitFailureTestSource` throws an exception for testing purpose.
val dataSourceName: String = classOf[CommitFailureTestSource].getCanonicalName
test("SPARK-7684: commitTask() failure should fallback to abortTask()") {
withTempPath { file =>
// Here we coalesce partition number to 1 to ensure that only a single task is issued. This
// prevents race condition happened when FileOutputCommitter tries to remove the `_temporary`
// directory while committing/aborting the job. See SPARK-8513 for more details.
val df = spark.range(0, 10).coalesce(1)
intercept[SparkException] {
df.write.format(dataSourceName).save(file.getCanonicalPath)
}
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
test("call failure callbacks before close writer - default") {
SimpleTextRelation.failCommitter = false
withTempPath { file =>
// fail the job in the middle of writing
val divideByZero = udf((x: Int) => { x / (x - 1)})
val df = spark.range(0, 10).coalesce(1).select(divideByZero(col("id")))
SimpleTextRelation.callbackCalled = false
intercept[SparkException] {
df.write.format(dataSourceName).save(file.getCanonicalPath)
}
assert(SimpleTextRelation.callbackCalled, "failure callback should be called")
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
test("call failure callbacks before close writer - partitioned") {
SimpleTextRelation.failCommitter = false
withTempPath { file =>
// fail the job in the middle of writing
val df = spark.range(0, 10).coalesce(1).select(col("id").mod(2).as("key"), col("id"))
SimpleTextRelation.callbackCalled = false
SimpleTextRelation.failWriter = true
intercept[SparkException] {
df.write.format(dataSourceName).partitionBy("key").save(file.getCanonicalPath)
}
assert(SimpleTextRelation.callbackCalled, "failure callback should be called")
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
}
Example 29
| Source File: QueryPartitionSuite.scala From XSQL with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive
import java.io.File
import java.sql.Timestamp
import com.google.common.io.Files
import org.apache.hadoop.fs.FileSystem
import org.apache.spark.internal.config._
import org.apache.spark.sql._
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.util.Utils
class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
import spark.implicits._
private def queryWhenPathNotExist(): Unit = {
withTempView("testData") {
withTable("table_with_partition", "createAndInsertTest") {
withTempDir { tmpDir =>
val testData = sparkContext.parallelize(
(1 to 10).map(i => TestData(i, i.toString))).toDF()
testData.createOrReplaceTempView("testData")
// create the table for test
sql(s"CREATE TABLE table_with_partition(key int,value string) " +
s"PARTITIONED by (ds string) location '${tmpDir.toURI}' ")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='1') " +
"SELECT key,value FROM testData")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='2') " +
"SELECT key,value FROM testData")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='3') " +
"SELECT key,value FROM testData")
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='4') " +
"SELECT key,value FROM testData")
// test for the exist path
checkAnswer(sql("select key,value from table_with_partition"),
testData.union(testData).union(testData).union(testData))
// delete the path of one partition
tmpDir.listFiles
.find { f => f.isDirectory && f.getName().startsWith("ds=") }
.foreach { f => Utils.deleteRecursively(f) }
// test for after delete the path
checkAnswer(sql("select key,value from table_with_partition"),
testData.union(testData).union(testData))
}
}
}
}
test("SPARK-5068: query data when path doesn't exist") {
withSQLConf(SQLConf.HIVE_VERIFY_PARTITION_PATH.key -> "true") {
queryWhenPathNotExist()
}
}
test("Replace spark.sql.hive.verifyPartitionPath by spark.files.ignoreMissingFiles") {
withSQLConf(SQLConf.HIVE_VERIFY_PARTITION_PATH.key -> "false") {
sparkContext.conf.set(IGNORE_MISSING_FILES.key, "true")
queryWhenPathNotExist()
}
}
test("SPARK-21739: Cast expression should initialize timezoneId") {
withTable("table_with_timestamp_partition") {
sql("CREATE TABLE table_with_timestamp_partition(value int) PARTITIONED BY (ts TIMESTAMP)")
sql("INSERT OVERWRITE TABLE table_with_timestamp_partition " +
"PARTITION (ts = '2010-01-01 00:00:00.000') VALUES (1)")
// test for Cast expression in TableReader
checkAnswer(sql("SELECT * FROM table_with_timestamp_partition"),
Seq(Row(1, Timestamp.valueOf("2010-01-01 00:00:00.000"))))
// test for Cast expression in HiveTableScanExec
checkAnswer(sql("SELECT value FROM table_with_timestamp_partition " +
"WHERE ts = '2010-01-01 00:00:00.000'"), Row(1))
}
}
}
Example 30
| Source File: TestHiveSuite.scala From XSQL with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.hive.test.{TestHiveSingleton, TestHiveSparkSession}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SQLTestUtils
class TestHiveSuite extends TestHiveSingleton with SQLTestUtils {
test("load test table based on case sensitivity") {
val testHiveSparkSession = spark.asInstanceOf[TestHiveSparkSession]
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
sql("SELECT * FROM SRC").queryExecution.analyzed
assert(testHiveSparkSession.getLoadedTables.contains("src"))
assert(testHiveSparkSession.getLoadedTables.size == 1)
}
testHiveSparkSession.reset()
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
val err = intercept[AnalysisException] {
sql("SELECT * FROM SRC").queryExecution.analyzed
}
assert(err.message.contains("Table or view not found"))
}
testHiveSparkSession.reset()
}
test("SPARK-15887: hive-site.xml should be loaded") {
assert(hiveClient.getConf("hive.in.test", "") == "true")
}
}
Example 31
| Source File: OptimizeHiveMetadataOnlyQuerySuite.scala From XSQL with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive
import org.scalatest.BeforeAndAfter
import org.apache.spark.metrics.source.HiveCatalogMetrics
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.catalyst.expressions.NamedExpression
import org.apache.spark.sql.catalyst.plans.logical.{Distinct, Filter, Project, SubqueryAlias}
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.internal.SQLConf.OPTIMIZER_METADATA_ONLY
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
class OptimizeHiveMetadataOnlyQuerySuite extends QueryTest with TestHiveSingleton
with BeforeAndAfter with SQLTestUtils {
import spark.implicits._
override def beforeAll(): Unit = {
super.beforeAll()
sql("CREATE TABLE metadata_only (id bigint, data string) PARTITIONED BY (part int)")
(0 to 10).foreach(p => sql(s"ALTER TABLE metadata_only ADD PARTITION (part=$p)"))
}
override protected def afterAll(): Unit = {
try {
sql("DROP TABLE IF EXISTS metadata_only")
} finally {
super.afterAll()
}
}
test("SPARK-23877: validate metadata-only query pushes filters to metastore") {
withSQLConf(OPTIMIZER_METADATA_ONLY.key -> "true") {
val startCount = HiveCatalogMetrics.METRIC_PARTITIONS_FETCHED.getCount
// verify the number of matching partitions
assert(sql("SELECT DISTINCT part FROM metadata_only WHERE part < 5").collect().length === 5)
// verify that the partition predicate was pushed down to the metastore
assert(HiveCatalogMetrics.METRIC_PARTITIONS_FETCHED.getCount - startCount === 5)
}
}
test("SPARK-23877: filter on projected expression") {
withSQLConf(OPTIMIZER_METADATA_ONLY.key -> "true") {
val startCount = HiveCatalogMetrics.METRIC_PARTITIONS_FETCHED.getCount
// verify the matching partitions
val partitions = spark.internalCreateDataFrame(Distinct(Filter(($"x" < 5).expr,
Project(Seq(($"part" + 1).as("x").expr.asInstanceOf[NamedExpression]),
spark.table("metadata_only").logicalPlan.asInstanceOf[SubqueryAlias].child)))
.queryExecution.toRdd, StructType(Seq(StructField("x", IntegerType))))
checkAnswer(partitions, Seq(1, 2, 3, 4).toDF("x"))
// verify that the partition predicate was not pushed down to the metastore
assert(HiveCatalogMetrics.METRIC_PARTITIONS_FETCHED.getCount - startCount == 11)
}
}
}
Example 32
| Source File: PruneFileSourcePartitionsSuite.scala From XSQL with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive.execution
import org.scalatest.Matchers._
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project, ResolvedHint}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, LogicalRelation, PruneFileSourcePartitions}
import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
import org.apache.spark.sql.functions.broadcast
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.types.StructType
class PruneFileSourcePartitionsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
object Optimize extends RuleExecutor[LogicalPlan] {
val batches = Batch("PruneFileSourcePartitions", Once, PruneFileSourcePartitions) :: Nil
}
test("PruneFileSourcePartitions should not change the output of LogicalRelation") {
withTable("test") {
withTempDir { dir =>
sql(
s"""
|CREATE EXTERNAL TABLE test(i int)
|PARTITIONED BY (p int)
|STORED AS parquet
|LOCATION '${dir.toURI}'""".stripMargin)
val tableMeta = spark.sharedState.externalCatalog.getTable("default", "test")
val catalogFileIndex = new CatalogFileIndex(spark, tableMeta, 0)
val dataSchema = StructType(tableMeta.schema.filterNot { f =>
tableMeta.partitionColumnNames.contains(f.name)
})
val relation = HadoopFsRelation(
location = catalogFileIndex,
partitionSchema = tableMeta.partitionSchema,
dataSchema = dataSchema,
bucketSpec = None,
fileFormat = new ParquetFileFormat(),
options = Map.empty)(sparkSession = spark)
val logicalRelation = LogicalRelation(relation, tableMeta)
val query = Project(Seq('i, 'p), Filter('p === 1, logicalRelation)).analyze
val optimized = Optimize.execute(query)
assert(optimized.missingInput.isEmpty)
}
}
}
test("SPARK-20986 Reset table's statistics after PruneFileSourcePartitions rule") {
withTable("tbl") {
spark.range(10).selectExpr("id", "id % 3 as p").write.partitionBy("p").saveAsTable("tbl")
sql(s"ANALYZE TABLE tbl COMPUTE STATISTICS")
val tableStats = spark.sessionState.catalog.getTableMetadata(TableIdentifier("tbl")).stats
assert(tableStats.isDefined && tableStats.get.sizeInBytes > 0, "tableStats is lost")
val df = sql("SELECT * FROM tbl WHERE p = 1")
val sizes1 = df.queryExecution.analyzed.collect {
case relation: LogicalRelation => relation.catalogTable.get.stats.get.sizeInBytes
}
assert(sizes1.size === 1, s"Size wrong for:\n ${df.queryExecution}")
assert(sizes1(0) == tableStats.get.sizeInBytes)
val relations = df.queryExecution.optimizedPlan.collect {
case relation: LogicalRelation => relation
}
assert(relations.size === 1, s"Size wrong for:\n ${df.queryExecution}")
val size2 = relations(0).stats.sizeInBytes
assert(size2 == relations(0).catalogTable.get.stats.get.sizeInBytes)
assert(size2 < tableStats.get.sizeInBytes)
}
}
test("SPARK-26576 Broadcast hint not applied to partitioned table") {
withTable("tbl") {
withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
spark.range(10).selectExpr("id", "id % 3 as p").write.partitionBy("p").saveAsTable("tbl")
val df = spark.table("tbl")
val qe = df.join(broadcast(df), "p").queryExecution
qe.optimizedPlan.collect { case _: ResolvedHint => } should have size 1
qe.sparkPlan.collect { case j: BroadcastHashJoinExec => j } should have size 1
}
}
}
}
Example 33
| Source File: HiveUtilsSuite.scala From XSQL with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive
import java.net.URL
import org.apache.hadoop.hive.conf.HiveConf.ConfVars
import org.apache.spark.SparkConf
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SQLTestUtils}
import org.apache.spark.util.{ChildFirstURLClassLoader, MutableURLClassLoader}
class HiveUtilsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
test("newTemporaryConfiguration overwrites listener configurations") {
Seq(true, false).foreach { useInMemoryDerby =>
val conf = HiveUtils.newTemporaryConfiguration(useInMemoryDerby)
assert(conf(ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname) === "")
assert(conf(ConfVars.METASTORE_EVENT_LISTENERS.varname) === "")
assert(conf(ConfVars.METASTORE_END_FUNCTION_LISTENERS.varname) === "")
}
}
test("newTemporaryConfiguration respect spark.hadoop.foo=bar in SparkConf") {
sys.props.put("spark.hadoop.foo", "bar")
Seq(true, false) foreach { useInMemoryDerby =>
val hiveConf = HiveUtils.newTemporaryConfiguration(useInMemoryDerby)
assert(!hiveConf.contains("spark.hadoop.foo"))
assert(hiveConf("foo") === "bar")
}
}
test("ChildFirstURLClassLoader's parent is null, get spark classloader instead") {
val conf = new SparkConf
val contextClassLoader = Thread.currentThread().getContextClassLoader
val loader = new ChildFirstURLClassLoader(Array(), contextClassLoader)
try {
Thread.currentThread().setContextClassLoader(loader)
HiveUtils.newClientForMetadata(
conf,
SparkHadoopUtil.newConfiguration(conf),
HiveUtils.newTemporaryConfiguration(useInMemoryDerby = true))
} finally {
Thread.currentThread().setContextClassLoader(contextClassLoader)
}
}
test("toHiveString correctly handles UDTs") {
val point = new ExamplePoint(50.0, 50.0)
val tpe = new ExamplePointUDT()
assert(HiveUtils.toHiveString((point, tpe)) === "(50.0, 50.0)")
}
}
Example 34
| Source File: OrcTest.scala From drizzle-spark with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive.orc
import java.io.File
import scala.reflect.ClassTag
import scala.reflect.runtime.universe.TypeTag
import org.apache.spark.sql._
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
private[sql] trait OrcTest extends SQLTestUtils with TestHiveSingleton {
import testImplicits._
protected def withOrcTable[T <: Product: ClassTag: TypeTag]
(data: Seq[T], tableName: String)
(f: => Unit): Unit = {
withOrcDataFrame(data) { df =>
df.createOrReplaceTempView(tableName)
withTempView(tableName)(f)
}
}
protected def makeOrcFile[T <: Product: ClassTag: TypeTag](
data: Seq[T], path: File): Unit = {
data.toDF().write.mode(SaveMode.Overwrite).orc(path.getCanonicalPath)
}
protected def makeOrcFile[T <: Product: ClassTag: TypeTag](
df: DataFrame, path: File): Unit = {
df.write.mode(SaveMode.Overwrite).orc(path.getCanonicalPath)
}
}
Example 35
| Source File: OrcTest.scala From XSQL with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.execution.datasources.orc
import java.io.File
import scala.reflect.ClassTag
import scala.reflect.runtime.universe.TypeTag
import org.scalatest.BeforeAndAfterAll
import org.apache.spark.sql._
import org.apache.spark.sql.internal.SQLConf.ORC_IMPLEMENTATION
import org.apache.spark.sql.test.SQLTestUtils
protected def withOrcTable[T <: Product: ClassTag: TypeTag]
(data: Seq[T], tableName: String)
(f: => Unit): Unit = {
withOrcDataFrame(data) { df =>
df.createOrReplaceTempView(tableName)
withTempView(tableName)(f)
}
}
protected def makeOrcFile[T <: Product: ClassTag: TypeTag](
data: Seq[T], path: File): Unit = {
data.toDF().write.mode(SaveMode.Overwrite).orc(path.getCanonicalPath)
}
protected def makeOrcFile[T <: Product: ClassTag: TypeTag](
df: DataFrame, path: File): Unit = {
df.write.mode(SaveMode.Overwrite).orc(path.getCanonicalPath)
}
protected def checkPredicatePushDown(df: DataFrame, numRows: Int, predicate: String): Unit = {
withTempPath { file =>
// It needs to repartition data so that we can have several ORC files
// in order to skip stripes in ORC.
df.repartition(numRows).write.orc(file.getCanonicalPath)
val actual = stripSparkFilter(spark.read.orc(file.getCanonicalPath).where(predicate)).count()
assert(actual < numRows)
}
}
}
Example 36
| Source File: ExecutorSideSQLConfSuite.scala From XSQL with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.internal
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.{AnalysisException, SparkSession}
import org.apache.spark.sql.execution.debug.codegenStringSeq
import org.apache.spark.sql.functions.col
import org.apache.spark.sql.test.SQLTestUtils
class ExecutorSideSQLConfSuite extends SparkFunSuite with SQLTestUtils {
import testImplicits._
protected var spark: SparkSession = null
// Create a new [[SparkSession]] running in local-cluster mode.
override def beforeAll(): Unit = {
super.beforeAll()
spark = SparkSession.builder()
.master("local-cluster[2,1,1024]")
.appName("testing")
.getOrCreate()
}
override def afterAll(): Unit = {
spark.stop()
spark = null
}
override def withSQLConf(pairs: (String, String)*)(f: => Unit): Unit = {
pairs.foreach { case (k, v) =>
SQLConf.get.setConfString(k, v)
}
try f finally {
pairs.foreach { case (k, _) =>
SQLConf.get.unsetConf(k)
}
}
}
test("ReadOnlySQLConf is correctly created at the executor side") {
withSQLConf("spark.sql.x" -> "a") {
val checks = spark.range(10).mapPartitions { _ =>
val conf = SQLConf.get
Iterator(conf.isInstanceOf[ReadOnlySQLConf] && conf.getConfString("spark.sql.x") == "a")
}.collect()
assert(checks.forall(_ == true))
}
}
test("case-sensitive config should work for json schema inference") {
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
withTempPath { path =>
val pathString = path.getCanonicalPath
spark.range(10).select('id.as("ID")).write.json(pathString)
spark.range(10).write.mode("append").json(pathString)
assert(spark.read.json(pathString).columns.toSet == Set("id", "ID"))
}
}
}
test("SPARK-24727 CODEGEN_CACHE_MAX_ENTRIES is correctly referenced at the executor side") {
withSQLConf(StaticSQLConf.CODEGEN_CACHE_MAX_ENTRIES.key -> "300") {
val checks = spark.range(10).mapPartitions { _ =>
val conf = SQLConf.get
Iterator(conf.isInstanceOf[ReadOnlySQLConf] &&
conf.getConfString(StaticSQLConf.CODEGEN_CACHE_MAX_ENTRIES.key) == "300")
}.collect()
assert(checks.forall(_ == true))
}
}
test("SPARK-22219: refactor to control to generate comment") {
Seq(true, false).foreach { flag =>
withSQLConf(StaticSQLConf.CODEGEN_COMMENTS.key -> flag.toString) {
val res = codegenStringSeq(spark.range(10).groupBy(col("id") * 2).count()
.queryExecution.executedPlan)
assert(res.length == 2)
assert(res.forall { case (_, code) =>
(code.contains("* Codegend pipeline") == flag) &&
(code.contains("// input[") == flag)
})
}
}
}
}
Example 37
| Source File: OrcTest.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive.orc
import java.io.File
import scala.reflect.ClassTag
import scala.reflect.runtime.universe.TypeTag
import org.apache.spark.sql._
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
private[sql] trait OrcTest extends SQLTestUtils with TestHiveSingleton {
import testImplicits._
protected def withOrcTable[T <: Product: ClassTag: TypeTag]
(data: Seq[T], tableName: String)
(f: => Unit): Unit = {
withOrcDataFrame(data) { df =>
df.createOrReplaceTempView(tableName)
withTempView(tableName)(f)
}
}
protected def makeOrcFile[T <: Product: ClassTag: TypeTag](
data: Seq[T], path: File): Unit = {
data.toDF().write.mode(SaveMode.Overwrite).orc(path.getCanonicalPath)
}
protected def makeOrcFile[T <: Product: ClassTag: TypeTag](
df: DataFrame, path: File): Unit = {
df.write.mode(SaveMode.Overwrite).orc(path.getCanonicalPath)
}
}
Example 38
| Source File: MetastoreRelationSuite.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive
import org.apache.spark.sql.{QueryTest, Row}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType}
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
class MetastoreRelationSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
test("makeCopy and toJSON should work") {
val table = CatalogTable(
identifier = TableIdentifier("test", Some("db")),
tableType = CatalogTableType.VIEW,
storage = CatalogStorageFormat.empty,
schema = StructType(StructField("a", IntegerType, true) :: Nil))
val relation = MetastoreRelation("db", "test")(table, null)
// No exception should be thrown
relation.makeCopy(Array("db", "test"))
// No exception should be thrown
relation.toJSON
}
test("SPARK-17409: Do Not Optimize Query in CTAS (Hive Serde Table) More Than Once") {
withTable("bar") {
withTempView("foo") {
sql("select 0 as id").createOrReplaceTempView("foo")
// If we optimize the query in CTAS more than once, the following saveAsTable will fail
// with the error: `GROUP BY position 0 is not in select list (valid range is [1, 1])`
sql("CREATE TABLE bar AS SELECT * FROM foo group by id")
checkAnswer(spark.table("bar"), Row(0) :: Nil)
val tableMetadata = spark.sessionState.catalog.getTableMetadata(TableIdentifier("bar"))
assert(tableMetadata.provider == Some("hive"), "the expected table is a Hive serde table")
}
}
}
}
Example 39
| Source File: PruneFileSourcePartitionsSuite.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive.execution
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, LogicalRelation, PruneFileSourcePartitions}
import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.types.StructType
class PruneFileSourcePartitionsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
object Optimize extends RuleExecutor[LogicalPlan] {
val batches = Batch("PruneFileSourcePartitions", Once, PruneFileSourcePartitions) :: Nil
}
test("PruneFileSourcePartitions should not change the output of LogicalRelation") {
withTable("test") {
withTempDir { dir =>
sql(
s"""
|CREATE EXTERNAL TABLE test(i int)
|PARTITIONED BY (p int)
|STORED AS parquet
|LOCATION '${dir.getAbsolutePath}'""".stripMargin)
val tableMeta = spark.sharedState.externalCatalog.getTable("default", "test")
val catalogFileIndex = new CatalogFileIndex(spark, tableMeta, 0)
val dataSchema = StructType(tableMeta.schema.filterNot { f =>
tableMeta.partitionColumnNames.contains(f.name)
})
val relation = HadoopFsRelation(
location = catalogFileIndex,
partitionSchema = tableMeta.partitionSchema,
dataSchema = dataSchema,
bucketSpec = None,
fileFormat = new ParquetFileFormat(),
options = Map.empty)(sparkSession = spark)
val logicalRelation = LogicalRelation(relation, catalogTable = Some(tableMeta))
val query = Project(Seq('i, 'p), Filter('p === 1, logicalRelation)).analyze
val optimized = Optimize.execute(query)
assert(optimized.missingInput.isEmpty)
}
}
}
}
Example 40
| Source File: HiveUtilsSuite.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.hive
import org.apache.hadoop.hive.conf.HiveConf.ConfVars
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.QueryTest
class HiveUtilsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
test("newTemporaryConfiguration overwrites listener configurations") {
Seq(true, false).foreach { useInMemoryDerby =>
val conf = HiveUtils.newTemporaryConfiguration(useInMemoryDerby)
assert(conf(ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname) === "")
assert(conf(ConfVars.METASTORE_EVENT_LISTENERS.varname) === "")
assert(conf(ConfVars.METASTORE_END_FUNCTION_LISTENERS.varname) === "")
}
}
}
Example 41
| Source File: CommitFailureTestRelationSuite.scala From sparkoscope with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.sources
import org.apache.hadoop.fs.Path
import org.apache.spark.SparkException
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql.functions._
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
class CommitFailureTestRelationSuite extends SQLTestUtils with TestHiveSingleton {
// When committing a task, `CommitFailureTestSource` throws an exception for testing purpose.
val dataSourceName: String = classOf[CommitFailureTestSource].getCanonicalName
test("SPARK-7684: commitTask() failure should fallback to abortTask()") {
withTempPath { file =>
// Here we coalesce partition number to 1 to ensure that only a single task is issued. This
// prevents race condition happened when FileOutputCommitter tries to remove the `_temporary`
// directory while committing/aborting the job. See SPARK-8513 for more details.
val df = spark.range(0, 10).coalesce(1)
intercept[SparkException] {
df.write.format(dataSourceName).save(file.getCanonicalPath)
}
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
test("call failure callbacks before close writer - default") {
SimpleTextRelation.failCommitter = false
withTempPath { file =>
// fail the job in the middle of writing
val divideByZero = udf((x: Int) => { x / (x - 1)})
val df = spark.range(0, 10).coalesce(1).select(divideByZero(col("id")))
SimpleTextRelation.callbackCalled = false
intercept[SparkException] {
df.write.format(dataSourceName).save(file.getCanonicalPath)
}
assert(SimpleTextRelation.callbackCalled, "failure callback should be called")
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
test("call failure callbacks before close writer - partitioned") {
SimpleTextRelation.failCommitter = false
withTempPath { file =>
// fail the job in the middle of writing
val df = spark.range(0, 10).coalesce(1).select(col("id").mod(2).as("key"), col("id"))
SimpleTextRelation.callbackCalled = false
SimpleTextRelation.failWriter = true
intercept[SparkException] {
df.write.format(dataSourceName).partitionBy("key").save(file.getCanonicalPath)
}
assert(SimpleTextRelation.callbackCalled, "failure callback should be called")
val fs = new Path(file.getCanonicalPath).getFileSystem(SparkHadoopUtil.get.conf)
assert(!fs.exists(new Path(file.getCanonicalPath, "_temporary")))
}
}
}
Example 42
| Source File: DeltaErrorsSuite.scala From delta with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.delta
import scala.sys.process.Process
import org.apache.hadoop.fs.Path
import org.scalatest.GivenWhenThen
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils}
trait DeltaErrorsSuiteBase
extends QueryTest
with SharedSparkSession with GivenWhenThen
with SQLTestUtils {
val MAX_URL_ACCESS_RETRIES = 3
val path = "/sample/path"
// Map of error name to the actual error message it throws
// When adding an error, add the name of the function throwing the error as the key and the value
// as the error being thrown
def errorsToTest: Map[String, Throwable] = Map(
"useDeltaOnOtherFormatPathException" ->
DeltaErrors.useDeltaOnOtherFormatPathException("operation", path, spark),
"useOtherFormatOnDeltaPathException" ->
DeltaErrors.useOtherFormatOnDeltaPathException("operation", path, path, "format", spark),
"createExternalTableWithoutLogException" ->
DeltaErrors.createExternalTableWithoutLogException(new Path(path), "tableName", spark),
"createExternalTableWithoutSchemaException" ->
DeltaErrors.createExternalTableWithoutSchemaException(new Path(path), "tableName", spark),
"createManagedTableWithoutSchemaException" ->
DeltaErrors.createManagedTableWithoutSchemaException("tableName", spark),
"multipleSourceRowMatchingTargetRowInMergeException" ->
DeltaErrors.multipleSourceRowMatchingTargetRowInMergeException(spark),
"concurrentModificationException" -> new ConcurrentWriteException(None))
def otherMessagesToTest: Map[String, String] = Map(
"deltaFileNotFoundHint" ->
DeltaErrors.deltaFileNotFoundHint(
DeltaErrors.generateDocsLink(
sparkConf,
DeltaErrors.faqRelativePath,
skipValidation = true), path))
def errorMessagesToTest: Map[String, String] =
errorsToTest.mapValues(_.getMessage) ++ otherMessagesToTest
def checkIfValidResponse(url: String, response: String): Boolean = {
response.contains("HTTP/1.1 200 OK") || response.contains("HTTP/2 200")
}
def getUrlsFromMessage(message: String): List[String] = {
val regexToFindUrl = "https://[^\\s]+".r
regexToFindUrl.findAllIn(message).toList
}
def testUrls(): Unit = {
errorMessagesToTest.foreach { case (errName, message) =>
getUrlsFromMessage(message).foreach { url =>
Given(s"*** Checking response for url: $url")
var response = ""
(1 to MAX_URL_ACCESS_RETRIES).foreach { attempt =>
if (attempt > 1) Thread.sleep(1000)
response = Process("curl -I " + url).!!
if (!checkIfValidResponse(url, response)) {
fail(
s"""
|A link to the URL: '$url' is broken in the error: $errName, accessing this URL
|does not result in a valid response, received the following response: $response
""".stripMargin)
}
}
}
}
}
test("Validate that links to docs in DeltaErrors are correct") {
testUrls()
}
}
class DeltaErrorsSuite
extends DeltaErrorsSuiteBase
Example 43
| Source File: EvolvabilitySuite.scala From delta with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.delta
import org.apache.spark.sql.delta.util.{FileNames, JsonUtils}
import org.apache.hadoop.fs.Path
import org.apache.spark.sql.functions.typedLit
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.util.Utils
class EvolvabilitySuite extends EvolvabilitySuiteBase with SQLTestUtils {
import testImplicits._
test("delta 0.1.0") {
testEvolvability("src/test/resources/delta/delta-0.1.0")
}
test("delta 0.1.0 - case sensitivity enabled") {
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
testEvolvability("src/test/resources/delta/delta-0.1.0")
}
}
testQuietly("future proofing against new features") {
val tempDir = Utils.createTempDir().toString
Seq(1, 2, 3).toDF().write.format("delta").save(tempDir)
val deltaLog = DeltaLog.forTable(spark, tempDir)
deltaLog.store.write(new Path(deltaLog.logPath, "00000000000000000001.json"),
Iterator("""{"some_new_feature":{"a":1}}"""))
// Shouldn't fail here
deltaLog.update()
val sq = spark.readStream.format("delta").load(tempDir.toString)
.groupBy()
.count()
.writeStream
.outputMode("complete")
.format("console")
.start()
// Also shouldn't fail
sq.processAllAvailable()
Seq(1, 2, 3).toDF().write.format("delta").mode("append").save(tempDir)
sq.processAllAvailable()
deltaLog.store.write(new Path(deltaLog.logPath, "00000000000000000003.json"),
Iterator("""{"some_new_feature":{"a":1}}"""))
sq.processAllAvailable()
sq.stop()
}
test("serialized partition values must contain null values") {
val tempDir = Utils.createTempDir().toString
val df1 = spark.range(5).withColumn("part", typedLit[String](null))
val df2 = spark.range(5).withColumn("part", typedLit("1"))
df1.union(df2).coalesce(1).write.partitionBy("part").format("delta").save(tempDir)
// Clear the cache
DeltaLog.clearCache()
val deltaLog = DeltaLog.forTable(spark, tempDir)
val dataThere = deltaLog.snapshot.allFiles.collect().forall { addFile =>
if (!addFile.partitionValues.contains("part")) {
fail(s"The partition values: ${addFile.partitionValues} didn't contain the column 'part'.")
}
val value = addFile.partitionValues("part")
value === null || value === "1"
}
assert(dataThere, "Partition values didn't match with null or '1'")
// Check serialized JSON as well
val contents = deltaLog.store.read(FileNames.deltaFile(deltaLog.logPath, 0L))
assert(contents.exists(_.contains(""""part":null""")), "null value should be written in json")
}
testQuietly("parse old version CheckpointMetaData") {
assert(JsonUtils.mapper.readValue[CheckpointMetaData]("""{"version":1,"size":1}""")
== CheckpointMetaData(1, 1, None))
}
}
Example 44
| Source File: HiveConvertToDeltaSuite.scala From delta with Apache License 2.0 | 5 votes |
|
package org.apache.spark.sql.delta
import org.apache.spark.sql.delta.test.DeltaHiveTest
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
abstract class HiveConvertToDeltaSuiteBase
extends ConvertToDeltaHiveTableTests
with SQLTestUtils {
override protected def convertToDelta(
identifier: String,
partitionSchema: Option[String] = None): Unit = {
if (partitionSchema.isEmpty) {
sql(s"convert to delta $identifier")
} else {
val stringSchema = partitionSchema.get
sql(s"convert to delta $identifier partitioned by ($stringSchema) ")
}
}
override protected def verifyExternalCatalogMetadata(tableName: String): Unit = {
val catalogTable = spark.sessionState.catalog.externalCatalog.getTable("default", tableName)
// Hive automatically adds some properties
val cleanProps = catalogTable.properties.filterKeys(_ != "transient_lastDdlTime")
// We can't alter the schema in the catalog at the moment :(
assert(cleanProps.isEmpty,
s"Table properties weren't empty for table $tableName: $cleanProps")
}
test("convert a Hive based parquet table") {
val tbl = "hive_parquet"
withTable(tbl) {
sql(
s"""
|CREATE TABLE $tbl (id int, str string)
|PARTITIONED BY (part string)
|STORED AS PARQUET
""".stripMargin)
sql(s"insert into $tbl VALUES (1, 'a', 1)")
val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tbl))
assert(catalogTable.provider === Some("hive"))
assert(catalogTable.storage.serde.exists(_.contains("parquet")))
convertToDelta(tbl, Some("part string"))
checkAnswer(
sql(s"select * from delta.`${getPathForTableName(tbl)}`"),
Row(1, "a", "1"))
verifyExternalCatalogMetadata(tbl)
val updatedTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tbl))
assert(updatedTable.provider === Some("delta"))
}
}
test("convert a Hive based external parquet table") {
val tbl = "hive_parquet"
withTempDir { dir =>
withTable(tbl) {
sql(
s"""
|CREATE EXTERNAL TABLE $tbl (id int, str string)
|PARTITIONED BY (part string)
|STORED AS PARQUET
|LOCATION '${dir.getCanonicalPath}'
""".stripMargin)
sql(s"insert into $tbl VALUES (1, 'a', 1)")
val catalogTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tbl))
assert(catalogTable.provider === Some("hive"))
assert(catalogTable.storage.serde.exists(_.contains("parquet")))
convertToDelta(tbl, Some("part string"))
checkAnswer(
sql(s"select * from delta.`${dir.getCanonicalPath}`"),
Row(1, "a", "1"))
verifyExternalCatalogMetadata(tbl)
val updatedTable = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tbl))
assert(updatedTable.provider === Some("delta"))
}
}
}
}
class HiveConvertToDeltaSuite extends HiveConvertToDeltaSuiteBase with DeltaHiveTest
