Java Code Examples for org.apache.spark.sql.SparkSession#emptyDataset()

The following examples show how to use org.apache.spark.sql.SparkSession#emptyDataset() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ValueSets.java    From bunsen with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns an empty ValueSets instance.
 *
 * @param spark the spark session
 * @return an empty ValueSets instance.
 */
public static ValueSets getEmpty(SparkSession spark) {

  Dataset<Row> emptyValueSets = valuesetRowConverter.emptyDataFrame(spark)
      .withColumn("timestamp", lit(null).cast("timestamp"));

  return new ValueSets(spark,
      spark.emptyDataset(URL_AND_VERSION_ENCODER),
      emptyValueSets,
      spark.emptyDataset(getValueEncoder()));
}
Example 2
Source File: ConceptMaps.java    From bunsen with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns the collection of concept maps from the tables in the given database.
 *
 * @param spark the spark session
 * @param databaseName name of the database containing the conceptmaps and mappings tables.
 * @return a ConceptMaps instance.
 */
public static ConceptMaps getFromDatabase(SparkSession spark, String databaseName) {

  Dataset<Mapping> mappings = spark.table(databaseName + "." + MAPPING_TABLE)
      .as(MAPPING_ENCODER);

  Dataset<Row> conceptMaps = spark.table(databaseName + "." + CONCEPT_MAP_TABLE);

  return new ConceptMaps(spark,
      spark.emptyDataset(URL_AND_VERSION_ENCODER),
      conceptMaps,
      mappings);
}
Example 3
Source File: ConceptMaps.java    From bunsen with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns an empty ConceptMaps instance.
 *
 * @param spark the spark session
 * @return an empty ConceptMaps instance.
 */
public static ConceptMaps getEmpty(SparkSession spark) {

  Dataset<Row> emptyConceptMaps = conceptMapConverter.emptyDataFrame(spark)
      .withColumn("timestamp", lit(null).cast("timestamp"));

  return new ConceptMaps(spark,
      spark.emptyDataset(URL_AND_VERSION_ENCODER),
      emptyConceptMaps,
      spark.emptyDataset(MAPPING_ENCODER));
}
Example 4
Source File: ValueSets.java    From bunsen with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns an empty ValueSets instance.
 *
 * @param spark the spark session
 * @return an empty ValueSets instance.
 */
public static ValueSets getEmpty(SparkSession spark) {

  Dataset<ValueSet> emptyValueSets = spark.emptyDataset(VALUE_SET_ENCODER)
      .withColumn("timestamp", lit(null).cast("timestamp"))
      .as(VALUE_SET_ENCODER);

  return new ValueSets(spark,
      spark.emptyDataset(URL_AND_VERSION_ENCODER),
      emptyValueSets,
      spark.emptyDataset(getValueEncoder()));
}
Example 5
Source File: ConceptMaps.java    From bunsen with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns the collection of concept maps from the tables in the given database.
 *
 * @param spark the spark session
 * @param databaseName name of the database containing the conceptmaps and mappings tables.
 * @return a ConceptMaps instance.
 */
public static ConceptMaps getFromDatabase(SparkSession spark, String databaseName) {

  Dataset<Mapping> mappings = spark.sql(
      "SELECT * FROM " + databaseName + "." + MAPPING_TABLE).as(MAPPING_ENCODER);

  Dataset<ConceptMap> conceptMaps = spark
      .sql("SELECT * FROM " + databaseName + "." + CONCEPT_MAP_TABLE)
      .as(CONCEPT_MAP_ENCODER);

  return new ConceptMaps(spark,
      spark.emptyDataset(URL_AND_VERSION_ENCODER),
      conceptMaps,
      mappings);
}
Example 6
Source File: ConceptMaps.java    From bunsen with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns an empty ConceptMaps instance.
 *
 * @param spark the spark session
 * @return an empty ConceptMaps instance.
 */
public static ConceptMaps getEmpty(SparkSession spark) {

  Dataset<ConceptMap> emptyConceptMaps = spark.emptyDataset(CONCEPT_MAP_ENCODER)
      .withColumn("timestamp", lit(null).cast("timestamp"))
      .as(CONCEPT_MAP_ENCODER);

  return new ConceptMaps(spark,
      spark.emptyDataset(URL_AND_VERSION_ENCODER),
      emptyConceptMaps,
      spark.emptyDataset(MAPPING_ENCODER));
}
Example 7
Source File: Hierarchies.java    From bunsen with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns an empty Hierarchies instance.
 *
 * @param spark the spark session
 * @return an empty Hierarchies instance.
 */
public static Hierarchies getEmpty(SparkSession spark) {

  return new Hierarchies(spark,
      spark.emptyDataset(URI_AND_VERSION_ENCODER),
      spark.emptyDataset(ANCESTOR_ENCODER));
}
Example 8
Source File: MockValueSets.java    From bunsen with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates an empty MockValueSets instance for test purposes.
 */
public MockValueSets(SparkSession spark, SparkRowConverter valuesetRowConverter) {
  super(spark,
      FhirVersionEnum.DSTU3,
      spark.emptyDataset(AbstractValueSets.getUrlAndVersionEncoder()),
      valuesetRowConverter.emptyDataFrame(spark),
      spark.emptyDataset(AbstractValueSets.getValueEncoder()),
      valuesetRowConverter);
}