Java Code Examples for org.apache.spark.sql.sources.v2.DataSourceOptions#get()

The following examples show how to use org.apache.spark.sql.sources.v2.DataSourceOptions#get() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: IcebergSource.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
protected Table findTable(DataSourceOptions options, Configuration conf) {
  Optional<String> path = options.get("path");
  Preconditions.checkArgument(path.isPresent(), "Cannot open table: path is not set");

  if (path.get().contains("/")) {
    HadoopTables tables = new HadoopTables(conf);
    return tables.load(path.get());
  } else {
    HiveCatalog hiveCatalog = HiveCatalogs.loadCatalog(conf);
    TableIdentifier tableIdentifier = TableIdentifier.parse(path.get());
    return hiveCatalog.loadTable(tableIdentifier);
  }
}
Example 2
Source File: IcebergSource.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
@Override
public Optional<DataSourceWriter> createWriter(String jobId, StructType dfStruct, SaveMode mode,
                                                 DataSourceOptions options) {
  Preconditions.checkArgument(mode == SaveMode.Append, "Save mode %s is not supported", mode);

  Table table = findTable(options);

  Schema dfSchema = SparkSchemaUtil.convert(table.schema(), dfStruct);
  List<String> errors = CheckCompatibility.writeCompatibilityErrors(table.schema(), dfSchema);
  if (!errors.isEmpty()) {
    StringBuilder sb = new StringBuilder();
    sb.append("Cannot write incompatible dataframe to table with schema:\n")
        .append(table.schema()).append("\nProblems:");
    for (String error : errors) {
      sb.append("\n* ").append(error);
    }
    throw new IllegalArgumentException(sb.toString());
  }

  Optional<String> formatOption = options.get("iceberg.write.format");
  FileFormat format;
  if (formatOption.isPresent()) {
    format = FileFormat.valueOf(formatOption.get().toUpperCase(Locale.ENGLISH));
  } else {
    format = FileFormat.valueOf(table.properties()
        .getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT)
        .toUpperCase(Locale.ENGLISH));
  }

  return Optional.of(new Writer(table, lazyConf(), format));
}
Example 3
Source File: IcebergSource.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
protected Table findTable(DataSourceOptions options) {
  Optional<String> location = options.get("path");
  Preconditions.checkArgument(location.isPresent(),
      "Cannot open table without a location: path is not set");

  HadoopTables tables = new HadoopTables(lazyConf());

  return tables.load(location.get());
}
Example 4
Source File: Writer.java    From iceberg with Apache License 2.0 4 votes vote down vote up 
private FileFormat getFileFormat(Map<String, String> tableProperties, DataSourceOptions options) {
  Optional<String> formatOption = options.get("write-format");
  String formatString = formatOption
      .orElse(tableProperties.getOrDefault(DEFAULT_FILE_FORMAT, DEFAULT_FILE_FORMAT_DEFAULT));
  return FileFormat.valueOf(formatString.toUpperCase(Locale.ENGLISH));
}