org.elasticsearch.spark.sql.api.java.JavaEsSparkSQL Java Examples

The following examples show how to use org.elasticsearch.spark.sql.api.java.JavaEsSparkSQL. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: AbstractJavaEsSparkSQLTest.java From elasticsearch-hadoop with Apache License 2.0

6 votes

@Test
public void testEsDataFrameReadMetadata() throws Exception {
	DataFrame artists = artistsAsDataFrame();
	String target = resource("sparksql-test-scala-dataframe-read-metadata", "data", version);
	JavaEsSparkSQL.saveToEs(artists, target);

	DataFrame dataframe = sqc.read().format("es").option("es.read.metadata", "true").load(target).where("id = 1");

	// Since _metadata field isn't a part of _source,
	// we want to check that it could be fetched in any position.
	assertEquals("sparksql-test-scala-dataframe-read-metadata", dataframe.selectExpr("_metadata['_index']").takeAsList(1).get(0).get(0));
	assertEquals("sparksql-test-scala-dataframe-read-metadata", dataframe.selectExpr("_metadata['_index']", "name").takeAsList(1).get(0).get(0));
	assertEquals("MALICE MIZER", dataframe.selectExpr("_metadata['_index']", "name").takeAsList(1).get(0).get(1));
	assertEquals("MALICE MIZER", dataframe.selectExpr("name", "_metadata['_index']").takeAsList(1).get(0).get(0));
	assertEquals("sparksql-test-scala-dataframe-read-metadata", dataframe.selectExpr("name", "_metadata['_index']").takeAsList(1).get(0).get(1));
}

Example #2

Source File: ElasticSearchPersistor.java From rdf2x with Apache License 2.0

5 votes

/**
 * Write a {@link DataFrame} to the specified output
 *
 * @param name name of output table
 * @param df   dataframe containing the data
 */
@Override
public void writeDataFrame(String name, DataFrame df) {
    Map<String, String> props = config.getProperties(name);
    log.info("Writing to ElasticSearch: {}", props);
    JavaEsSparkSQL.saveToEs(df, props);
}

Example #3

Source File: AbstractJavaEsSparkSQLTest.java From elasticsearch-hadoop with Apache License 2.0

5 votes

@Test
   public void testEsDataset1WriteWithId() throws Exception {
       Dataset<Row> dataset = artistsAsDataset();

	String target = resource("sparksql-test-scala-basic-write-id-mapping", "data", version);
	String docEndpoint = docEndpoint("sparksql-test-scala-basic-write-id-mapping", "data", version);

       JavaEsSparkSQL.saveToEs(dataset, target,
			ImmutableMap.of(ES_MAPPING_ID, "id"));
	assertTrue(RestUtils.exists(target));
	assertThat(RestUtils.get(target + "/_search?"), containsString("345"));
	assertThat(RestUtils.exists(docEndpoint + "/1"), is(true));
}

Example #4

Source File: AbstractJavaEsSparkSQLTest.java From elasticsearch-hadoop with Apache License 2.0

5 votes

@Test
public void testEsSchemaRDD1WriteWithMappingExclude() throws Exception {
    Dataset<Row> dataset = artistsAsDataset();

    String target = resource("sparksql-test-scala-basic-write-exclude-mapping", "data", version);
    JavaEsSparkSQL.saveToEs(dataset, target,
            ImmutableMap.of(ES_MAPPING_EXCLUDE, "url"));
    assertTrue(RestUtils.exists(target));
    assertThat(RestUtils.get(target + "/_search?"), not(containsString("url")));
}

Example #5

Source File: AbstractJavaEsSparkSQLTest.java From elasticsearch-hadoop with Apache License 2.0

5 votes

@Test
public void testEsdataFrame1Write() throws Exception {
	DataFrame dataFrame = artistsAsDataFrame();

	String target = resource("sparksql-test-scala-basic-write", "data", version);
	JavaEsSparkSQL.saveToEs(dataFrame, target);
	assertTrue(RestUtils.exists(target));
	assertThat(RestUtils.get(target + "/_search?"), containsString("345"));
}

Example #6

Source File: AbstractJavaEsSparkSQLTest.java From elasticsearch-hadoop with Apache License 2.0

5 votes

@Test
public void testEsdataFrame1WriteWithId() throws Exception {
	DataFrame dataFrame = artistsAsDataFrame();

	String target = resource("sparksql-test-scala-basic-write-id-mapping", "data", version);
	String docEndpoint = docEndpoint("sparksql-test-scala-basic-write-id-mapping", "data", version);

	JavaEsSparkSQL.saveToEs(dataFrame, target,
			ImmutableMap.of(ES_MAPPING_ID, "id"));
	assertTrue(RestUtils.exists(target));
	assertThat(RestUtils.get(target + "/_search?"), containsString("345"));
	assertThat(RestUtils.exists(docEndpoint + "/1"), is(true));
}

Example #7

Source File: AbstractJavaEsSparkSQLTest.java From elasticsearch-hadoop with Apache License 2.0

5 votes

@Test
public void testEsSchemaRDD1WriteWithMappingExclude() throws Exception {
	DataFrame dataFrame = artistsAsDataFrame();

    String target = resource("sparksql-test-scala-basic-write-exclude-mapping", "data", version);
    JavaEsSparkSQL.saveToEs(dataFrame, target,ImmutableMap.of(ES_MAPPING_EXCLUDE, "url"));
    assertTrue(RestUtils.exists(target));
    assertThat(RestUtils.get(target + "/_search?"), not(containsString("url")));
}

Example #8

Source File: AbstractJavaEsSparkSQLTest.java From elasticsearch-hadoop with Apache License 2.0

4 votes

@Test
   public void testEsDataset1Write() throws Exception {
       Dataset<Row> dataset = artistsAsDataset();

	String target = resource("sparksql-test-scala-basic-write", "data", version);
       JavaEsSparkSQL.saveToEs(dataset, target);
	assertTrue(RestUtils.exists(target));
	assertThat(RestUtils.get(target + "/_search?"), containsString("345"));
}

Example #9

Source File: TaskRestHandler.java From elasticsearch-rest-command with The Unlicense

3 votes

private JavaSchemaRDD  getSchemaRDD(JavaSQLContext sqlContext, String command) throws ParseException, CommandException, IOException{
	CommandParser parser = new CommandParser(command);

	Search search = new Search(parser, client, logger);
	
	if(search.indices.length == 0 || search.indices.length > 1)
		throw new InvalidParameterException(String.format("indices.length = %d", search.indices.length));
	
	if(search.sourceTypes.length == 0 || search.sourceTypes.length > 1)
		throw new InvalidParameterException(String.format("sourceTypes.length = %d", search.sourceTypes.length));
	
	String query = search.querySearch.toString();
	
	return JavaEsSparkSQL.esRDD(sqlContext, String.format("%s/%s", search.indices[0], search.sourceTypes[0]), query);
}