Java Code Examples for org.elasticsearch.spark.sql.api.java.JavaEsSparkSQL#saveToEs()

The following examples show how to use org.elasticsearch.spark.sql.api.java.JavaEsSparkSQL#saveToEs() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AbstractJavaEsSparkSQLTest.java    From elasticsearch-hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testEsDataFrameReadMetadata() throws Exception {
	DataFrame artists = artistsAsDataFrame();
	String target = resource("sparksql-test-scala-dataframe-read-metadata", "data", version);
	JavaEsSparkSQL.saveToEs(artists, target);

	DataFrame dataframe = sqc.read().format("es").option("es.read.metadata", "true").load(target).where("id = 1");

	// Since _metadata field isn't a part of _source,
	// we want to check that it could be fetched in any position.
	assertEquals("sparksql-test-scala-dataframe-read-metadata", dataframe.selectExpr("_metadata['_index']").takeAsList(1).get(0).get(0));
	assertEquals("sparksql-test-scala-dataframe-read-metadata", dataframe.selectExpr("_metadata['_index']", "name").takeAsList(1).get(0).get(0));
	assertEquals("MALICE MIZER", dataframe.selectExpr("_metadata['_index']", "name").takeAsList(1).get(0).get(1));
	assertEquals("MALICE MIZER", dataframe.selectExpr("name", "_metadata['_index']").takeAsList(1).get(0).get(0));
	assertEquals("sparksql-test-scala-dataframe-read-metadata", dataframe.selectExpr("name", "_metadata['_index']").takeAsList(1).get(0).get(1));
}
Example 2
Source File: ElasticSearchPersistor.java    From rdf2x with Apache License 2.0 5 votes vote down vote up 
/**
 * Write a {@link DataFrame} to the specified output
 *
 * @param name name of output table
 * @param df   dataframe containing the data
 */
@Override
public void writeDataFrame(String name, DataFrame df) {
    Map<String, String> props = config.getProperties(name);
    log.info("Writing to ElasticSearch: {}", props);
    JavaEsSparkSQL.saveToEs(df, props);
}
Example 3
Source File: AbstractJavaEsSparkSQLTest.java    From elasticsearch-hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
   public void testEsDataset1WriteWithId() throws Exception {
       Dataset<Row> dataset = artistsAsDataset();

	String target = resource("sparksql-test-scala-basic-write-id-mapping", "data", version);
	String docEndpoint = docEndpoint("sparksql-test-scala-basic-write-id-mapping", "data", version);

       JavaEsSparkSQL.saveToEs(dataset, target,
			ImmutableMap.of(ES_MAPPING_ID, "id"));
	assertTrue(RestUtils.exists(target));
	assertThat(RestUtils.get(target + "/_search?"), containsString("345"));
	assertThat(RestUtils.exists(docEndpoint + "/1"), is(true));
}
Example 4
Source File: AbstractJavaEsSparkSQLTest.java    From elasticsearch-hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testEsSchemaRDD1WriteWithMappingExclude() throws Exception {
    Dataset<Row> dataset = artistsAsDataset();

    String target = resource("sparksql-test-scala-basic-write-exclude-mapping", "data", version);
    JavaEsSparkSQL.saveToEs(dataset, target,
            ImmutableMap.of(ES_MAPPING_EXCLUDE, "url"));
    assertTrue(RestUtils.exists(target));
    assertThat(RestUtils.get(target + "/_search?"), not(containsString("url")));
}
Example 5
Source File: AbstractJavaEsSparkSQLTest.java    From elasticsearch-hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testEsdataFrame1Write() throws Exception {
	DataFrame dataFrame = artistsAsDataFrame();

	String target = resource("sparksql-test-scala-basic-write", "data", version);
	JavaEsSparkSQL.saveToEs(dataFrame, target);
	assertTrue(RestUtils.exists(target));
	assertThat(RestUtils.get(target + "/_search?"), containsString("345"));
}
Example 6
Source File: AbstractJavaEsSparkSQLTest.java    From elasticsearch-hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testEsdataFrame1WriteWithId() throws Exception {
	DataFrame dataFrame = artistsAsDataFrame();

	String target = resource("sparksql-test-scala-basic-write-id-mapping", "data", version);
	String docEndpoint = docEndpoint("sparksql-test-scala-basic-write-id-mapping", "data", version);

	JavaEsSparkSQL.saveToEs(dataFrame, target,
			ImmutableMap.of(ES_MAPPING_ID, "id"));
	assertTrue(RestUtils.exists(target));
	assertThat(RestUtils.get(target + "/_search?"), containsString("345"));
	assertThat(RestUtils.exists(docEndpoint + "/1"), is(true));
}
Example 7
Source File: AbstractJavaEsSparkSQLTest.java    From elasticsearch-hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testEsSchemaRDD1WriteWithMappingExclude() throws Exception {
	DataFrame dataFrame = artistsAsDataFrame();

    String target = resource("sparksql-test-scala-basic-write-exclude-mapping", "data", version);
    JavaEsSparkSQL.saveToEs(dataFrame, target,ImmutableMap.of(ES_MAPPING_EXCLUDE, "url"));
    assertTrue(RestUtils.exists(target));
    assertThat(RestUtils.get(target + "/_search?"), not(containsString("url")));
}
Example 8
Source File: AbstractJavaEsSparkSQLTest.java    From elasticsearch-hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
   public void testEsDataset1Write() throws Exception {
       Dataset<Row> dataset = artistsAsDataset();

	String target = resource("sparksql-test-scala-basic-write", "data", version);
       JavaEsSparkSQL.saveToEs(dataset, target);
	assertTrue(RestUtils.exists(target));
	assertThat(RestUtils.get(target + "/_search?"), containsString("345"));
}