Java Code Examples for org.elasticsearch.common.xcontent.XContentType#JSON
The following examples show how to use
org.elasticsearch.common.xcontent.XContentType#JSON .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: ArrayMapperTest.java From crate with Apache License 2.0 | 6 votes |
|
@Test
public void testParseDynamicNullArray() throws Exception {
String mapping = Strings.toString(XContentFactory.jsonBuilder()
.startObject().startObject(TYPE).startObject("properties")
.endObject().endObject().endObject());
DocumentMapper mapper = mapper(INDEX, mapping);
// parse source with null array
BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
.startObject()
.startArray("new_array_field").nullValue().endArray()
.endObject());
SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
ParsedDocument doc = mapper.parse(sourceToParse);
assertThat(doc.docs().get(0).getField("new_array_field"), is(nullValue()));
assertThat(mapper.mappers().getMapper("new_array_field"), is(nullValue()));
}
Example 2
| Source File: ArrayMapperTest.java From crate with Apache License 2.0 | 6 votes |
|
@Test
public void testInvalidArrayNonConvertableType() throws Exception {
String mapping = Strings.toString(XContentFactory.jsonBuilder()
.startObject().startObject(TYPE).startObject("properties")
.startObject("array_field")
.field("type", ArrayMapper.CONTENT_TYPE)
.startObject(ArrayMapper.INNER_TYPE)
.field("type", "double")
.endObject()
.endObject()
.endObject().endObject().endObject());
DocumentMapper mapper = mapper(INDEX, mapping);
expectedException.expect(MapperParsingException.class);
expectedException.expectMessage("failed to parse field [array_field] of type [double]");
BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
.startObject()
.array("array_field", true, false, true)
.endObject());
SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
mapper.parse(sourceToParse);
}
Example 3
| Source File: RequestResolver.java From deprecated-security-advanced-modules with Apache License 2.0 | 5 votes |
|
private static Tuple<XContentType, BytesReference> convertSource(XContentType type, BytesReference bytes) {
if(type == null) {
type = XContentType.JSON;
}
return new Tuple<XContentType, BytesReference>(type, bytes);
}
Example 4
| Source File: ArrayMapperTest.java From crate with Apache License 2.0 | 5 votes |
|
@Test
public void testParseNull() throws Exception {
// @formatter: off
String mapping = Strings.toString(XContentFactory.jsonBuilder()
.startObject()
.startObject(TYPE)
.startObject("properties")
.startObject("array_field")
.field("type", ArrayMapper.CONTENT_TYPE)
.startObject(ArrayMapper.INNER_TYPE)
.field("type", "double")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject());
// @formatter: on
DocumentMapper mapper = mapper(INDEX, mapping);
BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
.startObject()
.nullField("array_field")
.endObject());
SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
ParsedDocument parsedDoc = mapper.parse(sourceToParse);
assertThat(parsedDoc.docs().size(), is(1));
assertThat(parsedDoc.docs().get(0).getField("array_field"), is(nullValue()));
}
Example 5
| Source File: DocumentMapper.java From crate with Apache License 2.0 | 5 votes |
|
public ParsedDocument createNoopTombstoneDoc(String index, String reason) throws MapperParsingException {
final String id = ""; // _id won't be used.
final SourceToParse sourceToParse = new SourceToParse(index, id, new BytesArray("{}"), XContentType.JSON);
final ParsedDocument parsedDoc = documentParser.parseDocument(sourceToParse, noopTombstoneMetadataFieldMappers).toTombstone();
// Store the reason of a noop as a raw string in the _source field
final BytesRef byteRef = new BytesRef(reason);
parsedDoc.rootDoc().add(new StoredField(SourceFieldMapper.NAME, byteRef.bytes, byteRef.offset, byteRef.length));
return parsedDoc;
}
Example 6
| Source File: RestChannel.java From Elasticsearch with Apache License 2.0 | 5 votes |
|
public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException {
XContentType contentType = XContentType.fromRestContentType(request.param("format", request.header("Content-Type")));
if (contentType == null) {
// try and guess it from the auto detect source
if (autoDetectSource != null) {
contentType = XContentFactory.xContentType(autoDetectSource);
}
}
if (contentType == null) {
// default to JSON
contentType = XContentType.JSON;
}
String[] filters = useFiltering ? request.paramAsStringArrayOrEmptyIfAll("filter_path") : null;
XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(contentType), bytesOutput(), filters);
if (request.paramAsBoolean("pretty", false)) {
builder.prettyPrint().lfAtEnd();
}
builder.humanReadable(request.paramAsBoolean("human", builder.humanReadable()));
String casing = request.param("case");
if (casing != null) {
String msg = "Parameter 'case' has been deprecated, all responses will use underscore casing in the future";
DEPRECATION_LOGGER.deprecated(msg);
}
if (casing != null && "camelCase".equals(casing)) {
builder.fieldCaseConversion(XContentBuilder.FieldCaseConversion.CAMELCASE);
} else {
// we expect all REST interfaces to write results in underscore casing, so
// no need for double casing
builder.fieldCaseConversion(XContentBuilder.FieldCaseConversion.NONE);
}
return builder;
}
Example 7
| Source File: MappingMetaData.java From crate with Apache License 2.0 | 5 votes |
|
public MappingMetaData(String type, Map<String, Object> mapping) throws IOException {
this.type = type;
this.source = new CompressedXContent(
(builder, params) -> builder.mapContents(mapping), XContentType.JSON, ToXContent.EMPTY_PARAMS);
Map<String, Object> withoutType = mapping;
if (mapping.size() == 1 && mapping.containsKey(type)) {
withoutType = (Map<String, Object>) mapping.get(type);
}
initMappers(withoutType);
}
Example 8
| Source File: ElasticsearchIndexer.java From spring-content with Apache License 2.0 | 5 votes |
|
void ensureAttachmentPipeline() throws IOException {
GetPipelineRequest getRequest = new GetPipelineRequest(SPRING_CONTENT_ATTACHMENT);
GetPipelineResponse res = client.ingest().getPipeline(getRequest, RequestOptions.DEFAULT);
if (!res.isFound()) {
String source = "{\"description\":\"Extract attachment information encoded in Base64 with UTF-8 charset\"," +
"\"processors\":[{\"attachment\":{\"field\":\"data\"}}]}";
PutPipelineRequest put = new PutPipelineRequest(SPRING_CONTENT_ATTACHMENT,
new BytesArray(source.getBytes(StandardCharsets.UTF_8)),
XContentType.JSON);
WritePipelineResponse wpr = client.ingest().putPipeline(put, RequestOptions.DEFAULT);
Assert.isTrue(wpr.isAcknowledged(), "Attachment pipeline not acknowledged by server");
}
}
Example 9
| Source File: MappingContentBuilderTest.java From molgenis with GNU Lesser General Public License v3.0 | 4 votes |
|
@BeforeEach
void setUpBeforeMethod() {
mappingContentBuilder = new MappingContentBuilder(XContentType.JSON);
}
Example 10
| Source File: ArrayMapperTest.java From crate with Apache License 2.0 | 4 votes |
|
@Test
public void testObjectArrayMappingNewColumn() throws Exception {
// @formatter: off
String mapping = Strings.toString(XContentFactory.jsonBuilder()
.startObject()
.startObject(TYPE)
.startObject("properties")
.startObject("array_field")
.field("type", ArrayMapper.CONTENT_TYPE)
.startObject(ArrayMapper.INNER_TYPE)
.field("type", "object")
.field("dynamic", true)
.startObject("properties")
.startObject("s")
.field("type", "keyword")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject());
DocumentMapper mapper = mapper(INDEX, mapping);
// child object mapper
assertThat(mapper.objectMappers().get("array_field"), is(instanceOf(ObjectArrayMapper.class)));
BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
.startObject()
.startArray("array_field")
.startObject()
.field("s", "a")
.field("new", true)
.endObject()
.endArray()
.endObject());
SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
ParsedDocument doc = mapper.parse(sourceToParse);
Mapping mappingUpdate = doc.dynamicMappingsUpdate();
assertThat(mappingUpdate, notNullValue());
mapper = mapper.merge(mappingUpdate, true);
assertThat(doc.docs().size(), is(1));
String[] values = doc.docs().get(0).getValues("array_field.new");
assertThat(values, arrayContainingInAnyOrder(is("T"), is("1")));
String mappingSourceString = new CompressedXContent(mapper, XContentType.JSON, ToXContent.EMPTY_PARAMS).string();
assertThat(
mappingSourceString,
is("{\"default\":{" +
"\"properties\":{" +
"\"array_field\":{" +
"\"type\":\"array\"," +
"\"inner\":{" +
"\"dynamic\":\"true\"," +
"\"properties\":{" +
"\"new\":{\"type\":\"boolean\"}," +
"\"s\":{" +
"\"type\":\"keyword\"" +
"}" +
"}" +
"}" +
"}" +
"}}}"));
}
Example 11
| Source File: SettingsContentBuilder.java From molgenis with GNU Lesser General Public License v3.0 | 4 votes |
|
SettingsContentBuilder() {
this(XContentType.JSON);
}
Example 12
| Source File: ArrayMapperTest.java From crate with Apache License 2.0 | 4 votes |
|
@Test
public void testSimpleArrayMapping() throws Exception {
// @formatter:off
String mapping = Strings.toString(XContentFactory.jsonBuilder()
.startObject()
.startObject(TYPE)
.startObject("properties")
.startObject("array_field")
.field("type", ArrayMapper.CONTENT_TYPE)
.startObject(ArrayMapper.INNER_TYPE)
.field("type", "keyword")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject());
// @formatter:on
DocumentMapper mapper = mapper(INDEX, mapping);
assertThat(mapper.mappers().getMapper("array_field"), is(instanceOf(ArrayMapper.class)));
BytesReference bytesReference = BytesReference.bytes(JsonXContent.contentBuilder()
.startObject()
.array("array_field", "a", "b", "c")
.endObject());
SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
ParsedDocument doc = mapper.parse(sourceToParse);
assertThat(doc.dynamicMappingsUpdate() == null, is(true));
assertThat(doc.docs().size(), is(1));
ParseContext.Document fields = doc.docs().get(0);
Set<String> values = uniqueValuesFromFields(fields, "array_field");
assertThat(values, Matchers.containsInAnyOrder("a", "b", "c"));
assertThat(
mapper.mappingSource().string(),
is("{\"default\":{" +
"\"properties\":{" +
"\"array_field\":{" +
"\"type\":\"array\"," +
"\"inner\":{" +
"\"type\":\"keyword\"" +
"}" +
"}" +
"}" +
"}}"));
}
Example 13
| Source File: RandomObjects.java From crate with Apache License 2.0 | 4 votes |
|
/**
* Returns a tuple containing random stored field values and their corresponding expected values once printed out
* via {@link org.elasticsearch.common.xcontent.ToXContent#toXContent(XContentBuilder, ToXContent.Params)} and parsed back via
* {@link org.elasticsearch.common.xcontent.XContentParser#objectText()}.
* Generates values based on what can get printed out. Stored fields values are retrieved from lucene and converted via
* {@link org.elasticsearch.index.mapper.MappedFieldType#valueForDisplay(Object)} to either strings, numbers or booleans.
*
* @param random Random generator
* @param xContentType the content type, used to determine what the expected values are for float numbers.
*/
public static Tuple<List<Object>, List<Object>> randomStoredFieldValues(Random random, XContentType xContentType) {
int numValues = randomIntBetween(random, 1, 5);
List<Object> originalValues = new ArrayList<>();
List<Object> expectedParsedValues = new ArrayList<>();
int dataType = randomIntBetween(random, 0, 8);
for (int i = 0; i < numValues; i++) {
switch(dataType) {
case 0:
long randomLong = random.nextLong();
originalValues.add(randomLong);
expectedParsedValues.add(randomLong);
break;
case 1:
int randomInt = random.nextInt();
originalValues.add(randomInt);
expectedParsedValues.add(randomInt);
break;
case 2:
Short randomShort = (short) random.nextInt();
originalValues.add(randomShort);
expectedParsedValues.add(randomShort.intValue());
break;
case 3:
Byte randomByte = (byte)random.nextInt();
originalValues.add(randomByte);
expectedParsedValues.add(randomByte.intValue());
break;
case 4:
double randomDouble = random.nextDouble();
originalValues.add(randomDouble);
expectedParsedValues.add(randomDouble);
break;
case 5:
Float randomFloat = random.nextFloat();
originalValues.add(randomFloat);
if (xContentType == XContentType.SMILE) {
//with SMILE we get back a double (this will change in Jackson 2.9 where it will return a Float)
expectedParsedValues.add(randomFloat.doubleValue());
} else {
//with JSON AND YAML we get back a double, but with float precision.
expectedParsedValues.add(Double.parseDouble(randomFloat.toString()));
}
break;
case 6:
boolean randomBoolean = random.nextBoolean();
originalValues.add(randomBoolean);
expectedParsedValues.add(randomBoolean);
break;
case 7:
String randomString = random.nextBoolean() ? RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10) :
randomUnicodeOfLengthBetween(random, 3, 10);
originalValues.add(randomString);
expectedParsedValues.add(randomString);
break;
case 8:
byte[] randomBytes = RandomStrings.randomUnicodeOfLengthBetween(random, 10, 50).getBytes(StandardCharsets.UTF_8);
BytesArray randomBytesArray = new BytesArray(randomBytes);
originalValues.add(randomBytesArray);
if (xContentType == XContentType.JSON || xContentType == XContentType.YAML) {
//JSON and YAML write the base64 format
expectedParsedValues.add(Base64.getEncoder().encodeToString(randomBytes));
} else {
//SMILE and CBOR write the original bytes as they support binary format
expectedParsedValues.add(randomBytesArray);
}
break;
default:
throw new UnsupportedOperationException();
}
}
return Tuple.tuple(originalValues, expectedParsedValues);
}
Example 14
| Source File: Elasticsearch6DynamicSink.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public SinkFunctionProvider getSinkRuntimeProvider(Context context) {
return () -> {
SerializationSchema<RowData> format = this.format.createRuntimeEncoder(context, schema.toRowDataType());
final RowElasticsearchSinkFunction upsertFunction =
new RowElasticsearchSinkFunction(
IndexGeneratorFactory.createIndexGenerator(config.getIndex(), schema),
config.getDocumentType(),
format,
XContentType.JSON,
REQUEST_FACTORY,
KeyExtractor.createKeyExtractor(schema, config.getKeyDelimiter())
);
final ElasticsearchSink.Builder<RowData> builder = builderProvider.createBuilder(
config.getHosts(),
upsertFunction);
builder.setFailureHandler(config.getFailureHandler());
builder.setBulkFlushMaxActions(config.getBulkFlushMaxActions());
builder.setBulkFlushMaxSizeMb((int) (config.getBulkFlushMaxByteSize() >> 20));
builder.setBulkFlushInterval(config.getBulkFlushInterval());
builder.setBulkFlushBackoff(config.isBulkFlushBackoffEnabled());
config.getBulkFlushBackoffType().ifPresent(builder::setBulkFlushBackoffType);
config.getBulkFlushBackoffRetries().ifPresent(builder::setBulkFlushBackoffRetries);
config.getBulkFlushBackoffDelay().ifPresent(builder::setBulkFlushBackoffDelay);
// we must overwrite the default factory which is defined with a lambda because of a bug
// in shading lambda serialization shading see FLINK-18006
builder.setRestClientFactory(new DefaultRestClientFactory(config.getPathPrefix().orElse(null)));
final ElasticsearchSink<RowData> sink = builder.build();
if (config.isDisableFlushOnCheckpoint()) {
sink.disableFlushOnCheckpoint();
}
return sink;
};
}
Example 15
| Source File: JsonXContentGenerator.java From crate with Apache License 2.0 | 4 votes |
|
@Override
public XContentType contentType() {
return XContentType.JSON;
}
Example 16
| Source File: TransportShardUpsertAction.java From crate with Apache License 2.0 | 4 votes |
|
@Override
protected WriteReplicaResult<ShardUpsertRequest> processRequestItemsOnReplica(IndexShard indexShard, ShardUpsertRequest request) throws IOException {
Translog.Location location = null;
for (ShardUpsertRequest.Item item : request.items()) {
if (item.source() == null) {
if (logger.isTraceEnabled()) {
logger.trace("[{} (R)] Document with id {}, has no source, primary operation must have failed",
indexShard.shardId(), item.id());
}
continue;
}
SourceToParse sourceToParse = new SourceToParse(
indexShard.shardId().getIndexName(),
item.id(),
item.source(),
XContentType.JSON
);
Engine.IndexResult indexResult = indexShard.applyIndexOperationOnReplica(
item.seqNo(),
item.version(),
Translog.UNSET_AUTO_GENERATED_TIMESTAMP,
false,
sourceToParse
);
if (indexResult.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) {
// Even though the primary waits on all nodes to ack the mapping changes to the master
// (see MappingUpdatedAction.updateMappingOnMaster) we still need to protect against missing mappings
// and wait for them. The reason is concurrent requests. Request r1 which has new field f triggers a
// mapping update. Assume that that update is first applied on the primary, and only later on the replica
// (it’s happening concurrently). Request r2, which now arrives on the primary and which also has the new
// field f might see the updated mapping (on the primary), and will therefore proceed to be replicated
// to the replica. When it arrives on the replica, there’s no guarantee that the replica has already
// applied the new mapping, so there is no other option than to wait.
throw new TransportReplicationAction.RetryOnReplicaException(indexShard.shardId(),
"Mappings are not available on the replica yet, triggered update: " + indexResult.getRequiredMappingUpdate());
}
location = indexResult.getTranslogLocation();
}
return new WriteReplicaResult<>(request, location, null, indexShard, logger);
}
Example 17
| Source File: JsonXContentParser.java From Elasticsearch with Apache License 2.0 | 4 votes |
|
@Override
public XContentType contentType() {
return XContentType.JSON;
}
Example 18
| Source File: JsonSettingsLoader.java From Elasticsearch with Apache License 2.0 | 4 votes |
|
@Override
public XContentType contentType() {
return XContentType.JSON;
}
Example 19
| Source File: JsonXContentParser.java From crate with Apache License 2.0 | 4 votes |
|
@Override
public XContentType contentType() {
return XContentType.JSON;
}
Example 20
| Source File: Elasticsearch6UpsertTableSinkFactoryTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Test
public void testBuilder() {
final TableSchema schema = createTestSchema();
final TestElasticsearch6UpsertTableSink testSink = new TestElasticsearch6UpsertTableSink(
false,
schema,
Collections.singletonList(new Host(HOSTNAME, PORT, SCHEMA)),
INDEX,
DOC_TYPE,
KEY_DELIMITER,
KEY_NULL_LITERAL,
new JsonRowSerializationSchema(schema.toRowType()),
XContentType.JSON,
new DummyFailureHandler(),
createTestSinkOptions());
final DataStreamMock dataStreamMock = new DataStreamMock(
new StreamExecutionEnvironmentMock(),
Types.TUPLE(Types.BOOLEAN, schema.toRowType()));
testSink.emitDataStream(dataStreamMock);
final ElasticsearchSink.Builder<Tuple2<Boolean, Row>> expectedBuilder = new ElasticsearchSink.Builder<>(
Collections.singletonList(new HttpHost(HOSTNAME, PORT, SCHEMA)),
new ElasticsearchUpsertSinkFunction(
INDEX,
DOC_TYPE,
KEY_DELIMITER,
KEY_NULL_LITERAL,
new JsonRowSerializationSchema(schema.toRowType()),
XContentType.JSON,
Elasticsearch6UpsertTableSink.UPDATE_REQUEST_FACTORY,
new int[0]));
expectedBuilder.setFailureHandler(new DummyFailureHandler());
expectedBuilder.setBulkFlushBackoff(true);
expectedBuilder.setBulkFlushBackoffType(ElasticsearchSinkBase.FlushBackoffType.EXPONENTIAL);
expectedBuilder.setBulkFlushBackoffDelay(123);
expectedBuilder.setBulkFlushBackoffRetries(3);
expectedBuilder.setBulkFlushInterval(100);
expectedBuilder.setBulkFlushMaxActions(1000);
expectedBuilder.setBulkFlushMaxSizeMb(1);
expectedBuilder.setRestClientFactory(new DefaultRestClientFactory(100, "/myapp"));
assertEquals(expectedBuilder, testSink.builder);
}
