Java Code Examples for org.elasticsearch.common.xcontent.XContentHelper#convertToMap()

protected Map<String, Object> parseQuery(BytesReference source) {
  // nothing to parse...
  if (source == null || source.length() == 0) {
    return null;
  }

  try {
    Tuple<XContentType, Map<String, Object>> parsedSource = XContentHelper.convertToMap(source, false);
    return parsedSource.v2();
  }
  catch (Throwable e) {
    String sSource = "_na_";
    try {
      sSource = XContentHelper.convertToJson(source, false);
    }
    catch (Throwable e1) { /* ignore  */ }
    throw new ElasticsearchParseException("Failed to parse source [" + sSource + "]", e);
  }
}
 

protected Tuple<XContentType, Map<String, Object>> parseSource(BytesReference source) {
  // nothing to parse...
  if (source == null || source.length() == 0) {
    return null;
  }

  try {
    Tuple<XContentType, Map<String, Object>> parsedSource = XContentHelper.convertToMap(source, false);
    logger.debug("{}: Parsed source: {}", Thread.currentThread().getName(), parsedSource);
    return parsedSource;
  }
  catch (Throwable e) {
      String sSource = "_na_";
      try {
          sSource = XContentHelper.convertToJson(source, false);
      }
      catch (Throwable e1) { /* ignore  */ }
      throw new ElasticsearchParseException("Failed to parse source [" + sSource + "]", e);
  }
}
 

@Override
public void binaryField(final FieldInfo fieldInfo, final byte[] value) throws IOException {

    if (fieldInfo.name.equals("_source")) {
        final BytesReference bytesRef = new BytesArray(value);
        final Tuple<XContentType, Map<String, Object>> bytesRefTuple = XContentHelper.convertToMap(bytesRef, false, XContentType.JSON);
        Map<String, Object> filteredSource = bytesRefTuple.v2();
        MapUtils.deepTraverseMap(filteredSource, HASH_CB);
        final XContentBuilder xBuilder = XContentBuilder.builder(bytesRefTuple.v1().xContent()).map(filteredSource);
        delegate.binaryField(fieldInfo, BytesReference.toBytes(BytesReference.bytes(xBuilder)));
    } else {
        delegate.binaryField(fieldInfo, value);
    }
}
 

@SuppressWarnings("unchecked")
protected void assertThatMessageEquals(String message) throws Exception {
    Response response = (Response) testContext.get(RESPONSE);
    Map<String, Object> source = XContentHelper.convertToMap(JsonXContent.jsonXContent, response.body().byteStream(),false);
    Map<String, Object> doc =  (Map<String, Object>) source.get("_source");
    assertEquals("Exp. Message content to be equal", message, doc.get("msg"));
}
 

@SuppressWarnings("unchecked")
private void assertThatDefaultIndexPatternIs(String indexPattern) throws Exception {
    Response response = (Response) testContext.get(RESPONSE);
    Map<String, Object> source = XContentHelper.convertToMap(JsonXContent.jsonXContent, response.body().byteStream(),false);
    Map<String, Object> doc =  (Map<String, Object>) source.get("_source");
    assertEquals("Exp. default index to be equal", indexPattern, doc.get("defaultIndex"));
}
 

@SuppressWarnings("unchecked")
public void read(InputStream input) throws IOException {
    Map<String, Object> map = XContentHelper.convertToMap(JsonXContent.jsonXContent, input, true);
    freq = (Map<String, Integer>) map.get("freq");
    name = (String) map.get("name");
    nWords = (List<Integer>) map.get("n_words");
}
 

/**
 * Allows to partially validate an alias, without knowing which index it'll get applied to.
 * Useful with index templates containing aliases. Checks also that it is possible to parse
 * the alias filter via {@link org.elasticsearch.common.xcontent.XContentParser},
 * without validating it as a filter though.
 * @throws IllegalArgumentException if the alias is not valid
 */
public void validateAliasStandalone(Alias alias) {
    validateAliasStandalone(alias.name(), alias.indexRouting());
    if (Strings.hasLength(alias.filter())) {
        try {
            XContentHelper.convertToMap(XContentFactory.xContent(alias.filter()), alias.filter(), false);
        } catch (Exception e) {
            throw new IllegalArgumentException("failed to parse filter for alias [" + alias.name() + "]", e);
        }
    }
}
 

@Test
public void testStrictPartitionedTableInsert() throws Exception {
    execute("create table numbers (" +
            "  num int, " +
            "  odd boolean," +
            "  prime boolean" +
            ") partitioned by (odd) with (column_policy='strict', number_of_replicas=0)");
    ensureYellow();

    GetIndexTemplatesResponse response = client().admin().indices()
        .prepareGetTemplates(PartitionName.templateName(sqlExecutor.getCurrentSchema(), "numbers"))
        .execute().actionGet();
    assertThat(response.getIndexTemplates().size(), is(1));
    IndexTemplateMetaData template = response.getIndexTemplates().get(0);
    CompressedXContent mappingStr = template.mappings().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(mappingStr, is(notNullValue()));
    Tuple<XContentType, Map<String, Object>> typeAndMap =
        XContentHelper.convertToMap(mappingStr.compressedReference(), false, XContentType.JSON);
    @SuppressWarnings("unchecked")
    Map<String, Object> mapping = (Map<String, Object>) typeAndMap.v2().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(decodeMappingValue(mapping.get("dynamic")), is(ColumnPolicy.STRICT));

    execute("insert into numbers (num, odd, prime) values (?, ?, ?)",
        new Object[]{6, true, false});
    execute("refresh table numbers");

    Map<String, Object> sourceMap = getSourceMap(
        new PartitionName(new RelationName("doc", "numbers"), Arrays.asList("true")).asIndexName());
    assertThat(decodeMappingValue(sourceMap.get("dynamic")), is(ColumnPolicy.STRICT));

    expectedException.expect(SQLActionException.class);
    expectedException.expectMessage("Column perfect unknown");
    execute("insert into numbers (num, odd, prime, perfect) values (?, ?, ?, ?)",
        new Object[]{28, true, false, true});
}
 

@Test
public void testStrictPartitionedTableUpdate() throws Exception {
    execute("create table numbers (" +
            "  num int, " +
            "  odd boolean," +
            "  prime boolean" +
            ") partitioned by (odd) with (column_policy='strict', number_of_replicas=0)");
    ensureYellow();

    GetIndexTemplatesResponse response = client().admin().indices()
        .prepareGetTemplates(PartitionName.templateName(sqlExecutor.getCurrentSchema(), "numbers"))
        .execute().actionGet();
    assertThat(response.getIndexTemplates().size(), is(1));
    IndexTemplateMetaData template = response.getIndexTemplates().get(0);
    CompressedXContent mappingStr = template.mappings().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(mappingStr, is(notNullValue()));
    Tuple<XContentType, Map<String, Object>> typeAndMap = XContentHelper.convertToMap(mappingStr.compressedReference(), false);
    @SuppressWarnings("unchecked")
    Map<String, Object> mapping = (Map<String, Object>) typeAndMap.v2().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(decodeMappingValue(mapping.get("dynamic")), is(ColumnPolicy.STRICT));

    execute("insert into numbers (num, odd, prime) values (?, ?, ?)",
        new Object[]{6, true, false});
    execute("refresh table numbers");

    Map<String, Object> sourceMap = getSourceMap(
        new PartitionName(new RelationName("doc", "numbers"), Arrays.asList("true")).asIndexName());
    assertThat(decodeMappingValue(sourceMap.get("dynamic")), is(ColumnPolicy.STRICT));

    expectedException.expect(SQLActionException.class);
    expectedException.expectMessage("Column perfect unknown");
    execute("update numbers set num=?, perfect=? where num=6",
        new Object[]{28, true});
}
 

/**
 * Prepares an update request by converting it into an index request.
 * <p/>
 * TODO: detect a NOOP and return an update response if true
 */
@SuppressWarnings("unchecked")
private SourceAndVersion prepareUpdate(DocTableInfo tableInfo,
                                       ShardUpsertRequest request,
                                       ShardUpsertRequest.Item item,
                                       IndexShard indexShard) throws ElasticsearchException {
    final GetResult getResult = indexShard.getService().get(request.type(), item.id(),
            new String[]{RoutingFieldMapper.NAME, ParentFieldMapper.NAME, TTLFieldMapper.NAME},
            true, Versions.MATCH_ANY, VersionType.INTERNAL, FetchSourceContext.FETCH_SOURCE, false);

    if (!getResult.isExists()) {
        throw new DocumentMissingException(new ShardId(request.index(), request.shardId().id()), request.type(), item.id());
    }

    if (getResult.internalSourceRef() == null) {
        // no source, we can't do nothing, through a failure...
        throw new DocumentSourceMissingException(new ShardId(request.index(), request.shardId().id()), request.type(), item.id());
    }

    if (item.version() != Versions.MATCH_ANY && item.version() != getResult.getVersion()) {
        throw new VersionConflictEngineException(
                indexShard.shardId(), Constants.DEFAULT_MAPPING_TYPE, item.id(), getResult.getVersion(), item.version());
    }

    Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(getResult.internalSourceRef(), true);
    final Map<String, Object> updatedSourceAsMap;
    final XContentType updateSourceContentType = sourceAndContent.v1();

    updatedSourceAsMap = sourceAndContent.v2();

    SymbolToFieldExtractorContext ctx = new SymbolToFieldExtractorContext(functions, item.insertValues());

    Map<String, Object> pathsToUpdate = new LinkedHashMap<>();
    Map<String, Object> updatedGeneratedColumns = new LinkedHashMap<>();
    for (int i = 0; i < request.updateColumns().length; i++) {
        /**
         * NOTE: mapping isn't applied. So if an Insert was done using the ES Rest Endpoint
         * the data might be returned in the wrong format (date as string instead of long)
         */
        String columnPath = request.updateColumns()[i];
        Object value = SYMBOL_TO_FIELD_EXTRACTOR.convert(item.updateAssignments()[i], ctx).apply(getResult);
        ReferenceInfo referenceInfo = tableInfo.getReferenceInfo(ColumnIdent.fromPath(columnPath));
        if (referenceInfo instanceof GeneratedReferenceInfo) {
            updatedGeneratedColumns.put(columnPath, value);

        } else {
            pathsToUpdate.put(columnPath, value);
        }
    }

    processGeneratedColumns(tableInfo, pathsToUpdate, updatedGeneratedColumns, request.validateGeneratedColumns(), getResult);

    updateSourceByPaths(updatedSourceAsMap, pathsToUpdate);

    try {
        XContentBuilder builder = XContentFactory.contentBuilder(updateSourceContentType);
        builder.map(updatedSourceAsMap);
        return new SourceAndVersion(builder.bytes(), getResult.getVersion());
    } catch (IOException e) {
        throw new ElasticsearchGenerationException("Failed to generate [" + updatedSourceAsMap + "]", e);
    }
}
 

public static Tuple<XContentType, Map<String, Object>> sourceAsMapAndType(BytesReference source) throws ElasticsearchParseException {
    return XContentHelper.convertToMap(source, false);
}
 

public Map<String, Object> mappingDeltaAsMap() {
    if (mappingDelta == null) {
        return Collections.emptyMap();
    }
    return XContentHelper.convertToMap(JsonXContent.JSON_XCONTENT, mappingDelta, false);
}
 

@Test
public void testDynamicPartitionedTable() throws Exception {
    execute("create table numbers (" +
            "  num int, " +
            "  odd boolean," +
            "  prime boolean" +
            ") partitioned by (odd) with (column_policy='dynamic', number_of_replicas=0)");
    ensureYellow();

    GetIndexTemplatesResponse templateResponse = client().admin().indices()
        .prepareGetTemplates(PartitionName.templateName(sqlExecutor.getCurrentSchema(), "numbers"))
        .execute().actionGet();
    assertThat(templateResponse.getIndexTemplates().size(), is(1));
    IndexTemplateMetaData template = templateResponse.getIndexTemplates().get(0);
    CompressedXContent mappingStr = template.mappings().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(mappingStr, is(notNullValue()));
    Tuple<XContentType, Map<String, Object>> typeAndMap = XContentHelper.convertToMap(mappingStr.compressedReference(), false);
    @SuppressWarnings("unchecked")
    Map<String, Object> mapping = (Map<String, Object>) typeAndMap.v2().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(String.valueOf(mapping.get("dynamic")), is("true"));

    execute("insert into numbers (num, odd, prime) values (?, ?, ?)",
        new Object[]{6, true, false});
    execute("refresh table numbers");

    Map<String, Object> sourceMap = getSourceMap(
        new PartitionName(new RelationName("doc", "numbers"), Collections.singletonList("true")).asIndexName());
    assertThat(String.valueOf(sourceMap.get("dynamic")), is("true"));

    execute("insert into numbers (num, odd, prime, perfect) values (?, ?, ?, ?)",
        new Object[]{28, true, false, true});
    ensureYellow();
    execute("refresh table numbers");

    waitForMappingUpdateOnAll("numbers", "perfect");
    execute("select * from numbers order by num");
    assertThat(response.rowCount(), is(2L));
    assertThat(response.cols(), arrayContaining("num", "odd", "prime", "perfect"));
    assertThat(TestingHelpers.printedTable(response.rows()), is(
        "6| true| false| NULL\n" +
        "28| true| false| true\n"));

    execute("update numbers set prime=true, changed='2014-10-23T10:20', author='troll' where num=28");
    assertThat(response.rowCount(), is(1L));

    waitForMappingUpdateOnAll("numbers", "changed");
}
 

@Test
public void testAlterColumnPolicyOnPartitionedTableWithExistingPartitions() throws Exception {
    execute("create table dynamic_table (" +
            "  id integer, " +
            "  score double" +
            ") partitioned by (score) with (number_of_replicas=0, column_policy='strict')");
    ensureYellow();
    // create at least 2 partitions to test real multi partition logic (1 partition would behave similar to 1 normal table)
    execute("insert into dynamic_table (id, score) values (1, 10)");
    execute("insert into dynamic_table (id, score) values (1, 20)");
    execute("refresh table dynamic_table");
    ensureYellow();
    execute("alter table dynamic_table set (column_policy = 'dynamic')");
    waitNoPendingTasksOnAll();
    // After changing the column_policy it's possible to add new columns to existing and new
    // partitions
    execute("insert into dynamic_table (id, score, comment) values (2, 10, 'this is a new column')");
    execute("insert into dynamic_table (id, score, new_comment) values (2, 5, 'this is a new column on a new partition')");
    execute("refresh table dynamic_table");
    ensureYellow();
    GetIndexTemplatesResponse response = client().admin().indices()
        .prepareGetTemplates(PartitionName.templateName(sqlExecutor.getCurrentSchema(), "dynamic_table"))
        .execute().actionGet();
    assertThat(response.getIndexTemplates().size(), is(1));
    IndexTemplateMetaData template = response.getIndexTemplates().get(0);
    CompressedXContent mappingStr = template.mappings().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(mappingStr, is(notNullValue()));
    Tuple<XContentType, Map<String, Object>> typeAndMap =
        XContentHelper.convertToMap(mappingStr.compressedReference(), false, XContentType.JSON);
    @SuppressWarnings("unchecked")
    Map<String, Object> mapping = (Map<String, Object>) typeAndMap.v2().get(Constants.DEFAULT_MAPPING_TYPE);
    assertThat(decodeMappingValue(mapping.get("dynamic")), is(ColumnPolicy.DYNAMIC));

    execute("insert into dynamic_table (id, score, new_col) values (?, ?, ?)",
        new Object[]{6, 3, "hello"});
    execute("refresh table dynamic_table");
    ensureYellow();

    Map<String, Object> sourceMap = getSourceMap(
        new PartitionName(new RelationName("doc", "dynamic_table"), Arrays.asList("10.0")).asIndexName());
    assertThat(decodeMappingValue(sourceMap.get("dynamic")), is(ColumnPolicy.DYNAMIC));

    sourceMap = getSourceMap(new PartitionName(
        new RelationName("doc", "dynamic_table"), Arrays.asList("5.0")).asIndexName());
    assertThat(decodeMappingValue(sourceMap.get("dynamic")), is(ColumnPolicy.DYNAMIC));

    sourceMap = getSourceMap(new PartitionName(
        new RelationName("doc", "dynamic_table"), Arrays.asList("3.0")).asIndexName());
    assertThat(decodeMappingValue(sourceMap.get("dynamic")), is(ColumnPolicy.DYNAMIC));
}