org.elasticsearch.common.collect.Maps Java Examples

The following examples show how to use org.elasticsearch.common.collect.Maps. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ImageMapper.java    From elasticsearch-image with Apache License 2.0 5 votes vote down vote up 
@Override
public ImageMapper build(BuilderContext context) {
    Map<String, Mapper> featureMappers = Maps.newHashMap();
    Map<String, Mapper> hashMappers = Maps.newHashMap();
    Map<String, Mapper> metadataMappers = Maps.newHashMap();

    context.path().add(name);
    // add feature and hash mappers
    for (FeatureEnum featureEnum : features.keySet()) {
        Map<String, Object> featureMap = features.get(featureEnum);
        String featureName = featureEnum.name();

        // add feature mapper
        featureMappers.put(featureName, binaryField(featureName).store(true).includeInAll(false).index(false).build(context));


        // add hash mapper if hash is required
        if (featureMap.containsKey(HASH)){
            List<String> hashes = (List<String>) featureMap.get(HASH);
            for (String h : hashes) {
                String hashFieldName = featureName + "." + HASH + "." + h;
                hashMappers.put(hashFieldName, stringField(hashFieldName).store(true).includeInAll(false).index(true).build(context));
            }
        }
    }

    // add metadata mappers
    context.path().add(METADATA);
    for (Map.Entry<String, Mapper.Builder> entry : metadataBuilders.entrySet()){
        String metadataName = entry.getKey();
        Mapper.Builder metadataBuilder = entry.getValue();
        metadataMappers.put(metadataName, metadataBuilder.build(context));
    }
    context.path().remove();  // remove METADATA
    context.path().remove();  // remove name

    return new ImageMapper(name, threadPool, context.indexSettings(), features, featureMappers, hashMappers, metadataMappers);
}
Example #2
Source File: ImageIntegrationTests.java    From elasticsearch-image with Apache License 2.0 5 votes vote down vote up 
private byte[] getRandomImage() throws IOException, ImageWriteException {
    int width = randomIntBetween(100, 1000);
    int height = randomIntBetween(100, 1000);
    BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
    for (int j = 0; j < width; j ++) {
        for (int k = 0; k < height; k ++) {
            image.setRGB(j, k, randomInt(512));
        }
    }
    ImageFormat format = ImageFormat.IMAGE_FORMAT_TIFF;
    byte[] bytes = Sanselan.writeImageToBytes(image, format, Maps.newHashMap());
    return bytes;
}
Example #3
Source File: AbstractElasticSearchSinkTest.java    From mt-flume with Apache License 2.0 5 votes vote down vote up 
void initDefaults() {
  parameters = Maps.newHashMap();
  parameters.put(INDEX_NAME, DEFAULT_INDEX_NAME);
  parameters.put(INDEX_TYPE, DEFAULT_INDEX_TYPE);
  parameters.put(CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
  parameters.put(BATCH_SIZE, "1");
  parameters.put(TTL, "5");

  timestampedIndexName = DEFAULT_INDEX_NAME + '-'
      + ElasticSearchIndexRequestBuilderFactory.df.format(FIXED_TIME_MILLIS);
}
Example #4
Source File: TestElasticSearchDynamicSerializer.java    From mt-flume with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRoundTrip() throws Exception {
  ElasticSearchDynamicSerializer fixture = new ElasticSearchDynamicSerializer();
  Context context = new Context();
  fixture.configure(context);

  String message = "test body";
  Map<String, String> headers = Maps.newHashMap();
  headers.put("headerNameOne", "headerValueOne");
  headers.put("headerNameTwo", "headerValueTwo");
  headers.put("headerNameThree", "headerValueThree");
  Event event = EventBuilder.withBody(message.getBytes(charset));
  event.setHeaders(headers);

  XContentBuilder expected = jsonBuilder().startObject();
  expected.field("body", new String(message.getBytes(), charset));
  for (String headerName : headers.keySet()) {
    expected.field(headerName, new String(headers.get(headerName).getBytes(),
        charset));
  }
  expected.endObject();

  XContentBuilder actual = fixture.getContentBuilder(event);

  assertEquals(new String(expected.bytes().array()), new String(actual
      .bytes().array()));

}
Example #5
Source File: OntologyMapper.java    From BioSolr with Apache License 2.0 5 votes vote down vote up 
@Override
public OntologyMapper build(BuilderContext context) {
	ContentPath.Type origPathType = context.path().pathType();
	context.path().pathType(pathType);

	Map<String, FieldMapper<String>> fieldMappers = Maps.newHashMap();

	context.path().add(name);

	if (propertyBuilders != null) {
		for (String property : propertyBuilders.keySet()) {
			StringFieldMapper sfm = propertyBuilders.get(property).build(context);
			fieldMappers.put(sfm.names().indexName(), sfm);
		}
	}

	// Initialise field mappers for the pre-defined fields
	for (FieldMappings mapping : ontologySettings.getFieldMappings()) {
		if (!fieldMappers.containsKey(context.path().fullPathAsText(mapping.getFieldName()))) {
			StringFieldMapper mapper = MapperBuilders.stringField(mapping.getFieldName())
					.store(true)
					.index(true)
					.tokenized(!mapping.isUriField())
					.build(context);
			fieldMappers.put(mapper.names().indexName(), mapper);
		}
	}

	context.path().remove(); // remove name
	context.path().pathType(origPathType);

	return new OntologyMapper(buildNames(context), fieldType, docValues, indexAnalyzer, searchAnalyzer,
			postingsProvider, docValuesProvider,
			similarity, fieldDataSettings, context.indexSettings(),
			new MultiFields.Builder().build(this, context),
			ontologySettings, fieldMappers, threadPool);
}
Example #6
Source File: OntologyMapper.java    From BioSolr with Apache License 2.0 5 votes vote down vote up 
@Override
public OntologyMapper build(BuilderContext context) {
	ContentPath.Type origPathType = context.path().pathType();
	context.path().pathType(pathType);

	Map<String, FieldMapper<String>> fieldMappers = Maps.newHashMap();

	context.path().add(name);

	if (propertyBuilders != null) {
		for (String property : propertyBuilders.keySet()) {
			StringFieldMapper sfm = propertyBuilders.get(property).build(context);
			fieldMappers.put(sfm.names().indexName(), sfm);
		}
	}

	// Initialise field mappers for the pre-defined fields
	for (FieldMappings mapping : ontologySettings.getFieldMappings()) {
		if (!fieldMappers.containsKey(context.path().fullPathAsText(mapping.getFieldName()))) {
			StringFieldMapper mapper = MapperBuilders.stringField(mapping.getFieldName())
					.store(true)
					.index(true)
					.tokenized(!mapping.isUriField())
					.build(context);
			fieldMappers.put(mapper.names().indexName(), mapper);
		}
	}

	context.path().remove(); // remove name
	context.path().pathType(origPathType);

	return new OntologyMapper(buildNames(context), fieldType, docValues, indexAnalyzer, searchAnalyzer,
			postingsProvider, docValuesProvider,
			similarity, fieldDataSettings, context.indexSettings(),
			new MultiFields.Builder().build(this, context),
			ontologySettings, fieldMappers, threadPool);
}
Example #7
Source File: OntologyMapper.java    From BioSolr with Apache License 2.0 5 votes vote down vote up 
@Override
public OntologyMapper build(BuilderContext context) {
	ContentPath.Type origPathType = context.path().pathType();
	context.path().pathType(pathType);

	Map<String, FieldMapper<String>> fieldMappers = Maps.newHashMap();

	context.path().add(name);

	if (propertyBuilders != null) {
		for (String property : propertyBuilders.keySet()) {
			StringFieldMapper sfm = propertyBuilders.get(property).build(context);
			fieldMappers.put(sfm.names().indexName(), sfm);
		}
	}

	// Initialise field mappers for the pre-defined fields
	for (FieldMappings mapping : ontologySettings.getFieldMappings()) {
		if (!fieldMappers.containsKey(context.path().fullPathAsText(mapping.getFieldName()))) {
			StringFieldMapper mapper = MapperBuilders.stringField(mapping.getFieldName())
					.store(true)
					.index(true)
					.tokenized(!mapping.isUriField())
					.build(context);
			fieldMappers.put(mapper.names().indexName(), mapper);
		}
	}

	context.path().remove(); // remove name
	context.path().pathType(origPathType);

	return new OntologyMapper(buildNames(context), fieldType, docValues, indexAnalyzer, searchAnalyzer,
			postingsProvider, docValuesProvider,
			similarity, fieldDataSettings, context.indexSettings(),
			new MultiFields.Builder().build(this, context),
			ontologySettings, fieldMappers, threadPool);
}
Example #8
Source File: AbstractElasticSearchSinkTest.java    From ingestion with Apache License 2.0 5 votes vote down vote up 
void initDefaults() {
  parameters = Maps.newHashMap();
  parameters.put(INDEX_NAME, DEFAULT_INDEX_NAME);
  parameters.put(INDEX_TYPE, DEFAULT_INDEX_TYPE);
  parameters.put(CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
  parameters.put(BATCH_SIZE, "1");
  parameters.put(TTL, "5");

  timestampedIndexName = DEFAULT_INDEX_NAME + '-'
      + ElasticSearchIndexRequestBuilderFactory.df.format(FIXED_TIME_MILLIS);
}
Example #9
Source File: TestElasticSearchDynamicSerializer.java    From ingestion with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRoundTrip() throws Exception {
  ElasticSearchDynamicSerializer fixture = new ElasticSearchDynamicSerializer();
  Context context = new Context();
  fixture.configure(context);

  String message = "test body";
  Map<String, String> headers = Maps.newHashMap();
  headers.put("headerNameOne", "headerValueOne");
  headers.put("headerNameTwo", "headerValueTwo");
  headers.put("headerNameThree", "headerValueThree");
  Event event = EventBuilder.withBody(message.getBytes(charset));
  event.setHeaders(headers);

  XContentBuilder expected = jsonBuilder().startObject();
  expected.field("body", new String(message.getBytes(), charset));
  for (String headerName : headers.keySet()) {
    expected.field(headerName, new String(headers.get(headerName).getBytes(),
        charset));
  }
  expected.endObject();

  XContentBuilder actual = fixture.getContentBuilder(event);

  assertEquals(new String(expected.bytes().array()), new String(actual
      .bytes().array()));

}
Example #10
Source File: ElasticSearchSerializerWithMappingTest.java    From ingestion with Apache License 2.0 5 votes vote down vote up 
private Event createExampleEvent(long timestamp) {
	String message = "test body";
    Map<String, String> headers = Maps.newHashMap();
    headers.put(TIMESTAMP_HEADER, String.valueOf(timestamp));
    Event event = EventBuilder.withBody(message.getBytes(charset));
    event.setHeaders(headers);
    return event;
}
Example #11
Source File: ElasticSearchLogStashEventSerializer.java    From mt-flume with Apache License 2.0 4 votes vote down vote up 
private void appendHeaders(XContentBuilder builder, Event event)
    throws IOException {
  Map<String, String> headers = Maps.newHashMap(event.getHeaders());

  String timestamp = headers.get("timestamp");
  if (!StringUtils.isBlank(timestamp)
      && StringUtils.isBlank(headers.get("@timestamp"))) {
    long timestampMs = Long.parseLong(timestamp);
    builder.field("@timestamp", new Date(timestampMs));
  }

  String source = headers.get("source");
  if (!StringUtils.isBlank(source)
      && StringUtils.isBlank(headers.get("@source"))) {
    ContentBuilderUtil.appendField(builder, "@source",
        source.getBytes(charset));
  }

  String type = headers.get("type");
  if (!StringUtils.isBlank(type)
      && StringUtils.isBlank(headers.get("@type"))) {
    ContentBuilderUtil.appendField(builder, "@type", type.getBytes(charset));
  }

  String host = headers.get("host");
  if (!StringUtils.isBlank(host)
      && StringUtils.isBlank(headers.get("@source_host"))) {
    ContentBuilderUtil.appendField(builder, "@source_host",
        host.getBytes(charset));
  }

  String srcPath = headers.get("src_path");
  if (!StringUtils.isBlank(srcPath)
      && StringUtils.isBlank(headers.get("@source_path"))) {
    ContentBuilderUtil.appendField(builder, "@source_path",
        srcPath.getBytes(charset));
  }

  builder.startObject("@fields");
  for (String key : headers.keySet()) {
    byte[] val = headers.get(key).getBytes(charset);
    ContentBuilderUtil.appendField(builder, key, val);
  }
  builder.endObject();
}
Example #12
Source File: TestElasticSearchLogStashEventSerializer.java    From mt-flume with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRoundTrip() throws Exception {
  ElasticSearchLogStashEventSerializer fixture = new ElasticSearchLogStashEventSerializer();
  Context context = new Context();
  fixture.configure(context);

  String message = "test body";
  Map<String, String> headers = Maps.newHashMap();
  long timestamp = System.currentTimeMillis();
  headers.put("timestamp", String.valueOf(timestamp));
  headers.put("source", "flume_tail_src");
  headers.put("host", "test@localhost");
  headers.put("src_path", "/tmp/test");
  headers.put("headerNameOne", "headerValueOne");
  headers.put("headerNameTwo", "headerValueTwo");
  headers.put("type", "sometype");
  Event event = EventBuilder.withBody(message.getBytes(charset));
  event.setHeaders(headers);

  XContentBuilder expected = jsonBuilder().startObject();
  expected.field("@message", new String(message.getBytes(), charset));
  expected.field("@timestamp", new Date(timestamp));
  expected.field("@source", "flume_tail_src");
  expected.field("@type", "sometype");
  expected.field("@source_host", "test@localhost");
  expected.field("@source_path", "/tmp/test");
  expected.startObject("@fields");
  expected.field("timestamp", String.valueOf(timestamp));
  expected.field("src_path", "/tmp/test");
  expected.field("host", "test@localhost");
  expected.field("headerNameTwo", "headerValueTwo");
  expected.field("source", "flume_tail_src");
  expected.field("headerNameOne", "headerValueOne");
  expected.field("type", "sometype");
  expected.endObject();

  expected.endObject();

  XContentBuilder actual = fixture.getContentBuilder(event);
  assertEquals(new String(expected.bytes().array()), new String(actual
      .bytes().array()));
}
Example #13
Source File: TestElasticSearchLogStashEventSerializer.java    From mt-flume with Apache License 2.0 4 votes vote down vote up 
@Test
public void shouldHandleInvalidJSONDuringComplexParsing() throws Exception {
  ElasticSearchLogStashEventSerializer fixture = new ElasticSearchLogStashEventSerializer();
  Context context = new Context();
  fixture.configure(context);

  String message = "{flume: somethingnotvalid}";
  Map<String, String> headers = Maps.newHashMap();
  long timestamp = System.currentTimeMillis();
  headers.put("timestamp", String.valueOf(timestamp));
  headers.put("source", "flume_tail_src");
  headers.put("host", "test@localhost");
  headers.put("src_path", "/tmp/test");
  headers.put("headerNameOne", "headerValueOne");
  headers.put("headerNameTwo", "headerValueTwo");
  headers.put("type", "sometype");
  Event event = EventBuilder.withBody(message.getBytes(charset));
  event.setHeaders(headers);

  XContentBuilder expected = jsonBuilder().startObject();
  expected.field("@message", new String(message.getBytes(), charset));
  expected.field("@timestamp", new Date(timestamp));
  expected.field("@source", "flume_tail_src");
  expected.field("@type", "sometype");
  expected.field("@source_host", "test@localhost");
  expected.field("@source_path", "/tmp/test");
  expected.startObject("@fields");
  expected.field("timestamp", String.valueOf(timestamp));
  expected.field("src_path", "/tmp/test");
  expected.field("host", "test@localhost");
  expected.field("headerNameTwo", "headerValueTwo");
  expected.field("source", "flume_tail_src");
  expected.field("headerNameOne", "headerValueOne");
  expected.field("type", "sometype");
  expected.endObject();

  expected.endObject();

  XContentBuilder actual = fixture.getContentBuilder(event);
  assertEquals(new String(expected.bytes().array()), new String(actual
      .bytes().array()));
}
Example #14
Source File: ElasticSearchLogStashEventSerializer.java    From ingestion with Apache License 2.0 4 votes vote down vote up 
private void appendHeaders(XContentBuilder builder, Event event)
    throws IOException {
  Map<String, String> headers = Maps.newHashMap(event.getHeaders());

  String timestamp = headers.get("timestamp");
  if (!StringUtils.isBlank(timestamp)
      && StringUtils.isBlank(headers.get("@timestamp"))) {
    long timestampMs = Long.parseLong(timestamp);
    builder.field("@timestamp", new Date(timestampMs));
  }

  String source = headers.get("source");
  if (!StringUtils.isBlank(source)
      && StringUtils.isBlank(headers.get("@source"))) {
    ContentBuilderUtil.appendField(builder, "@source",
        source.getBytes(charset));
  }

  String type = headers.get("type");
  if (!StringUtils.isBlank(type)
      && StringUtils.isBlank(headers.get("@type"))) {
    ContentBuilderUtil.appendField(builder, "@type", type.getBytes(charset));
  }

  String host = headers.get("host");
  if (!StringUtils.isBlank(host)
      && StringUtils.isBlank(headers.get("@source_host"))) {
    ContentBuilderUtil.appendField(builder, "@source_host",
        host.getBytes(charset));
  }

  String srcPath = headers.get("src_path");
  if (!StringUtils.isBlank(srcPath)
      && StringUtils.isBlank(headers.get("@source_path"))) {
    ContentBuilderUtil.appendField(builder, "@source_path",
        srcPath.getBytes(charset));
  }

  builder.startObject("@fields");
  for (String key : headers.keySet()) {
    byte[] val = headers.get(key).getBytes(charset);
    ContentBuilderUtil.appendField(builder, key, val);
  }
  builder.endObject();
}
Example #15
Source File: TestElasticSearchLogStashEventSerializer.java    From ingestion with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRoundTrip() throws Exception {
  ElasticSearchLogStashEventSerializer fixture = new ElasticSearchLogStashEventSerializer();
  Context context = new Context();
  fixture.configure(context);

  String message = "test body";
  Map<String, String> headers = Maps.newHashMap();
  long timestamp = System.currentTimeMillis();
  headers.put("timestamp", String.valueOf(timestamp));
  headers.put("source", "flume_tail_src");
  headers.put("host", "test@localhost");
  headers.put("src_path", "/tmp/test");
  headers.put("headerNameOne", "headerValueOne");
  headers.put("headerNameTwo", "headerValueTwo");
  headers.put("type", "sometype");
  Event event = EventBuilder.withBody(message.getBytes(charset));
  event.setHeaders(headers);

  XContentBuilder expected = jsonBuilder()
      .startObject();
          expected.field("@message", new String(message.getBytes(), charset));
          expected.field("@timestamp", new Date(timestamp));
          expected.field("@source", "flume_tail_src");
          expected.field("@type", "sometype");
          expected.field("@source_host", "test@localhost");
          expected.field("@source_path", "/tmp/test");

          expected.startObject("@fields");
              expected.field("timestamp", String.valueOf(timestamp));
              expected.field("src_path", "/tmp/test");
              expected.field("host", "test@localhost");
              expected.field("headerNameTwo", "headerValueTwo");
              expected.field("source", "flume_tail_src");
              expected.field("headerNameOne", "headerValueOne");
              expected.field("type", "sometype");
          expected.endObject();

      expected.endObject();

  XContentBuilder actual = fixture.getContentBuilder(event);
  
  JsonParser parser = new JsonParser();
  assertEquals(parser.parse(expected.string()),parser.parse(actual.string()));
}
Example #16
Source File: TestElasticSearchLogStashEventSerializer.java    From ingestion with Apache License 2.0 4 votes vote down vote up 
@Test
public void shouldHandleInvalidJSONDuringComplexParsing() throws Exception {
  ElasticSearchLogStashEventSerializer fixture = new ElasticSearchLogStashEventSerializer();
  Context context = new Context();
  fixture.configure(context);

  String message = "{flume: somethingnotvalid}";
  Map<String, String> headers = Maps.newHashMap();
  long timestamp = System.currentTimeMillis();
  headers.put("timestamp", String.valueOf(timestamp));
  headers.put("source", "flume_tail_src");
  headers.put("host", "test@localhost");
  headers.put("src_path", "/tmp/test");
  headers.put("headerNameOne", "headerValueOne");
  headers.put("headerNameTwo", "headerValueTwo");
  headers.put("type", "sometype");
  Event event = EventBuilder.withBody(message.getBytes(charset));
  event.setHeaders(headers);

  XContentBuilder expected = jsonBuilder().
      startObject();
          expected.field("@message", new String(message.getBytes(), charset));
          expected.field("@timestamp", new Date(timestamp));
          expected.field("@source", "flume_tail_src");
          expected.field("@type", "sometype");
          expected.field("@source_host", "test@localhost");
          expected.field("@source_path", "/tmp/test");

          expected.startObject("@fields");
              expected.field("timestamp", String.valueOf(timestamp));
              expected.field("src_path", "/tmp/test");
              expected.field("host", "test@localhost");
              expected.field("headerNameTwo", "headerValueTwo");
              expected.field("source", "flume_tail_src");
              expected.field("headerNameOne", "headerValueOne");
              expected.field("type", "sometype");
          expected.endObject();

      expected.endObject();

  XContentBuilder actual = fixture.getContentBuilder(event);

  JsonParser parser = new JsonParser();
  assertEquals(parser.parse(expected.string()),parser.parse(actual.string()));
}