org.graylog2.plugin.Message Java Examples

private Message parseEvent(JsonNode event) {
    final String beatsType = event.path("@metadata").path("beat").asText("beat");
    final String rootPath = useBeatPrefix ? beatsType : "";
    final String message = event.path("message").asText("-");
    final String timestampField = event.path("@timestamp").asText();
    final DateTime timestamp = Tools.dateTimeFromString(timestampField);

    final JsonNode beat = event.path("beat");
    final String hostname = beat.path("hostname").asText(BEATS_UNKNOWN);

    final Message gelfMessage = new Message(message, hostname, timestamp);
    gelfMessage.addField("beats_type", beatsType);
    gelfMessage.addField("facility", "beats");

    addFlattened(gelfMessage, rootPath, event);
    return gelfMessage;
}
 

@Test
public void decodeMessagesReturnsNullIfNetFlowParserThrowsFlowException() throws Exception {
    final byte[] b = "Foobar".getBytes(StandardCharsets.UTF_8);
    final InetSocketAddress source = new InetSocketAddress(InetAddress.getLocalHost(), 12345);
    final RawMessage rawMessage = new RawMessage(b, source) {
        private boolean triggered = false;
        @Override
        public byte[] getPayload() {
            if (triggered) {
                return new byte[]{};
            }
            triggered = true;
            throw new FlowException("Boom!");
        }
    };

    final Collection<Message> messages = codec.decodeMessages(rawMessage);
    assertThat(messages).isNull();
}
 

@Override
public void sendEmails(Stream stream, EmailRecipients recipients, AlertCondition.CheckResult checkResult, List<Message> backlog) throws TransportConfigurationException, EmailException {
    if(!configuration.isEnabled()) {
        throw new TransportConfigurationException("Email transport is not enabled in server configuration file!");
    }

    if (recipients == null || recipients.isEmpty()) {
        throw new RuntimeException("Cannot send emails: empty recipient list.");
    }

    final Set<String> recipientsSet = recipients.getEmailRecipients();
    if (recipientsSet.size() == 0) {
        final Notification notification = notificationService.buildNow()
            .addNode(nodeId.toString())
            .addType(Notification.Type.GENERIC)
            .addSeverity(Notification.Severity.NORMAL)
            .addDetail("title", "Stream \"" + stream.getTitle() + "\" is alerted, but no recipients have been defined!")
            .addDetail("description", "To fix this, go to the alerting configuration of the stream and add at least one alert recipient.");
        notificationService.publishIfFirst(notification);
    }

    for (String email : recipientsSet) {
        sendEmail(email, stream, checkResult, backlog);
    }
}
 

@Test
public void decodeMessagesSuccessfullyDecodesNetFlowV5() throws Exception {
    final Collection<Message> messages = decodeResult(aggregateRawPacket("netflow-data/netflow-v5-1.dat"));
    assertThat(messages)
            .isNotNull()
            .hasSize(2);
    final Message message = Iterables.get(messages, 0);
    assertThat(message).isNotNull();

    assertThat(message.getMessage()).isEqualTo("NetFlowV5 [10.0.2.2]:54435 <> [10.0.2.15]:22 proto:6 pkts:5 bytes:230");
    assertThat(message.getTimestamp()).isEqualTo(DateTime.parse("2015-05-02T18:38:08.280Z"));
    assertThat(message.getSource()).isEqualTo(source.getAddress().getHostAddress());
    assertThat(message.getFields())
            .containsEntry("nf_src_address", "10.0.2.2")
            .containsEntry("nf_dst_address", "10.0.2.15")
            .containsEntry("nf_proto_name", "TCP")
            .containsEntry("nf_src_as", 0)
            .containsEntry("nf_dst_as", 0)
            .containsEntry("nf_snmp_input", 0)
            .containsEntry("nf_snmp_output", 0);
}
 

private Collection<Message> decodePcapStream(String resourceName) throws IOException {

        final List<Message> allMessages = Lists.newArrayList();
        try (InputStream inputStream = Resources.getResource(resourceName).openStream()) {
            final Pcap pcap = Pcap.openStream(inputStream);
            pcap.loop(packet -> {
                        if (packet.hasProtocol(Protocol.UDP)) {
                            final UDPPacket udp = (UDPPacket) packet.getPacket(Protocol.UDP);
                            final InetSocketAddress source = new InetSocketAddress(udp.getSourceIP(), udp.getSourcePort());
                            final CodecAggregator.Result result = codecAggregator.addChunk(Unpooled.copiedBuffer(udp.getPayload().getArray()), source);
                            if (result.isValid() && result.getMessage() != null) {
                                final Collection<Message> c = codec.decodeMessages(convertToRawMessage(result, source));
                                if (c != null) {
                                    allMessages.addAll(c);
                                }
                            }
                        }
                        return true;
                    }
            );
        }
        return allMessages;
    }
 

protected List<Message> getAlarmBacklog(AlertCondition.CheckResult result) {
    final AlertCondition alertCondition = result.getTriggeredCondition();
    final List<MessageSummary> matchingMessages = result.getMatchingMessages();

    final int effectiveBacklogSize = Math.min(alertCondition.getBacklog(), matchingMessages.size());

    if (effectiveBacklogSize == 0) {
        return Collections.emptyList();
    }

    final List<MessageSummary> backlogSummaries = matchingMessages.subList(0, effectiveBacklogSize);

    final List<Message> backlog = Lists.newArrayListWithCapacity(effectiveBacklogSize);

    for (MessageSummary messageSummary : backlogSummaries) {
        backlog.add(messageSummary.getRawMessage());
    }

    return backlog;
}
 

@Nullable
@Override
public Message decodeLogData(@Nonnull final CloudWatchLogEntry logEvent, @Nonnull final String logGroup, @Nonnull final String logStream) {
    try {
        final FlowLogMessage flowLogMessage = FlowLogMessage.fromLogEvent(logEvent);

        if (flowLogMessage == null) {
            return null;
        }

        final String source = configuration.getString(CloudTrailCodec.Config.CK_OVERRIDE_SOURCE, "aws-flowlogs");
        final Message result = new Message(
                buildSummary(flowLogMessage),
                source,
                flowLogMessage.getTimestamp()
        );
        result.addFields(buildFields(flowLogMessage));
        result.addField(AWS.FIELD_LOG_GROUP, logGroup);
        result.addField(AWS.FIELD_LOG_STREAM, logStream);
        result.addField(AWS.SOURCE_GROUP_IDENTIFIER, true);

        return result;
    } catch (Exception e) {
        throw new RuntimeException("Could not deserialize AWS FlowLog record.", e);
    }
}
 

@Test
public void decodeMessagesHandlesGenericBeatWithCloudGCE() throws Exception {
    final Message message = codec.decode(messageFromJson("generic-with-cloud-gce.json"));
    assertThat(message).isNotNull();
    assertThat(message.getMessage()).isEqualTo("-");
    assertThat(message.getSource()).isEqualTo("unknown");
    assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 4, 1, 0, 0, DateTimeZone.UTC));
    assertThat(message.getField("facility")).isEqualTo("beats");
    assertThat(message.getField("beats_type")).isEqualTo("beat");
    assertThat(message.getField("beat_foo")).isEqualTo("bar");
    assertThat(message.getField("beat_meta_cloud_provider")).isEqualTo("gce");
    assertThat(message.getField("beat_meta_cloud_machine_type")).isEqualTo("projects/1234567890/machineTypes/f1-micro");
    assertThat(message.getField("beat_meta_cloud_instance_id")).isEqualTo("1234556778987654321");
    assertThat(message.getField("beat_meta_cloud_project_id")).isEqualTo("my-dev");
    assertThat(message.getField("beat_meta_cloud_availability_zone")).isEqualTo("projects/1234567890/zones/us-east1-b");
}
 

@Test
public void decodeMessagesHandlesGenericBeatWithCloudEC2() throws Exception {
    final Message message = codec.decode(messageFromJson("generic-with-cloud-ec2.json"));
    assertThat(message).isNotNull();
    assertThat(message.getMessage()).isEqualTo("-");
    assertThat(message.getSource()).isEqualTo("unknown");
    assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 4, 1, 0, 0, DateTimeZone.UTC));
    assertThat(message.getField("facility")).isEqualTo("beats");
    assertThat(message.getField("beats_type")).isEqualTo("beat");
    assertThat(message.getField("beat_foo")).isEqualTo("bar");
    assertThat(message.getField("beat_meta_cloud_provider")).isEqualTo("ec2");
    assertThat(message.getField("beat_meta_cloud_machine_type")).isEqualTo("t2.medium");
    assertThat(message.getField("beat_meta_cloud_instance_id")).isEqualTo("i-4e123456");
    assertThat(message.getField("beat_meta_cloud_region")).isEqualTo("us-east-1");
    assertThat(message.getField("beat_meta_cloud_availability_zone")).isEqualTo("us-east-1c");
}
 

@Nullable
@Override
public Message decodeLogData(@Nonnull final CloudWatchLogEntry logEvent, @Nonnull final String logGroup, @Nonnull final String logStream) {
    try {
        final String source = configuration.getString(CloudTrailCodec.Config.CK_OVERRIDE_SOURCE, "aws-raw-logs");
        Message result = new Message(
                logEvent.message,
                source,
                new DateTime(logEvent.timestamp)
        );
        result.addField(AWS.FIELD_LOG_GROUP, logGroup);
        result.addField(AWS.FIELD_LOG_STREAM, logStream);

        return result;
    } catch (Exception e) {
        throw new RuntimeException("Could not deserialize AWS FlowLog record.", e);
    }
}
 

@Nullable
@Override
public Message decode(@Nonnull RawMessage rawMessage) {
    try {
        final CloudTrailRecord record = objectMapper.readValue(rawMessage.getPayload(), CloudTrailRecord.class);
        final String source = configuration.getString(Config.CK_OVERRIDE_SOURCE, "aws-cloudtrail");
        final Message message = new Message(record.getConstructedMessage(), source, DateTime.parse(record.eventTime));

        message.addFields(record.additionalFieldsAsMap());
        message.addField("full_message", record.getFullMessage());
        message.addField(AWS.SOURCE_GROUP_IDENTIFIER, true);

        return message;
    } catch (Exception e) {
        throw new RuntimeException("Could not deserialize CloudTrail record.", e);
    }
}
 

@Override
public void write(Message msg) throws RuntimeException {
    boolean shortMode = configuration.getBoolean(SlackConfiguration.CK_SHORT_MODE);
    String message = shortMode ? buildShortMessageBody(msg) : buildFullMessageBody(stream, msg);
    SlackMessage slackMessage = createSlackMessage(configuration, message);

    // Add custom message
    String template = configuration.getString(SlackConfiguration.CK_CUSTOM_MESSAGE);
    Boolean hasTemplate = !isNullOrEmpty(template);
    if (!shortMode && hasTemplate) {
        String customMessage = buildCustomMessage(stream, msg, template);
        slackMessage.setCustomMessage(customMessage);
    }

    // Add attachments
    boolean addDetails = configuration.getBoolean(SlackConfiguration.CK_ADD_DETAILS);
    if (!shortMode && addDetails) {
        buildDetailsAttachment(msg, slackMessage);
    }

    try {
        client.send(slackMessage);
    } catch (SlackClient.SlackClientException e) {
        throw new RuntimeException("Could not send message to Slack.", e);
    }
}
 

private String buildFullMessageBody(Stream stream, Message msg) {
    String graylogUri = configuration.getString(SlackConfiguration.CK_GRAYLOG2_URL);
    String titleLink;
    if (!isNullOrEmpty(graylogUri)) {
        titleLink = "<" + buildStreamLink(graylogUri, stream) + "|" + stream.getTitle() + ">";
    } else {
        titleLink = "_" + stream.getTitle() + "_";
    }

    String messageLink;
    if (!isNullOrEmpty(graylogUri)) {
        String index = "graylog_deflector"; // would use msg.getFieldAs(String.class, "_index"), but it returns null
        messageLink = "<" + buildMessageLink(graylogUri, index, msg.getId()) + "|New message>";
    } else {
        messageLink = "New message";
    }

    boolean notifyChannel = configuration.getBoolean(SlackConfiguration.CK_NOTIFY_CHANNEL);
    String audience = notifyChannel ? "@channel " : "";
    return String.format("%s*%s in Graylog stream %s*:\n> %s", audience, messageLink, titleLink, msg.getMessage());
}
 

@Test
public void decodeMessagesHandlesFilebeatMessages() throws Exception {
    final Message message = codec.decode(messageFromJson("filebeat.json"));
    assertThat(message).isNotNull();
    assertThat(message.getMessage()).isEqualTo("TEST");
    assertThat(message.getSource()).isEqualTo("example.local");
    assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 4, 1, 0, 0, DateTimeZone.UTC));
    assertThat(message.getField("facility")).isEqualTo("beats");
    assertThat(message.getField("beats_type")).isEqualTo("filebeat");
    assertThat(message.getField("filebeat_source")).isEqualTo("/tmp/test.log");
    assertThat(message.getField("filebeat_input_type")).isEqualTo("log");
    assertThat(message.getField("filebeat_count")).isEqualTo(1);
    assertThat(message.getField("filebeat_offset")).isEqualTo(0);
    @SuppressWarnings("unchecked") final List<String> tags = (List<String>) message.getField("filebeat_tags");
    assertThat(tags).containsOnly("foobar", "test");
}
 

@Test
public void decodeMessagesHandlesFilebeatMessagesWithoutPrefix() throws Exception {
    configuration = new Configuration(Collections.singletonMap("beats_prefix", false));
    codec = new BeatsCodec(configuration, objectMapper);

    final Message message = codec.decode(messageFromJson("filebeat.json"));
    assertThat(message).isNotNull();
    assertThat(message.getMessage()).isEqualTo("TEST");
    assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 4, 1, 0, 0, DateTimeZone.UTC));
    assertThat(message.getField("facility")).isEqualTo("beats");
    assertThat(message.getField("beats_type")).isEqualTo("filebeat");
    assertThat(message.getField("source")).isEqualTo("/tmp/test.log");
    assertThat(message.getField("input_type")).isEqualTo("log");
    assertThat(message.getField("count")).isEqualTo(1);
    assertThat(message.getField("offset")).isEqualTo(0);
    @SuppressWarnings("unchecked") final List<String> tags = (List<String>) message.getField("tags");
    assertThat(tags).containsOnly("foobar", "test");
}
 

private void buildDetailsAttachment(Message msg, SlackMessage slackMessage) {
    slackMessage.addDetailsAttachmentField(new SlackMessage.AttachmentField("Stream Description", stream.getDescription(), false));
    slackMessage.addDetailsAttachmentField(new SlackMessage.AttachmentField("Source", msg.getSource(), true));

    for (Map.Entry<String, Object> field : msg.getFields().entrySet()) {
        if (Message.RESERVED_FIELDS.contains(field.getKey())) continue;
        slackMessage.addDetailsAttachmentField(new SlackMessage.AttachmentField(field.getKey(), field.getValue().toString(), true));
    }
}
 

private String buildCustomMessage(Stream stream, Message msg, String template) {
    Map<String, Object> model = getModel(stream, msg);
    try {
        return templateEngine.transform(template, model);
    } catch (Exception ex) {
        return ex.toString();
    }
}
 

private Map<String, Object> getModel(Stream stream, Message msg) {
    Map<String, Object> model = new HashMap<>();

    String graylogUri = configuration.getString(SlackConfiguration.CK_GRAYLOG2_URL);
    model.put("stream", stream);
    model.put("message", msg);

    if (!isNullOrEmpty(graylogUri)) {
        model.put("stream_url", buildStreamLink(graylogUri, stream));
    }

    return model;
}
 

private String buildCustomMessage(Stream stream, AlertCondition.CheckResult result, String template) {
    List<Message> backlog = getAlarmBacklog(result);
    Map<String, Object> model = getModel(stream, result, backlog);
    try {
        return templateEngine.transform(template, model);
    } catch (Exception ex) {
        return ex.toString();
    }
}
 

@Override
public Messages process(Messages messages) {
    if (config == null || !config.lookupsEnabled() || !table.isLoaded()) {
        return messages;
    }

    for (Message message : messages) {
        Object awsGroupId = message.getField(AWS.SOURCE_GROUP_IDENTIFIER);
        if(awsGroupId != null && awsGroupId.equals(true)) {
            // This is a message from one of our own inputs and we want to do a lookup.
            TRANSLATABLE_FIELD_NAMES.stream().filter(fieldName -> message.hasField(fieldName)).forEach(fieldName -> {
                // Make it so!
                message.addField(
                        fieldName + "_entity",
                        table.findByIp(message.getField(fieldName).toString()).getName()
                );

                message.addField(
                        fieldName + "_entity_description",
                        table.findByIp(message.getField(fieldName).toString()).getDescription()
                );

                message.addField(
                        fieldName + "_entity_aws_type",
                        table.findByIp(message.getField(fieldName).toString()).getAWSType()
                );
            });

        }
    }

    return messages;
}
 

private List<Message> getAlarmBacklog(AlertCondition.CheckResult result) {
    final AlertCondition alertCondition = result.getTriggeredCondition();
    final List<MessageSummary> matchingMessages = result.getMatchingMessages();
    final int effectiveBacklogSize = Math.min(alertCondition.getBacklog(), matchingMessages.size());

    if (effectiveBacklogSize == 0) return Collections.emptyList();
    final List<MessageSummary> backlogSummaries = matchingMessages.subList(0, effectiveBacklogSize);
    final List<Message> backlog = Lists.newArrayListWithCapacity(effectiveBacklogSize);
    for (MessageSummary messageSummary : backlogSummaries) {
        backlog.add(messageSummary.getRawMessage());
    }

    return backlog;
}
 

@Nullable
@Override
public Message decode(@Nonnull RawMessage rawMessage) {
    final byte[] payload = rawMessage.getPayload();
    final JsonNode event;
    try {
        event = objectMapper.readTree(payload);
    } catch (IOException e) {
        LOG.error("Couldn't decode raw message {}", rawMessage);
        return null;
    }

    return parseEvent(event);
}
 

@Test
public void pcap_pmacctd_NetFlowV5() throws Exception {
    final Collection<Message> allMessages = decodePcapStream("netflow-data/pmacctd-netflow5.pcap");

    assertThat(allMessages)
            .hasSize(42)
            .allSatisfy(message -> assertThat(message.getField("nf_version")).isEqualTo(5));
}
 

@Test
public void pcap_softflowd_NetFlowV5() throws Exception {
    final Collection<Message> allMessages = decodePcapStream("netflow-data/netflow5.pcap");

    assertThat(allMessages)
            .hasSize(4)
            .allSatisfy(message -> assertThat(message.getField("nf_version")).isEqualTo(5));
}
 

@Test
public void pcap_netgraph_NetFlowV5() throws Exception {
    final Collection<Message> allMessages = decodePcapStream("netflow-data/netgraph-netflow5.pcap");
    assertThat(allMessages)
            .hasSize(120)
            .allSatisfy(message -> assertThat(message.getField("nf_version")).isEqualTo(5));
}
 

private Map<String, Object> getModel(Stream stream, AlertCondition.CheckResult result, List<Message> backlog) {
    Map<String, Object> model = new HashMap<>();
    String graylogUri = configuration.getString(SlackConfiguration.CK_GRAYLOG2_URL);
    model.put("stream", stream);
    model.put("check_result", result);
    model.put("alert_condition", result.getTriggeredCondition());
    model.put("backlog", backlog);
    model.put("backlog_size", backlog.size());
    if (!isNullOrEmpty(graylogUri)) {
        model.put("stream_url", buildStreamLink(graylogUri, stream));
    }

    return model;
}
 

@Test
public void decodeMessagesReturnsNullIfMessageWasInvalid() throws Exception {
    final byte[] b = "Foobar".getBytes(StandardCharsets.UTF_8);
    final InetSocketAddress source = new InetSocketAddress(InetAddress.getLocalHost(), 12345);
    final RawMessage rawMessage = new RawMessage(b, source);

    final Collection<Message> messages = codec.decodeMessages(rawMessage);
    assertThat(messages).isNull();
}
 

@VisibleForTesting
Collection<Message> decodeV9(InetSocketAddress sender, ByteBuf buffer) throws InvalidProtocolBufferException {
    final List<NetFlowV9Packet> netFlowV9Packets = decodeV9Packets(buffer);

    return netFlowV9Packets.stream().map(netFlowV9Packet -> netFlowV9Packet.records().stream()
            .filter(record -> record instanceof NetFlowV9Record)
            .map(record -> NetFlowFormatter.toMessage(netFlowV9Packet.header(), record, sender))
            .collect(Collectors.toList())
    ).flatMap(Collection::stream)
     .collect(Collectors.toList());
}
 

@Test
public void decodeMessagesHandlesPacketbeatMessages() throws Exception {
    final Message message = codec.decode(messageFromJson("packetbeat-dns.json"));
    assertThat(message).isNotNull();
    assertThat(message.getSource()).isEqualTo("example.local");
    assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 4, 1, 0, 0, DateTimeZone.UTC));
    assertThat(message.getField("facility")).isEqualTo("beats");
    assertThat(message.getField("beats_type")).isEqualTo("packetbeat");
    assertThat(message.getField("packetbeat_type")).isEqualTo("dns");
    assertThat(message.getField("packetbeat_status")).isEqualTo("OK");
    assertThat(message.getField("packetbeat_method")).isEqualTo("QUERY");
    assertThat(message.getField("packetbeat_dns_answers_0_type")).isEqualTo("A");
    assertThat(message.getField("packetbeat_dns_flags_recursion_allowed")).isEqualTo(true);
}
 

@Test
public void decodeMessagesHandlesTopbeatMessages() throws Exception {
    final Message message = codec.decode(messageFromJson("topbeat-system.json"));
    assertThat(message).isNotNull();
    assertThat(message.getSource()).isEqualTo("example.local");
    assertThat(message.getTimestamp()).isEqualTo(new DateTime(2016, 4, 1, 0, 0, DateTimeZone.UTC));
    assertThat(message.getField("facility")).isEqualTo("beats");
    assertThat(message.getField("beats_type")).isEqualTo("topbeat");
    assertThat(message.getField("topbeat_type")).isEqualTo("system");
}