com.maxmind.geoip2.model.IspResponse Java Examples

public String solve(String key) {
	String value = null;
	try {
		String parts[] = key.split("/");
		if (parts.length < 3)
			return null;
		String sip = parts[2];
		InetAddress ip = InetAddress.getByName(sip);
		IspResponse r = reader.isp(ip);
		switch (parts[1]) {
		case "org":
			return r.getOrganization();
		case "isp":
			return r.getIsp();
		case "json":
			return r.toJson();
		default:
			return null;
		}
	} catch (Exception e) {

	}
	return value;
}
 

public String solve(String key) {
	String value = null;
	try {
		String parts[] = key.split("/");
		if (parts.length < 3)
			return null;
		String sip = parts[2];
		InetAddress ip = InetAddress.getByName(sip);
		IspResponse r = reader.isp(ip);
		switch (parts[1]) {
		case "org":
			return r.getOrganization();
		case "isp":
			return r.getIsp();
		case "json":
			return r.toJson();
		default:
			return null;
		}
	} catch (Exception e) {

	}
	return value;
}
 

private IspResponse getIspResponse(final String ipAddress) throws Exception {
    final String maxMindIspResponse = "{\n" +
        "         \"isp\" : \"Apache NiFi - Test ISP\",\n" +
        "         \"organization\" : \"Apache NiFi - Test Organization\",\n" +
        "         \"autonomous_system_number\" : 1337,\n" +
        "         \"autonomous_system_organization\" : \"Apache NiFi - Test Chocolate\", \n" +
        "         \"ip_address\" : \"" + ipAddress + "\"\n" +
        "      }\n";

    InjectableValues inject = new InjectableValues.Std().addValue("locales", Collections.singletonList("en"));
    ObjectMapper mapper = new ObjectMapper();
    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);


    return new ObjectMapper().readerFor(IspResponse.class).with(inject).readValue(maxMindIspResponse);

}
 

protected void extractIspFields(final Parsable<?> parsable, final String inputname, IspResponse response) throws DissectionFailure {
    if (wantIspName) {
        parsable.addDissection(inputname, "STRING", "isp.name", response.getIsp());
    }
    if (wantIspOrganization) {
        parsable.addDissection(inputname, "STRING", "isp.organization", response.getOrganization());
    }
}
 

public boolean contains(String key) {
	try {
		InetAddress ip = InetAddress.getByName(key);
		IspResponse r = reader.isp(ip);
		String test = r.getIsp().toLowerCase();
		if (test.contains("hosting")) {
			return false;
		}
	} catch (Exception error) {
		error.printStackTrace();
	}
	return true;
}
 

public void dissect(final Parsable<?> parsable, final String inputname, final InetAddress ipAddress) throws DissectionFailure {
    IspResponse response;
    try {
        response = reader.isp(ipAddress);
    } catch (IOException | GeoIp2Exception e) {
        return;
    }

    extractAsnFields(parsable, inputname, response);
    extractIspFields(parsable, inputname, response);
}
 

private IspResponse getIspResponseWithoutASNDetail(final String ipAddress) throws Exception {
    final String maxMindIspResponse = "{\n" +
        "         \"isp\" : \"Apache NiFi - Test ISP\",\n" +
        "         \"organization\" : \"Apache NiFi - Test Organization\",\n" +
        "         \"autonomous_system_number\" : null,\n" +
        "         \"ip_address\" : \"" + ipAddress + "\"\n" +
        "      }\n";

    InjectableValues inject = new InjectableValues.Std().addValue("locales", Collections.singletonList("en"));
    ObjectMapper mapper = new ObjectMapper();
    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);


    return new ObjectMapper().readerFor(IspResponse.class).with(inject).readValue(maxMindIspResponse);
}
 

@Test
public void evaluatingExpressionLanguageShouldAndFindingIpFieldWithSuccessfulLookUpShouldFlowToFoundRelationship() throws Exception {
    testRunner.setProperty(ISPEnrichIP.GEO_DATABASE_FILE, "./");
    testRunner.setProperty(ISPEnrichIP.IP_ADDRESS_ATTRIBUTE, "${ip.fields:substringBefore(',')}");

    final IspResponse ispResponse = getIspResponse("1.2.3.4");
    when(databaseReader.isp(InetAddress.getByName("1.2.3.4"))).thenReturn(ispResponse);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("ip.fields", "ip0,ip1,ip2");
    attributes.put("ip0", "1.2.3.4");

    testRunner.enqueue(new byte[0], attributes);

    testRunner.run();

    List<MockFlowFile> notFound = testRunner.getFlowFilesForRelationship(ISPEnrichIP.REL_NOT_FOUND);
    List<MockFlowFile> found = testRunner.getFlowFilesForRelationship(ISPEnrichIP.REL_FOUND);

    assertEquals(0, notFound.size());
    assertEquals(1, found.size());

    FlowFile finishedFound = found.get(0);
    assertNotNull(finishedFound.getAttribute("ip0.isp.lookup.micros"));
    assertEquals("Apache NiFi - Test ISP", finishedFound.getAttribute("ip0.isp.name"));
    assertEquals("Apache NiFi - Test Organization", finishedFound.getAttribute("ip0.isp.organization"));
    assertEquals("1337", finishedFound.getAttribute("ip0.isp.asn"));
    assertEquals("Apache NiFi - Test Chocolate", finishedFound.getAttribute("ip0.isp.asn.organization"));

}
 

@Test
public void successfulMaxMindResponseShouldFlowToFoundRelationshipWhenAsnIsNotSet() throws Exception {
    testRunner.setProperty(ISPEnrichIP.GEO_DATABASE_FILE, "./");
    testRunner.setProperty(ISPEnrichIP.IP_ADDRESS_ATTRIBUTE, "ip");

    final IspResponse ispResponse = getIspResponseWithoutASNDetail("1.2.3.4");

    when(databaseReader.isp(InetAddress.getByName("1.2.3.4"))).thenReturn(ispResponse);


    final Map<String, String> attributes = new HashMap<>();
    attributes.put("ip", "1.2.3.4");

    testRunner.enqueue(new byte[0], attributes);

    testRunner.run();

    List<MockFlowFile> notFound = testRunner.getFlowFilesForRelationship(ISPEnrichIP.REL_NOT_FOUND);
    List<MockFlowFile> found = testRunner.getFlowFilesForRelationship(ISPEnrichIP.REL_FOUND);

    assertEquals(0, notFound.size());
    assertEquals(1, found.size());

    FlowFile finishedFound = found.get(0);
    assertNotNull(finishedFound.getAttribute("ip.isp.lookup.micros"));
    assertNotNull(finishedFound.getAttribute("ip.isp.lookup.micros"));
    assertEquals("Apache NiFi - Test ISP", finishedFound.getAttribute("ip.isp.name"));
    assertEquals("Apache NiFi - Test Organization", finishedFound.getAttribute("ip.isp.organization"));
    assertNull(finishedFound.getAttribute("ip.isp.asn"));
    assertNull(finishedFound.getAttribute("ip.isp.asn.organization"));
}
 

@Test
public void successfulMaxMindResponseShouldFlowToFoundRelationship() throws Exception {
    testRunner.setProperty(ISPEnrichIP.GEO_DATABASE_FILE, "./");
    testRunner.setProperty(ISPEnrichIP.IP_ADDRESS_ATTRIBUTE, "ip");

    final IspResponse ispResponse = getIspResponse("1.2.3.4");

    when(databaseReader.isp(InetAddress.getByName("1.2.3.4"))).thenReturn(ispResponse);

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("ip", "1.2.3.4");

    testRunner.enqueue(new byte[0], attributes);

    testRunner.run();

    List<MockFlowFile> notFound = testRunner.getFlowFilesForRelationship(ISPEnrichIP.REL_NOT_FOUND);
    List<MockFlowFile> found = testRunner.getFlowFilesForRelationship(ISPEnrichIP.REL_FOUND);

    assertEquals(0, notFound.size());
    assertEquals(1, found.size());

    FlowFile finishedFound = found.get(0);
    assertNotNull(finishedFound.getAttribute("ip.isp.lookup.micros"));
    assertEquals("Apache NiFi - Test ISP", finishedFound.getAttribute("ip.isp.name"));
    assertEquals("Apache NiFi - Test Organization", finishedFound.getAttribute("ip.isp.organization"));
    assertEquals("1337", finishedFound.getAttribute("ip.isp.asn"));
    assertEquals("Apache NiFi - Test Chocolate", finishedFound.getAttribute("ip.isp.asn.organization"));
}
 

private Record createRecord(final IspResponse isp) {
    if (isp == null) {
        return null;
    }

    final Map<String, Object> values = new HashMap<>(4);
    values.put(IspSchema.ASN.getFieldName(), isp.getAutonomousSystemNumber());
    values.put(IspSchema.ASN_ORG.getFieldName(), isp.getAutonomousSystemOrganization());
    values.put(IspSchema.NAME.getFieldName(), isp.getIsp());
    values.put(IspSchema.ORG.getFieldName(), isp.getOrganization());

    return new MapRecord(IspSchema.ISP_SCHEMA, values);
}
 

public boolean contains(String key) {
	try {
		InetAddress ip = InetAddress.getByName(key);
		IspResponse r = reader.isp(ip);
		String test = r.getIsp().toLowerCase();
		if (test.contains("hosting")) {
			return false;
		}
	} catch (Exception error) {
		error.printStackTrace();
	}
	return true;
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final DatabaseReader dbReader = databaseReaderRef.get();
    final String ipAttributeName = context.getProperty(IP_ADDRESS_ATTRIBUTE).evaluateAttributeExpressions(flowFile).getValue();
    final String ipAttributeValue = flowFile.getAttribute(ipAttributeName);

    if (StringUtils.isEmpty(ipAttributeName)) {
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("FlowFile '{}' attribute '{}' was empty. Routing to failure",
                new Object[]{flowFile, IP_ADDRESS_ATTRIBUTE.getDisplayName()});
        return;
    }

    InetAddress inetAddress = null;
    IspResponse response = null;

    try {
        inetAddress = InetAddress.getByName(ipAttributeValue);
    } catch (final IOException ioe) {
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("Could not resolve the IP for value '{}', contained within the attribute '{}' in " +
                        "FlowFile '{}'. This is usually caused by issue resolving the appropriate DNS record or " +
                        "providing the processor with an invalid IP address ",
                new Object[]{ipAttributeValue, IP_ADDRESS_ATTRIBUTE.getDisplayName(), flowFile}, ioe);
        return;
    }
    final StopWatch stopWatch = new StopWatch(true);
    try {
        response = dbReader.isp(inetAddress);
        stopWatch.stop();
    } catch (final IOException ex) {
        // Note IOException is captured again as dbReader also makes InetAddress.getByName() calls.
        // Most name or IP resolutions failure should have been triggered in the try loop above but
        // environmental conditions may trigger errors during the second resolution as well.
        session.transfer(flowFile, REL_NOT_FOUND);
        getLogger().warn("Failure while trying to find enrichment data for {} due to {}", new Object[]{flowFile, ex}, ex);
        return;
    }

    if (response == null) {
        session.transfer(flowFile, REL_NOT_FOUND);
        return;
    }

    final Map<String, String> attrs = new HashMap<>();
    attrs.put(new StringBuilder(ipAttributeName).append(".isp.lookup.micros").toString(), String.valueOf(stopWatch.getDuration(TimeUnit.MICROSECONDS)));



    // During test I observed behavior where null values in ASN data could trigger NPEs. Instead of relying on the
    // underlying database to be free from Nulls wrapping ensure equality to null without assigning a variable
    // seem like good option to "final int asn ..." as with the other returned data.
    if (!(response.getAutonomousSystemNumber() == null)) {
        attrs.put(new StringBuilder(ipAttributeName).append(".isp.asn").toString(), String.valueOf(response.getAutonomousSystemNumber()));
    }
    final String asnOrg = response.getAutonomousSystemOrganization();
    if (asnOrg != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".isp.asn.organization").toString(), asnOrg);
    }

    final String ispName = response.getIsp();
    if (ispName != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".isp.name").toString(), ispName);
    }

    final String organisation = response.getOrganization();
    if (organisation  != null) {
        attrs.put(new StringBuilder(ipAttributeName).append(".isp.organization").toString(), organisation);
    }

    flowFile = session.putAllAttributes(flowFile, attrs);

    session.transfer(flowFile, REL_FOUND);
}
 

@Override
public PubsubMessage apply(PubsubMessage message) {
  message = PubsubConstraints.ensureNonNull(message);

  try {
    if (ispReader == null) {
      loadResourcesOnFirstMessage();
    }

    if (message.getAttributeMap().containsKey(Attribute.ISP_NAME)) {
      // Return early since ISP lookup has already been performed.
      countIspAlreadyApplied.inc();
      return message;
    }

    // copy attributes
    Map<String, String> attributes = new HashMap<String, String>(message.getAttributeMap());

    // Determine client ip
    String ip;
    String xff = attributes.get(Attribute.X_FORWARDED_FOR);

    if (xff != null) {
      // Google's load balancer will append the immediate sending client IP and a global
      // forwarding rule IP to any existing content in X-Forwarded-For as documented in:
      // https://cloud.google.com/load-balancing/docs/https/#components
      //
      // In practice, many of the "first" addresses are bogus or internal,
      // so we target the immediate sending client IP.
      String[] ips = xff.split("\\s*,\\s*");
      ip = ips[Math.max(ips.length - 2, 0)];
      countIpForwarded.inc();
    } else {
      ip = attributes.getOrDefault(Attribute.REMOTE_ADDR, "");
      countIpRemoteAddr.inc();
    }

    try {
      attributes.put(Attribute.ISP_DB_VERSION, DateTimeFormatter.ISO_INSTANT
          .format(Instant.ofEpochMilli(ispReader.getMetadata().getBuildDate().getTime())));

      // Throws UnknownHostException
      InetAddress ipAddress = InetAddress.getByName(ip);
      foundIp.inc();

      IspResponse response = ispReader.isp(ipAddress);
      foundIsp.inc();

      attributes.put(Attribute.ISP_NAME, response.getIsp());
      attributes.put(Attribute.ISP_ORGANIZATION, response.getOrganization());
    } catch (UnknownHostException | GeoIp2Exception ignore) {
      // ignore these exceptions
    }

    // remove null attributes because the coder can't handle them
    attributes.values().removeIf(Objects::isNull);

    return new PubsubMessage(message.getPayload(), attributes);
  } catch (IOException e) {
    // Re-throw unchecked, so that the pipeline will fail at run time if it occurs
    throw new UncheckedIOException(e);
  }
}
 

/**
 * Look up an IP address in a GeoIP2 ISP database.
 *
 * @param ipAddress IPv4 or IPv6 address to lookup.
 * @return an IspResponse for the requested IP address.
 * @throws IOException if there is an IO error
 */
public IspResponse isp(InetAddress ipAddress) throws IOException {
    return get(ipAddress, IspResponse.class, false, "GeoIP2-ISP");
}
 

/**
 * Look up an IP address in a GeoIP2 ISP database.
 *
 * @param ipAddress IPv4 or IPv6 address to lookup.
 * @return an IspResponse for the requested IP address.
 * @throws GeoIp2Exception if there is an error looking up the IP
 * @throws IOException if there is an IO error
 */
public IspResponse isp(InetAddress ipAddress) throws IOException, GeoIp2Exception {
    return this.get(ipAddress, IspResponse.class, false, "GeoIP2-ISP");
}
 

/**
 * Look up an IP address in a GeoIP2 ISP database.
 *
 * @param ipAddress IPv4 or IPv6 address to lookup.
 * @return an IspResponse for the requested IP address.
 * @throws GeoIp2Exception if there is an error looking up the IP
 * @throws IOException if there is an IO error
 */
public IspResponse isp(InetAddress ipAddress) throws IOException,
        GeoIp2Exception {
    return this.get(ipAddress, IspResponse.class, false, "GeoIP2-ISP");
}