com.maxmind.db.CHMCache Java Examples

private static synchronized DatabaseReader getOrCreateSingletonIspReader(
    ValueProvider<String> ispDatabase) throws IOException {
  if (singletonIspReader == null) {
    File mmdb;

    try {
      InputStream inputStream = BeamFileInputStream.open(ispDatabase.get());
      Path mmdbPath = Paths.get(System.getProperty("java.io.tmpdir"), "GeoIspLookup.mmdb");
      Files.copy(inputStream, mmdbPath, StandardCopyOption.REPLACE_EXISTING);
      mmdb = mmdbPath.toFile();
    } catch (IOException e) {
      throw new IOException("Exception thrown while fetching configured geoIspDatabase", e);
    }
    singletonIspReader = new DatabaseReader.Builder(mmdb).withCache(new CHMCache()).build();
  }
  return singletonIspReader;
}
 

public AddGeoLocalisationTransformator() {
    super(TypeValidation.ADD_GEO_LOCALISATION,"Translate an ip into geo localisation");

    // TODO: Replace this bullshit code
    try {
        File tmpFile = File.createTempFile("bwx", "dat");
        tmpFile.deleteOnExit();
        InputStream is = AddGeoLocalisationTransformator.class.getResourceAsStream("/GeoLite2-City.mmdb");
        OutputStream os = new FileOutputStream(tmpFile);

        byte[] buffer = new byte[4000];
        int len;
        while ((len = is.read(buffer)) != -1) {
            os.write(buffer, 0, len);
        }

        is.close();
        os.close();

        reader = new DatabaseReader.Builder(tmpFile).withCache(new CHMCache()).build();
    } catch (Exception ex) {
        ex.printStackTrace();
    }
}
 

/**
 * Returns a singleton object for reading from the GeoCity database.
 *
 * <p>We copy the configured database file to a static temp location so that the MaxMind API can
 * save on heap usage by using memory mapping. The reader is threadsafe and this singleton pattern
 * allows multiple worker threads on the same machine to share a single reader instance.
 *
 * <p>Note that we do not clean up the temp mmdb file, but it's a static path, so running locally
 * will overwrite the existing path every time rather than creating an unbounded number of copies.
 * This also assumes that only one JVM per machine is running this code. In the production case
 * where this is running on Cloud Dataflow, we should always have a clean environment and the temp
 * state will be cleaned up along with the workers once the job finishes. However, behavior is
 * undefined if you run multiple local jobs concurrently.
 *
 * @throws IOException if the configured file path is not a valid .mmdb file
 */
private static synchronized DatabaseReader getOrCreateSingletonGeoCityReader(
    ValueProvider<String> geoCityDatabase) throws IOException {
  if (singletonGeoCityReader == null) {
    File mmdb;
    try {
      InputStream inputStream;
      Metadata metadata = FileSystems.matchSingleFileSpec(geoCityDatabase.get());
      ReadableByteChannel channel = FileSystems.open(metadata.resourceId());
      inputStream = Channels.newInputStream(channel);
      Path mmdbPath = Paths.get(System.getProperty("java.io.tmpdir"), "GeoCityLookup.mmdb");
      Files.copy(inputStream, mmdbPath, StandardCopyOption.REPLACE_EXISTING);
      mmdb = mmdbPath.toFile();
    } catch (IOException e) {
      throw new IOException("Exception thrown while fetching configured geoCityDatabase", e);
    }
    singletonGeoCityReader = new DatabaseReader.Builder(mmdb).withCache(new CHMCache()).build();
  }
  return singletonGeoCityReader;
}
 

@Override
public void setConf(AbstractConfig config) {
  this.config = (GeoIpOperationConfig) config;
  AmazonS3Client client = this.s3Factory.newInstance();

  AmazonS3URI uri = new AmazonS3URI(this.config.getGeoLiteDb());
  GetObjectRequest req = new GetObjectRequest(uri.getBucket(), uri.getKey());
  S3Object obj = client.getObject(req);

  try {
    this.databaseReader =
        new DatabaseReader.Builder(obj.getObjectContent()).withCache(new CHMCache()).build();
  } catch (IOException e) {
    throw new ConfigurationException("Unable to read " + this.config.getGeoLiteDb(), e);
  }
}
 

@VisibleForTesting
static void loadDatabaseReader(String location) {
    try {
        databaseReader = new DatabaseReader.Builder(Resources.getResource(location).openStream())
                .fileMode(Reader.FileMode.MEMORY)
                .withCache(new CHMCache())
                .build();
    } catch (Exception e) {
        throw new SawmillException("Failed to load geoip database", e);
    }
}
 

public static void main(String[] args) throws IOException, InvalidDatabaseException {
    File file = new File(args.length > 0 ? args[0] : "GeoLite2-City.mmdb");
    System.out.println("No caching");
    loop("Warming up", file, WARMUPS, NoCache.getInstance());
    loop("Benchmarking", file, BENCHMARKS, NoCache.getInstance());

    System.out.println("With caching");
    loop("Warming up", file, WARMUPS, new CHMCache());
    loop("Benchmarking", file, BENCHMARKS, new CHMCache());
}
 

@Override
public void prepareForRun() throws InvalidDissectorException {
    // This creates the DatabaseReader object, which should be reused across lookups.
    try {
        reader = new DatabaseReader
            .Builder(openDatabaseFile(databaseFileName))
            .fileMode(Reader.FileMode.MEMORY)
            .withCache(new CHMCache())
            .build();
    } catch (IOException e) {
        throw new InvalidDissectorException(this.getClass().getCanonicalName() + ":" + e.getMessage());
    }
}
 

public static void main(String[] args) throws GeoIp2Exception, IOException {
    File file = new File(args.length > 0 ? args[0] : "GeoLite2-City.mmdb");
    System.out.println("No caching");
    loop("Warming up", file, WARMUPS, NoCache.getInstance());
    loop("Benchmarking", file, BENCHMARKS, NoCache.getInstance());

    System.out.println("With caching");
    loop("Warming up", file, WARMUPS, new CHMCache());
    loop("Benchmarking", file, BENCHMARKS, new CHMCache());
}
 

@Scheduled(fixedDelay = 86400 * 1000) //Every 24H
@PostConstruct
private void downloadIPLocationDatabase() throws Exception {
    if(config.getGeoIP().isEnabled()){
        logger.info("Downloading GEOIP Database from: "+config.getGeoIP().getUrl());

        File tmpGEOIPFileRound = File.createTempFile("geoIP", "mmdb");
        FileOutputStream fos = new FileOutputStream(tmpGEOIPFileRound);
        HttpURLConnection connection = getURLConnection(config.getGeoIP().getUrl());
        if (connection.getResponseCode() != 200) {
            return;
        }
        TarArchiveInputStream tarGzGeoIPStream = new TarArchiveInputStream(new GZIPInputStream(connection.getInputStream()));
        TarArchiveEntry entry= null;
        int offset;
        long pointer=0;
        while ((entry = tarGzGeoIPStream.getNextTarEntry()) != null) {
            pointer+=entry.getSize();
            if(entry.getName().endsWith("GeoLite2-City.mmdb")){
                byte[] content = new byte[(int) entry.getSize()];
                offset=0;
                //FileInputStream fis = new FileInputStream(entry.getFile());
                //IOUtils.copy(fis,fos);
                //tarGzGeoIPStream.skip(pointer);
                //tarGzGeoIPStream.read(content,offset,content.length-offset);
                //IOUtils.write(content,fos);
                int r;
                byte[] b = new byte[1024];
                while ((r = tarGzGeoIPStream.read(b)) != -1) {
                    fos.write(b, 0, r);
                }
                //fis.close();
                break;
            }
        }
        tarGzGeoIPStream.close();
        fos.flush();
        fos.close();
        connection.disconnect();
        geoIPDB = new DatabaseReader.Builder(tmpGEOIPFileRound).withCache(new CHMCache()).build();
        if (tmpGEOIPFile != null && tmpGEOIPFile.exists()) {
            tmpGEOIPFile.delete();
        }
        tmpGEOIPFile = tmpGEOIPFileRound;
    }
}