com.maxmind.geoip2.DatabaseReader Java Examples

private static synchronized DatabaseReader getOrCreateSingletonIspReader(
    ValueProvider<String> ispDatabase) throws IOException {
  if (singletonIspReader == null) {
    File mmdb;

    try {
      InputStream inputStream = BeamFileInputStream.open(ispDatabase.get());
      Path mmdbPath = Paths.get(System.getProperty("java.io.tmpdir"), "GeoIspLookup.mmdb");
      Files.copy(inputStream, mmdbPath, StandardCopyOption.REPLACE_EXISTING);
      mmdb = mmdbPath.toFile();
    } catch (IOException e) {
      throw new IOException("Exception thrown while fetching configured geoIspDatabase", e);
    }
    singletonIspReader = new DatabaseReader.Builder(mmdb).withCache(new CHMCache()).build();
  }
  return singletonIspReader;
}
 

public AddGeoLocalisationTransformator() {
    super(TypeValidation.ADD_GEO_LOCALISATION,"Translate an ip into geo localisation");

    // TODO: Replace this bullshit code
    try {
        File tmpFile = File.createTempFile("bwx", "dat");
        tmpFile.deleteOnExit();
        InputStream is = AddGeoLocalisationTransformator.class.getResourceAsStream("/GeoLite2-City.mmdb");
        OutputStream os = new FileOutputStream(tmpFile);

        byte[] buffer = new byte[4000];
        int len;
        while ((len = is.read(buffer)) != -1) {
            os.write(buffer, 0, len);
        }

        is.close();
        os.close();

        reader = new DatabaseReader.Builder(tmpFile).withCache(new CHMCache()).build();
    } catch (Exception ex) {
        ex.printStackTrace();
    }
}
 

private static void bench(DatabaseReader r, int count, int seed) throws GeoIp2Exception, UnknownHostException {
    Random random = new Random(seed);
    long startTime = System.nanoTime();
    byte[] address = new byte[4];
    for (int i = 0; i < count; i++) {
        random.nextBytes(address);
        InetAddress ip = InetAddress.getByAddress(address);
        CityResponse t;
        try {
            t = r.city(ip);
        } catch (AddressNotFoundException | IOException e) {
        }
        if (TRACE) {
            if (i % 50000 == 0) {
                System.out.println(i + " " + ip);
                System.out.println(t);
            }
        }
    }
    long endTime = System.nanoTime();

    long duration = endTime - startTime;
    long qps = count * 1000000000L / duration;
    System.out.println("Requests per second: " + qps);
}
 

@Test
public void givenIP_whenFetchingCity_thenReturnsCityData() throws IOException, GeoIp2Exception {
    
    ClassLoader classLoader = getClass().getClassLoader();
    File database = new File(classLoader.getResource("GeoLite2-City.mmdb").getFile());
    DatabaseReader dbReader = new DatabaseReader.Builder(database).build();

    InetAddress ipAddress = InetAddress.getByName("google.com");
    CityResponse response = dbReader.city(ipAddress);

    String countryName = response.getCountry().getName();
    String cityName = response.getCity().getName();
    String postal = response.getPostal().getCode();
    String state = response.getLeastSpecificSubdivision().getName();

}
 

private static synchronized void init() throws IOException {
    // A File object pointing to your GeoIP2 or GeoLite2 database:
    // http://dev.maxmind.com/geoip/geoip2/geolite2/
    if (database == null) {
        String s = System.getProperty(GEOIP_FILE_PROPERTY);
        if(s != null && !s.isEmpty()){
            //Use user-specified GEOIP file - mainly for testing purposes
            File f = new File(s);
            if(f.exists() && f.isFile()){
                database = f;
            } else {
                log.warn("GeoIP file (system property {}) is set to \"{}\" but this is not a valid file, using default database", GEOIP_FILE_PROPERTY, s);
                database = GeoIPFetcher.fetchCityDB();
            }
        } else {
            database = GeoIPFetcher.fetchCityDB();
        }
    }

    // This creates the DatabaseReader object, which should be reused across lookups.
    if (reader == null) {
        reader = new DatabaseReader.Builder(database).build();
    }
}
 

private static DatabaseReader loadExternalDatabase(final Path location) throws IOException {
    /*
     * We have 2 strategies here.
     * 1) If compressed, we load via a stream (which means into memory).
     * 2) If it's not compressed, we pass a File in directly which allows DatabaseReader
     *    to mmap(2) the file.
     */
    final DatabaseReader reader;
    if ("gz".equals(com.google.common.io.Files.getFileExtension(location.toString()))) {
        try (final InputStream rawStream = Files.newInputStream(location);
             final InputStream bufferedStream = new BufferedInputStream(rawStream);
             final InputStream dbStream = new GZIPInputStream(bufferedStream)) {
            logger.debug("Loading compressed GeoIP database: {}", location);
            reader = new DatabaseReader.Builder(dbStream).build();
        }
    } else {
        logger.debug("Loading GeoIP database: {}", location);
        reader = new DatabaseReader.Builder(location.toFile()).build();
    }
    logger.info("Loaded GeoIP database: {}", location);
    return reader;
}
 

/**
 * Returns a {@link Location} object with location information which may
 * not have very strictly accurate information.
 * 
 * @param ipStr         the IP Address for which a {@link Location} will be obtained.
 * @return
 * @throws Exception
 */
public static Location getIPLocation(String ipStr) throws Exception {
    System.out.println("gres = " + Locator.class.getClassLoader().getResource("GeoLite2-City.mmdb"));
    InputStream is = Locator.class.getClassLoader().getResourceAsStream("GeoLite2-City.mmdb");
    DatabaseReader reader = new DatabaseReader.Builder(is).build();
    CityResponse response = reader.city(InetAddress.getByName(ipStr));

    System.out.println("City " +response.getCity());
    System.out.println("ZIP Code " +response.getPostal().getCode());
    System.out.println("Country " +response.getCountry());
    System.out.println("Location " +response.getLocation());
    
    return new Location(response.getCity().toString(), response.getPostal().getCode(), 
        response.getCountry().toString(), response.getLocation().getTimeZone(), response.getLocation().getLatitude(), 
            response.getLocation().getLongitude(), response.getPostal().getConfidence(),
                response.getLocation().getAccuracyRadius(), response.getLocation().getPopulationDensity(),
                    response.getLocation().getAverageIncome());
}
 

@Override
public void setConf(AbstractConfig config) {
  this.config = (GeoIpOperationConfig) config;
  AmazonS3Client client = this.s3Factory.newInstance();

  AmazonS3URI uri = new AmazonS3URI(this.config.getGeoLiteDb());
  GetObjectRequest req = new GetObjectRequest(uri.getBucket(), uri.getKey());
  S3Object obj = client.getObject(req);

  try {
    this.databaseReader =
        new DatabaseReader.Builder(obj.getObjectContent()).withCache(new CHMCache()).build();
  } catch (IOException e) {
    throw new ConfigurationException("Unable to read " + this.config.getGeoLiteDb(), e);
  }
}
 

@Override
public void initDatabase(LocationDatabaseType type, String pathToDatabase) {
    Objects.requireNonNull(type);
    Objects.requireNonNull(pathToDatabase);
    LOG.info("Init {} as type {} with file {}", getClass().toString(), type.toString(), pathToDatabase);
    dbType = type;
    try {
        File f = new File(pathToDatabase);
        reader = new DatabaseReader.Builder(f).fileMode(FileMode.MEMORY_MAPPED).build();
        // mismatch
        if (!type.getGeoLite2Name().equals(reader.getMetadata().getDatabaseType())) {
            LOG.error("DatabaseType {} not match with the databasefile {}. Exiting.",
                         type.toString(), pathToDatabase);
            destroy();
        }
    } catch (Throwable e) {
        LOG.error("Couldn't initation geolocation database ", e);
        reader = null;
    }
}
 

/**
 * Returns a singleton object for reading from the GeoCity database.
 *
 * <p>We copy the configured database file to a static temp location so that the MaxMind API can
 * save on heap usage by using memory mapping. The reader is threadsafe and this singleton pattern
 * allows multiple worker threads on the same machine to share a single reader instance.
 *
 * <p>Note that we do not clean up the temp mmdb file, but it's a static path, so running locally
 * will overwrite the existing path every time rather than creating an unbounded number of copies.
 * This also assumes that only one JVM per machine is running this code. In the production case
 * where this is running on Cloud Dataflow, we should always have a clean environment and the temp
 * state will be cleaned up along with the workers once the job finishes. However, behavior is
 * undefined if you run multiple local jobs concurrently.
 *
 * @throws IOException if the configured file path is not a valid .mmdb file
 */
private static synchronized DatabaseReader getOrCreateSingletonGeoCityReader(
    ValueProvider<String> geoCityDatabase) throws IOException {
  if (singletonGeoCityReader == null) {
    File mmdb;
    try {
      InputStream inputStream;
      Metadata metadata = FileSystems.matchSingleFileSpec(geoCityDatabase.get());
      ReadableByteChannel channel = FileSystems.open(metadata.resourceId());
      inputStream = Channels.newInputStream(channel);
      Path mmdbPath = Paths.get(System.getProperty("java.io.tmpdir"), "GeoCityLookup.mmdb");
      Files.copy(inputStream, mmdbPath, StandardCopyOption.REPLACE_EXISTING);
      mmdb = mmdbPath.toFile();
    } catch (IOException e) {
      throw new IOException("Exception thrown while fetching configured geoCityDatabase", e);
    }
    singletonGeoCityReader = new DatabaseReader.Builder(mmdb).withCache(new CHMCache()).build();
  }
  return singletonGeoCityReader;
}
 

public Future<?> setDataPath(String dataFilePath) {
    try (TarArchiveInputStream tarInput = new TarArchiveInputStream(new GZIPInputStream(
            new FileInputStream(dataFilePath)))) {

        TarArchiveEntry currentEntry;
        boolean hasDatabaseFile = false;
        while ((currentEntry = tarInput.getNextTarEntry()) != null) {
            if (currentEntry.getName().contains(DATABASE_FILE_NAME)) {
                hasDatabaseFile = true;
                break;
            }
        }
        if (!hasDatabaseFile) {
            return Future.failedFuture(String.format("Database file %s not found in %s archive", DATABASE_FILE_NAME,
                    dataFilePath));
        }

        databaseReader = new DatabaseReader.Builder(tarInput).fileMode(Reader.FileMode.MEMORY).build();
        return Future.succeededFuture();
    } catch (IOException e) {
        return Future.failedFuture(
                String.format("IO Exception occurred while trying to read an archive/db file: %s", e.getMessage()));
    }
}
 

@PostConstruct
    public void init() {
        try {
            String path = env.getProperty("geolite2.city.db.path");
            if (StringUtil.isNotBlank(path)) {
                addressDatabasePath = path;
            }
//            File database = new File(addressDatabasePath);
            File database = ResourceUtils.getFile("classpath:" + addressDatabasePath);
            reader = new DatabaseReader.Builder(database).build();
        } catch (Exception e) {
            logger.error("IP地址服务初始化异常:" + e.getMessage(), e);
        }
    }
 

private static void loop(String msg, File file, int loops, NodeCache cache) throws GeoIp2Exception, IOException {
    System.out.println(msg);
    for (int i = 0; i < loops; i++) {
        DatabaseReader r = new DatabaseReader.Builder(file).fileMode(FileMode.MEMORY_MAPPED).withCache(cache).build();
        bench(r, COUNT, i);
    }
    System.out.println();
}
 

@Autowired
public TailService(@Value("${geoip2.cityfile}") String cityFile,
		@Value("${access.logs}") String accessLogs,
		SimpMessageSendingOperations messagingTemplate) {
	this.messagingTemplate = messagingTemplate;

	String databaseFile = cityFile;
	if (databaseFile != null) {
		Path database = Paths.get(databaseFile);
		if (Files.exists(database)) {
			try {
				this.reader = new DatabaseReader.Builder(database.toFile()).build();
			}
			catch (IOException e) {
				LoggerFactory.getLogger(getClass()).error("GeoIPCityService init", e);
			}
		}
	}

	this.tailers = new ArrayList<>();

	for (String logFile : accessLogs.split(",")) {
		Path p = Paths.get(logFile.trim());
		this.tailers.add(new Tailer(p.toFile(), new ListenerAdapter()));
	}

	this.executor = Executors.newFixedThreadPool(this.tailers.size());
	for (Tailer tailer : this.tailers) {
		this.executor.execute(tailer);
	}
}
 

@Override
public void prepareForRun() throws InvalidDissectorException {
    // This creates the DatabaseReader object, which should be reused across lookups.
    try {
        reader = new DatabaseReader
            .Builder(openDatabaseFile(databaseFileName))
            .fileMode(Reader.FileMode.MEMORY)
            .withCache(new CHMCache())
            .build();
    } catch (IOException e) {
        throw new InvalidDissectorException(this.getClass().getCanonicalName() + ":" + e.getMessage());
    }
}
 

@Test
public void lookupShouldReturnCountryIsoWhenDatabaseReaderWasSet() throws NoSuchFieldException, IOException,
        GeoIp2Exception {
    // given
    final Country country = new Country(null, null, null, "fr", null);
    final Continent continent = new Continent(null, "eu", null, null);
    final City city = new City(singletonList("test"), null, null, singletonMap("test", "Paris"));
    final Location location = new Location(null, null, 48.8566, 2.3522,
            null, null, null);
    final ArrayList<Subdivision> subdivisions = new ArrayList<>();
    subdivisions.add(new Subdivision(null, null, null, "paris", null));
    final CityResponse cityResponse = new CityResponse(city, continent, country, location, null,
            null, null, null, subdivisions, null);

    final DatabaseReader databaseReader = Mockito.mock(DatabaseReader.class);
    given(databaseReader.city(any())).willReturn(cityResponse);

    FieldSetter.setField(maxMindGeoLocationService,
            maxMindGeoLocationService.getClass().getDeclaredField("databaseReader"), databaseReader);

    // when
    final Future<GeoInfo> future = maxMindGeoLocationService.lookup(TEST_IP, null);

    // then
    assertThat(future.succeeded()).isTrue();
    assertThat(future.result())
            .isEqualTo(GeoInfo.builder()
                    .vendor("maxmind")
                    .continent("eu")
                    .country("fr")
                    .region("paris")
                    .city("Paris")
                    .lat(48.8566f)
                    .lon(2.3522f)
                    .build());
}
 

public static void main(String[] args) throws IOException {       // a
  String servers     = args[0];
  String groupId     = args[1];
  String inTopic     = args[2];
  String goodTopic   = args[3];
  String badTopic    = args[4];                                   // b
  String maxmindFile = args[5];                                   // b

  Consumer consumer = new Consumer(servers, groupId, inTopic);
  DatabaseReader maxmind = new DatabaseReader
          .Builder(new File(maxmindFile)).build();                // c
  FullProducer producer = new FullProducer(
    servers, goodTopic, badTopic, maxmind);                       // d
  consumer.run(producer);
}
 

public static synchronized DatabaseReader getDatabaseReader() {
    if (databaseReader!=null) return databaseReader;
    try {
        File f = new File(MAXMIND_DB_PATH);
        databaseReader = new DatabaseReader.Builder(f).build();
    } catch (IOException e) {
        lookupFailed = true;
        log.debug("MaxMind geo lookup unavailable; either download and unpack the latest "+
                "binary from "+MAXMIND_DB_URL+" into "+MAXMIND_DB_PATH+", "+
                "or specify a different HostGeoLookup implementation with the key "+
                BrooklynSystemProperties.HOST_GEO_LOOKUP_IMPL.getPropertyName()+" (error trying to read: "+e+")");
    }
    return databaseReader;
}
 

public synchronized ClientInfo manage(IStreamProcessor iStreamProcessor, HttpServletRequest request) throws UnknownHostException {
    ClientInfo client = new ClientInfo();
    request.getQueryString();
    MultiValueMap<String, String> parameters = UriComponentsBuilder.fromUriString(ProxyLiveUtils.getURL(request)).build().getQueryParams();
    String clientUser = parameters.getFirst("user");
    if (clientUser == null || clientUser.trim().equals("null")) {
        clientUser = "guest";
    }
    client.setClientUser(clientUser);
    client.setIp(InetAddress.getByName(ProxyLiveUtils.getRequestIP(request)));
    client.setBrowserInfo(ProxyLiveUtils.getBrowserInfo(request));
    client = addClientInfo(client);
    if (geoIPService.isServiceEnabled()) {
        try {
            DatabaseReader geoDBReader = geoIPService.geoGEOInfoReader();
            CityResponse cityResponse = geoDBReader.city(client.getIp());

            if (cityResponse.getLocation() != null) {
                GEOInfo geoInfo = new GEOInfo();
                geoInfo.setCity(cityResponse.getCity());
                geoInfo.setCountry(cityResponse.getCountry());
                geoInfo.setLocation(cityResponse.getLocation());
                client.setGeoInfo(geoInfo);
            }
        }catch(AddressNotFoundException anfe){
        }catch(Exception ex ){
            logger.error("Error parsing user geodata", ex);
        }
    }
    if (!client.getStreams().contains(iStreamProcessor)) {
        client.getStreams().add(iStreamProcessor);
    }
    return client;
}
 

@Override
public void destroy() {
  for (DatabaseReader reader : readers.values()) {
    IOUtils.closeQuietly(reader);
  }
  super.destroy();
}
 

private void createResponseCache() {
  responseCache = CacheBuilder.newBuilder().maximumSize(1000).build(
      new CacheLoader<Field, Map<GeolocationDBType, AbstractResponse>>() {
        @Override
        public Map<GeolocationDBType, AbstractResponse> load(Field field) throws Exception {
          Map<GeolocationDBType, AbstractResponse> responses = Maps.newHashMap();
          // Each time we load an entry, we'll opportunistically just load that entry for all
          // available databases.
          for (Map.Entry<GeolocationDBType, DatabaseReader> entry : readers.entrySet()) {
            DatabaseReader reader = Utils.checkNotNull(entry.getValue(), "DatabaseReader");
            AbstractResponse resp = null;
            switch (entry.getKey()) {
              case COUNTRY:
                resp = reader.country(toAddress(field));
                break;
              case CITY:
                resp = reader.city(toAddress(field));
                break;
              case ANONYMOUS_IP:
                resp = reader.anonymousIp(toAddress(field));
                break;
              case DOMAIN:
                resp = reader.domain(toAddress(field));
                break;
              case ISP:
                resp = reader.isp(toAddress(field));
                break;
              case CONNECTION_TYPE:
                resp =  reader.connectionType(toAddress(field));
                break;
            }
            responses.put(entry.getKey(), resp);
          }

          return responses;
        }
      });
}
 

public GeoIpOperation(String pathToIpAddress, String destFieldName, DatabaseReader databaseReader,
    List<GeoProperty> geoProperties, boolean required) {
  this.databaseReader = databaseReader;
  this.geoProperties = geoProperties;
  this.pathToIpAddress = pathToIpAddress;
  this.destFieldName = destFieldName;
  this.required = required;
}
 

@VisibleForTesting
static void loadDatabaseReader(String location) {
    try {
        databaseReader = new DatabaseReader.Builder(Resources.getResource(location).openStream())
                .fileMode(Reader.FileMode.MEMORY)
                .withCache(new CHMCache())
                .build();
    } catch (Exception e) {
        throw new SawmillException("Failed to load geoip database", e);
    }
}
 

private static synchronized void init() throws IOException {
    // A File object pointing to your GeoIP2 or GeoLite2 database:
    // http://dev.maxmind.com/geoip/geoip2/geolite2/
    if (database == null) {
        database = GeoIPFetcher.fetchCityDB();
    }

    // This creates the DatabaseReader object, which should be reused across lookups.
    if (reader == null) {
        reader = new DatabaseReader.Builder(database).build();
    }
}
 

public MMDB(String fileName) throws Exception {
	File f = new File("data/GeoIP2-ISP.mmdb");
	reader = new DatabaseReader.Builder(f).build();
}
 

public MMDB(String fileName) throws Exception {
	File f = new File("data/GeoIP2-ISP.mmdb");
	reader = new DatabaseReader.Builder(f).build();
}
 

public DatabaseReader geoGEOInfoReader(){
     return geoIPDB;
}
 

@Scheduled(fixedDelay = 86400 * 1000) //Every 24H
@PostConstruct
private void downloadIPLocationDatabase() throws Exception {
    if(config.getGeoIP().isEnabled()){
        logger.info("Downloading GEOIP Database from: "+config.getGeoIP().getUrl());

        File tmpGEOIPFileRound = File.createTempFile("geoIP", "mmdb");
        FileOutputStream fos = new FileOutputStream(tmpGEOIPFileRound);
        HttpURLConnection connection = getURLConnection(config.getGeoIP().getUrl());
        if (connection.getResponseCode() != 200) {
            return;
        }
        TarArchiveInputStream tarGzGeoIPStream = new TarArchiveInputStream(new GZIPInputStream(connection.getInputStream()));
        TarArchiveEntry entry= null;
        int offset;
        long pointer=0;
        while ((entry = tarGzGeoIPStream.getNextTarEntry()) != null) {
            pointer+=entry.getSize();
            if(entry.getName().endsWith("GeoLite2-City.mmdb")){
                byte[] content = new byte[(int) entry.getSize()];
                offset=0;
                //FileInputStream fis = new FileInputStream(entry.getFile());
                //IOUtils.copy(fis,fos);
                //tarGzGeoIPStream.skip(pointer);
                //tarGzGeoIPStream.read(content,offset,content.length-offset);
                //IOUtils.write(content,fos);
                int r;
                byte[] b = new byte[1024];
                while ((r = tarGzGeoIPStream.read(b)) != -1) {
                    fos.write(b, 0, r);
                }
                //fis.close();
                break;
            }
        }
        tarGzGeoIPStream.close();
        fos.flush();
        fos.close();
        connection.disconnect();
        geoIPDB = new DatabaseReader.Builder(tmpGEOIPFileRound).withCache(new CHMCache()).build();
        if (tmpGEOIPFile != null && tmpGEOIPFile.exists()) {
            tmpGEOIPFile.delete();
        }
        tmpGEOIPFile = tmpGEOIPFileRound;
    }
}
 

public GeoIpOperation setup(List<GeoProperty> geoProperties, boolean required)
    throws IOException {
  InputStream is = this.getClass().getResourceAsStream("my-ip-data.mmdb");
  DatabaseReader dr = new DatabaseReader.Builder(is).build();
  return new GeoIpOperation("ip_address", "geo_ip", dr, geoProperties, required);
}
 

public RawDBDemoGeoIPLocationService() throws IOException {
    File database = new File("your-path-to-db-file");
    dbReader = new DatabaseReader.Builder(database).build();
}