org.elasticsearch.common.xcontent.support.XContentMapValues#nodeStringValue

Source File: GitHubRiver.java From elasticsearch-river-github with Apache License 2.0

5 votes

@SuppressWarnings({"unchecked"})
@Inject
public GitHubRiver(RiverName riverName, RiverSettings settings, Client client) {
    super(riverName, settings);
    this.client = client;

    if (!settings.settings().containsKey("github")) {
        throw new IllegalArgumentException("Need river settings - owner and repository.");
    }

    // get settings
    Map<String, Object> githubSettings = (Map<String, Object>) settings.settings().get("github");
    owner = XContentMapValues.nodeStringValue(githubSettings.get("owner"), null);
    repository = XContentMapValues.nodeStringValue(githubSettings.get("repository"), null);

    index = String.format("%s&%s", owner, repository);
    userRequestedInterval = XContentMapValues.nodeIntegerValue(githubSettings.get("interval"), 60);

    // auth (optional)
    username = null;
    password = null;
    if (githubSettings.containsKey("authentication")) {
        Map<String, Object> auth = (Map<String, Object>) githubSettings.get("authentication");
        username = XContentMapValues.nodeStringValue(auth.get("username"), null);
        password = XContentMapValues.nodeStringValue(auth.get("password"), null);
    }

    // endpoint (optional - default to github.com)
    endpoint = XContentMapValues.nodeStringValue(githubSettings.get("endpoint"), "https://api.github.com");

    logger.info("Created GitHub river.");
}

Source File: LdapLoginSource.java From elasticsearch-imap with Apache License 2.0

5 votes

public LdapLoginSource(Map<String, Object> settings, String masterUser, String masterPassword) {
fUserNames = new ArrayList<>();
fUserPasswords = new ArrayList<>();

fMasterUser = masterUser;
fMasterPassword = masterPassword;

String url = XContentMapValues.nodeStringValue(settings.get("ldap_url"), null);
String base = XContentMapValues.nodeStringValue(settings.get("ldap_base"), null);
String user = XContentMapValues.nodeStringValue(settings.get("ldap_user"), null);
String password = XContentMapValues.nodeStringValue(settings.get("ldap_password"), null);

fConnector = new SimpleLdapConnector(url, base, user, password, true);

fNameField = XContentMapValues.nodeStringValue(settings.get("ldap_name_field"), DEF_USER_NAME_FIELD);
fPasswordField = XContentMapValues.nodeStringValue(settings.get("ldap_password_field"), DEF_USER_PW_FIELD);
fLdapFilter = fNameField + "=*";

fLogger = ESLoggerFactory.getLogger(LdapLoginSource.class.getName());

fLock = new Object();
fInitialized = false;

//start refreshing thread once initialized; interval in minutes
String refreshParam = XContentMapValues.nodeStringValue(settings.get("ldap_refresh_interval"), DEF_REFRESH_INT);
fRefreshInt = Long.parseLong(refreshParam) * 60000L;
if(fRefreshInt > 0) {
    //TODO: actually stop refreshing thread somehow
    fActive = true;
    Thread t = new Thread(this);
    t.setDaemon(true);
    t.start();
}
   }

Source File: IMAPImporter.java From elasticsearch-imap with Apache License 2.0

5 votes

@SuppressWarnings("unchecked")
private List<String> arrayNodeToList(Object arrayNode) {
    ArrayList<String> list = new ArrayList<>();
    if(XContentMapValues.isArray(arrayNode)) {
        for(Object node : (List<Object>) arrayNode) {
            String value = XContentMapValues.nodeStringValue(node, null);
            if(value != null) {
                list.add(value);
            }
        }
    }
    return list;
}

Source File: IMAPImporter.java From elasticsearch-imap with Apache License 2.0

5 votes

private void getUserLogins(final Map<String, Object> imapSettings) {
    String userSource = XContentMapValues.nodeStringValue(imapSettings.get("user_source"), null);
    ILoginSource source = null;

    if ("ldap".equals(userSource)) {
        //master user credentials for Dovecot
        String masterUser = XContentMapValues.nodeStringValue(imapSettings.get("master_user"), null);
        String masterPassword = XContentMapValues.nodeStringValue(imapSettings.get("master_password"), null);
        source = new LdapLoginSource(imapSettings, masterUser, masterPassword);
    } else {
        //read logins directly
        String _user = XContentMapValues.nodeStringValue(imapSettings.get("user"), null);
        String _password = XContentMapValues.nodeStringValue(imapSettings.get("password"), null);

        if (_user != null && !_user.isEmpty()) {
            users.add(_user);
            passwords.add(_password);
        }

        List<String> _users = arrayNodeToList(imapSettings.get("users"));
        List<String> _passwords = arrayNodeToList(imapSettings.get("passwords"));

        //TODO: inject master user credentials?
        if (_users != null && !_users.isEmpty()) {
            users.addAll(_users);
            passwords.addAll(_passwords);
        }
    }

    //read from generic source
    if (source != null) {
        users.addAll(source.getUserNames());
        passwords.addAll(source.getUserPasswords());
    }
}

Source File: HBaseRiver.java From Elasticsearch-HBase-River with Apache License 2.0

5 votes

/**
 * Fetch the value of a configuration that has a default value and is therefore optional.
 * 
 * @param config Key of the configuration to fetch
 * @param defaultValue The value to set if no value could be found
 * @return
 */
@SuppressWarnings({ "unchecked" })
private String readConfig(final String config, final String defaultValue) {
	if (this.settings.settings().containsKey(CONFIG_SPACE)) {
		Map<String, Object> mysqlSettings = (Map<String, Object>) this.settings.settings().get(CONFIG_SPACE);
		return XContentMapValues.nodeStringValue(mysqlSettings.get(config), defaultValue);
	}
	return defaultValue;
}

Source File: Neo4jDriver.java From elasticsearch-river-neo4j with Apache License 2.0

5 votes

@Inject
public Neo4jDriver(RiverName riverName, RiverSettings settings, @RiverIndexName final String riverIndexName, final Client client) {
    super(riverName, settings);
    this.client = client;

    uri = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("neo4j.uri", settings.settings()), DEFAULT_NEO_URI);
    List<Object> neo4jLabels = XContentMapValues.extractRawValues("neo4j.labels", settings.settings());
    String label;
    if(XContentMapValues.isArray(neo4jLabels)) {
        for (Object neo4jLabel : neo4jLabels) {
            label = XContentMapValues.nodeStringValue(neo4jLabel, null);
            labels.add(DynamicLabel.label(label));
        }
    }
    timestampField = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("neo4j.timestampField", settings.settings()), DEFAULT_NEO_TIMESTAMP_FIELD);
    interval = XContentMapValues.nodeIntegerValue(XContentMapValues.extractValue("neo4j.interval", settings.settings()), DEFAULT_NEO_INTERVAL);
    index = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("index.name", settings.settings()), DEFAULT_NEO_INDEX);
    type = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("index.type", settings.settings()), DEFAULT_NEO_TYPE);
    indexFromLabel = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("index.name.label",
        settings.settings()), null);
    typeFromLabel = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("index.type.label",
        settings.settings()), null);

    logger.debug("Neo4j settings [uri={}]", new Object[]{uri});
    logger.debug("River settings [indexName={}, type={}, interval={}, timestampField={}, indexLabel={}, " +
            "typelabel={}]",
        new Object[]{index,
            type,
            interval,
            timestampField,
            indexFromLabel,
            typeFromLabel}
    );

}

Source File: IsPrimeSearchScriptFactory.java From elasticsearch-native-script-example with Apache License 2.0

5 votes

/**
 * This method is called for every search on every shard.
 *
 * @param params list of script parameters passed with the query
 * @return new native script
 */
@Override
public ExecutableScript newScript(@Nullable Map<String, Object> params) {
    // Example of a mandatory string parameter
    // The XContentMapValues helper class can be used to simplify parameter parsing
    String fieldName = params == null ? null : XContentMapValues.nodeStringValue(params.get("field"), defaultFieldName);
    if (!Strings.hasLength(fieldName)) {
        throw new IllegalArgumentException("Missing the field parameter");
    }

    // Example of an optional integer  parameter
    int certainty = params == null ? 10 : XContentMapValues.nodeIntegerValue(params.get("certainty"), 10);
    return new IsPrimeSearchScript(fieldName, certainty);
}

Source File: FieldAliasMapper.java From crate with Apache License 2.0

5 votes

@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
    throws MapperParsingException {

    FieldAliasMapper.Builder builder = new FieldAliasMapper.Builder(name);
    Object pathField = node.remove(Names.PATH);
    String path = XContentMapValues.nodeStringValue(pathField, null);
    if (path == null) {
        throw new MapperParsingException("The [path] property must be specified for field [" + name + "].");
    }
    return builder.path(path);
}

Source File: KafkaRiverConfig.java From elasticsearch-river-kafka with Apache License 2.0

4 votes

public KafkaRiverConfig(RiverSettings settings)
{
	if (settings.settings().containsKey("kafka")) {
        Map<String, Object> kafkaSettings = (Map<String, Object>) settings.settings().get("kafka");
        
        topic = (String)kafkaSettings.get("topic");
        zookeeper = XContentMapValues.nodeStringValue(kafkaSettings.get("zookeeper"), "localhost");
        factoryClass = XContentMapValues.nodeStringValue(kafkaSettings.get("message_handler_factory_class"), "org.elasticsearch.river.kafka.JsonMessageHandlerFactory");
        brokerHost = XContentMapValues.nodeStringValue(kafkaSettings.get("broker_host"), "localhost");
        brokerPort = XContentMapValues.nodeIntegerValue(kafkaSettings.get("broker_port"), 9092);
        partition = XContentMapValues.nodeIntegerValue(kafkaSettings.get("partition"), 0);
    }
	else
	{
		zookeeper = "localhost";
		brokerHost = "localhost";
		brokerPort = 9092;
		topic = "default_topic";
		partition = 0;
		factoryClass = "org.elasticsearch.river.kafka.JsonMessageHandlerFactory";
	}
    
    if (settings.settings().containsKey("index")) {
        Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index");
        bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size_bytes"), 10*1024*1024);
        if (indexSettings.containsKey("bulk_timeout")) {
            bulkTimeout = TimeValue.parseTimeValue(XContentMapValues.nodeStringValue(indexSettings.get("bulk_timeout"), "10ms"), TimeValue.timeValueMillis(10000));
        } else {
            bulkTimeout = TimeValue.timeValueMillis(10);
        }
    } else {
        bulkSize = 10*1024*1024;
        bulkTimeout = TimeValue.timeValueMillis(10000);
    }
    
    if (settings.settings().containsKey("statsd")) {
        Map<String, Object> statsdSettings = (Map<String, Object>) settings.settings().get("statsd");
        statsdHost = (String)statsdSettings.get("host");
        statsdPort = XContentMapValues.nodeIntegerValue(statsdSettings.get("port"), 8125);
        statsdPrefix = XContentMapValues.nodeStringValue(statsdSettings.get("prefix"), "es-kafka-river");
    }
	else
	{
		statsdHost = null;
		statsdPort = -1;
		statsdPrefix = null;
	}
}

Source File: S3River.java From es-amazon-s3-river with Apache License 2.0

4 votes

@Inject
@SuppressWarnings({ "unchecked" })
protected S3River(RiverName riverName, RiverSettings settings, Client client, ThreadPool threadPool) throws Exception{
   super(riverName, settings);
   this.client = client;
   this.threadPool = threadPool;
   this.riverStatus = RiverStatus.UNKNOWN;
   
   // Deal with connector settings.
   if (settings.settings().containsKey("amazon-s3")){
      Map<String, Object> feed = (Map<String, Object>)settings.settings().get("amazon-s3");
      
      // Retrieve feed settings.
      String feedname = XContentMapValues.nodeStringValue(feed.get("name"), null);
      String bucket = XContentMapValues.nodeStringValue(feed.get("bucket"), null);
      String pathPrefix = XContentMapValues.nodeStringValue(feed.get("pathPrefix"), null);
      String downloadHost = XContentMapValues.nodeStringValue(feed.get("download_host"), null);
      int updateRate = XContentMapValues.nodeIntegerValue(feed.get("update_rate"), 15 * 60 * 1000);
      boolean jsonSupport = XContentMapValues.nodeBooleanValue(feed.get("json_support"), false);
      double indexedCharsRatio  = XContentMapValues.nodeDoubleValue(feed.get("indexed_chars_ratio"), 0.0);
      
      String[] includes = S3RiverUtil.buildArrayFromSettings(settings.settings(), "amazon-s3.includes");
      String[] excludes = S3RiverUtil.buildArrayFromSettings(settings.settings(), "amazon-s3.excludes");
      
      // Retrieve connection settings.
      String accessKey = XContentMapValues.nodeStringValue(feed.get("accessKey"), null);
      String secretKey = XContentMapValues.nodeStringValue(feed.get("secretKey"), null);
      boolean useIAMRoleForEC2 = XContentMapValues.nodeBooleanValue(feed.get("use_EC2_IAM"), false);
      
      feedDefinition = new S3RiverFeedDefinition(feedname, bucket, pathPrefix, downloadHost,
            updateRate, Arrays.asList(includes), Arrays.asList(excludes), accessKey, secretKey, useIAMRoleForEC2,
            jsonSupport, indexedCharsRatio);
   } else {
      logger.error("You didn't define the amazon-s3 settings. Exiting... See https://github.com/lbroudoux/es-amazon-s3-river");
      indexName = null;
      typeName = null;
      bulkSize = 100;
      feedDefinition = null;
      s3 = null;
      return;
   }
   
   // Deal with index settings if provided.
   if (settings.settings().containsKey("index")) {
      Map<String, Object> indexSettings = (Map<String, Object>)settings.settings().get("index");
      
      indexName = XContentMapValues.nodeStringValue(indexSettings.get("index"), riverName.name());
      typeName = XContentMapValues.nodeStringValue(indexSettings.get("type"), S3RiverUtil.INDEX_TYPE_DOC);
      bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size"), 100);
   } else {
      indexName = riverName.name();
      typeName = S3RiverUtil.INDEX_TYPE_DOC;
      bulkSize = 100;
   }
   
   // We need to connect to Amazon S3 after ensure mandatory settings are here.
   if (feedDefinition.getBucket() == null){
      logger.error("Amazon S3 bucket should not be null. Please fix this.");
      throw new IllegalArgumentException("Amazon S3 bucket should not be null.");
   }
   // Connect using the appropriate authentication process.
   if (feedDefinition.getAccessKey() == null && feedDefinition.getSecretKey() == null) {
      s3 = new S3Connector(feedDefinition.isUseIAMRoleForEC2());
   } else {
      s3 = new S3Connector(feedDefinition.getAccessKey(), feedDefinition.getSecretKey());
   }
   try {
      s3.connectUserBucket(feedDefinition.getBucket(), feedDefinition.getPathPrefix());
   } catch (AmazonS3Exception ase){
      logger.error("Exception while connecting Amazon S3 user bucket. "
            + "Either access key, secret key, IAM Role or bucket name are incorrect");
      throw ase;
   }

   this.riverStatus = RiverStatus.INITIALIZED;
}

Source File: RiverConfig.java From elasticsearch-river-kafka with Apache License 2.0

4 votes

public RiverConfig(RiverName riverName, RiverSettings riverSettings) {

        // Extract kafka related configuration
        if (riverSettings.settings().containsKey("kafka")) {
            Map<String, Object> kafkaSettings = (Map<String, Object>) riverSettings.settings().get("kafka");

            topic = (String) kafkaSettings.get(TOPIC);
            zookeeperConnect = XContentMapValues.nodeStringValue(kafkaSettings.get(ZOOKEEPER_CONNECT), "localhost");
            zookeeperConnectionTimeout = XContentMapValues.nodeIntegerValue(kafkaSettings.get(ZOOKEEPER_CONNECTION_TIMEOUT), 10000);
            messageType = MessageType.fromValue(XContentMapValues.nodeStringValue(kafkaSettings.get(MESSAGE_TYPE),
                    MessageType.JSON.toValue()));
        } else {
            zookeeperConnect = "localhost";
            zookeeperConnectionTimeout = 10000;
            topic = "elasticsearch-river-kafka";
            messageType = MessageType.JSON;
        }

        // Extract ElasticSearch related configuration
        if (riverSettings.settings().containsKey("index")) {
            Map<String, Object> indexSettings = (Map<String, Object>) riverSettings.settings().get("index");
            indexName = XContentMapValues.nodeStringValue(indexSettings.get(INDEX_NAME), riverName.name());
            typeName = XContentMapValues.nodeStringValue(indexSettings.get(MAPPING_TYPE), "status");
            bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get(BULK_SIZE), 100);
            concurrentRequests = XContentMapValues.nodeIntegerValue(indexSettings.get(CONCURRENT_REQUESTS), 1);
            actionType = ActionType.fromValue(XContentMapValues.nodeStringValue(indexSettings.get(ACTION_TYPE),
                    ActionType.INDEX.toValue()));
            flushInterval = TimeValue.parseTimeValue(XContentMapValues.nodeStringValue(indexSettings.get(FLUSH_INTERVAL), "12h"), FLUSH_12H);
        } else {
            indexName = riverName.name();
            typeName = "status";
            bulkSize = 100;
            concurrentRequests = 1;
            actionType = ActionType.INDEX;
            flushInterval = FLUSH_12H;
        }
        
        // Extract StatsD related configuration
        if (riverSettings.settings().containsKey("statsd")) {
            Map<String, Object> statsdSettings = (Map<String, Object>) riverSettings.settings().get("statsd");
            statsdHost = XContentMapValues.nodeStringValue(statsdSettings.get(STATSD_HOST), "localhost");
            statsdPrefix = XContentMapValues.nodeStringValue(statsdSettings.get(STATSD_PREFIX), "kafka_river");
            statsdPort = XContentMapValues.nodeIntegerValue(statsdSettings.get(STATSD_PORT), 8125);
            statsdIntervalInSeconds = XContentMapValues.nodeIntegerValue(statsdSettings.get(STATSD_INTERVAL_IN_SECONDS), 10);
        }
    }

Source File: AbstractIMAPRiverScenario.java From elasticsearch-imap with Apache License 2.0

3 votes

protected void renameScenarioIMAP(final String resource) throws Exception {

        Map<String, Object> settings = settings("/" + resource);

        final Properties props = new Properties();
        final String indexName = XContentMapValues.nodeStringValue(settings.get("mail_index_name"), "imapriver");
        final String typeName = XContentMapValues.nodeStringValue(settings.get("mail_type_name"), "mail");
        final String user = XContentMapValues.nodeStringValue(settings.get("user"), null);
        final String password = XContentMapValues.nodeStringValue(settings.get("password"), null);

        for (final Map.Entry<String, Object> entry : settings.entrySet()) {

            if (entry != null && entry.getKey().startsWith("mail.")) {
                props.setProperty(entry.getKey(), String.valueOf(entry.getValue()));
            }
        }

        createInitialIMAPTestdata(props, user, password, 10, true); // 3*10

        registerRiver("myrivunit", resource);

        Thread.sleep(1000);

        Thread.sleep(25 * 1000);

        Assert.assertEquals(30, getCount(indexName, typeName));

        renameMailbox(props, "ES-IMAP-RIVER-TESTS", user, password);

        Thread.sleep(25 * 1000);

        deleteMailsFromUserMailbox(props, "renamed_from_ES-IMAP-RIVER-TESTS", 4, 2, user, password); // delete
        // 2

        Thread.sleep(25 * 1000);

        Assert.assertEquals(28, getCount(indexName, typeName));

    }

Source File: AbstractIMAPRiverScenario.java From elasticsearch-imap with Apache License 2.0

2 votes

protected void deleteScenarioIMAP(final String resource, boolean keep_expunged) throws Exception {

        Map<String, Object> settings = settings("/" + resource);

        final Properties props = new Properties();
        final String indexName = XContentMapValues.nodeStringValue(settings.get("mail_index_name"), "imapriver");
        final String typeName = XContentMapValues.nodeStringValue(settings.get("mail_type_name"), "mail");
        final String user = XContentMapValues.nodeStringValue(settings.get("user"), null);
        final String password = XContentMapValues.nodeStringValue(settings.get("password"), null);

        for (final Map.Entry<String, Object> entry : settings.entrySet()) {

            if (entry != null && entry.getKey().startsWith("mail.")) {
                props.setProperty(entry.getKey(), String.valueOf(entry.getValue()));
            }
        }

        createInitialIMAPTestdata(props, user, password, 11, true); // 3*11
                                                                    // =
                                                                    // 33

        registerRiver("myrivunit", resource);

        Thread.sleep(1000);

        Thread.sleep(25 * 1000);

        Assert.assertEquals(33, getCount(indexName, typeName));

        deleteMailsFromUserMailbox(props, "INBOX", 5, 3, user, password); // delete
        // 3

        Thread.sleep(25 * 1000);

        Assert.assertEquals(keep_expunged?33:30, getCount(indexName, typeName));

        createInitialIMAPTestdata(props, user, password, 7, false); // 3*7=21

        Thread.sleep(25 * 1000);

        Assert.assertEquals(keep_expunged?54:51, getCount(indexName, typeName));
        deleteMailsFromUserMailbox(props, "INBOX", 4, 2, user, password); // delete
        // 2

        Thread.sleep(25 * 1000);

        Assert.assertEquals(keep_expunged?54:49, getCount(indexName, typeName));

    }

Source File: AbstractIMAPRiverScenario.java From elasticsearch-imap with Apache License 2.0

2 votes

protected void deleteScenarioPOP(final String resource) throws Exception {

        Map<String, Object> settings = settings("/" + resource);

        final Properties props = new Properties();
        final String indexName = XContentMapValues.nodeStringValue(settings.get("mail_index_name"), "imapriver");
        final String typeName = XContentMapValues.nodeStringValue(settings.get("mail_type_name"), "mail");
        final String user = XContentMapValues.nodeStringValue(settings.get("user"), null);
        final String password = XContentMapValues.nodeStringValue(settings.get("password"), null);

        for (final Map.Entry<String, Object> entry : settings.entrySet()) {

            if (entry != null && entry.getKey().startsWith("mail.")) {
                props.setProperty(entry.getKey(), String.valueOf(entry.getValue()));
            }
        }

        putMailInMailbox(10);

        registerRiver("myrivunit", resource);

        Thread.sleep(1000);

        Thread.sleep(5 * 1000);

        deleteMailsFromUserMailbox(props, "INBOX", 1, 3, user, password); // delete
        // 3

        Thread.sleep(5 * 1000);

        putMailInMailbox(5);

        Thread.sleep(5 * 1000);

        deleteMailsFromUserMailbox(props, "INBOX", 4, 2, user, password); // delete
        // 2

        Thread.sleep(5 * 1000);

        Assert.assertEquals(10, getCount(indexName, typeName));

    }

Java Code Examples for org.elasticsearch.common.xcontent.support.XContentMapValues#nodeStringValue()