Java Code Examples for org.elasticsearch.common.settings.Settings.Builder#put()

private Builder applySettings() {
    Builder settingsBuilder = Settings.builder();

    settingsBuilder.put("client.transport.ping_timeout", "15s");
    settingsBuilder.put("client.transport.nodes_sampler_interval", "5s");
    // YG: to ensure reliable connection & resolve NoNodeAvailableException
    settingsBuilder.put("client.transport.sniff", true);
    settingsBuilder.put("network.bind_host", 0);

    // YG: for supporting ES Auth with ES Shield
    Optional.ofNullable(config.getElastic().getAuth())
            .ifPresent(auth -> settingsBuilder.put("xpack.security.user", String.join(":", auth.getUser(), auth.getPwd())));

    if (Objects.nonNull(config.getElastic().getClusterName())) {
        settingsBuilder.put("cluster.name", config.getElastic().getClusterName());
    } else {
        settingsBuilder.put("client.transport.ignore_cluster_name", true);
    }
    return settingsBuilder;
}
 

public void testFailsNodeThatDoesNotRespond() {
    final Builder settingsBuilder = Settings.builder();
    if (randomBoolean()) {
        settingsBuilder.put(FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), randomIntBetween(1, 10));
    }
    if (randomBoolean()) {
        settingsBuilder.put(FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), randomIntBetween(100, 100000) + "ms");
    }
    if (randomBoolean()) {
        settingsBuilder.put(FOLLOWER_CHECK_TIMEOUT_SETTING.getKey(), randomIntBetween(1, 100000) + "ms");
    }
    final Settings settings = settingsBuilder.build();

    testBehaviourOfFailingNode(settings, () -> null,
        "followers check retry count exceeded",
        (FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings) - 1) * FOLLOWER_CHECK_INTERVAL_SETTING.get(settings).millis()
            + FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings) * FOLLOWER_CHECK_TIMEOUT_SETTING.get(settings).millis());
}
 

public void testFailsNodeThatRejectsCheck() {
    final Builder settingsBuilder = Settings.builder();
    if (randomBoolean()) {
        settingsBuilder.put(FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), randomIntBetween(1, 10));
    }
    if (randomBoolean()) {
        settingsBuilder.put(FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), randomIntBetween(100, 100000) + "ms");
    }
    final Settings settings = settingsBuilder.build();

    testBehaviourOfFailingNode(settings, () -> {
            throw new ElasticsearchException("simulated exception");
        },
        "followers check retry count exceeded",
        (FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings) - 1) * FOLLOWER_CHECK_INTERVAL_SETTING.get(settings).millis());
}
 

private Settings getSettings(int nodeOrdinal, long nodeSeed, Settings others) {
    Builder builder = Settings.builder().put(defaultSettings)
        .put(getRandomNodeSettings(nodeSeed));
    Settings settings = nodeConfigurationSource.nodeSettings(nodeOrdinal);
    if (settings != null) {
        if (settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey()) != null) {
            throw new IllegalStateException("Tests must not set a '" + ClusterName.CLUSTER_NAME_SETTING.getKey()
                    + "' as a node setting set '" + ClusterName.CLUSTER_NAME_SETTING.getKey() + "': ["
                    + settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey()) + "]");
        }
        builder.put(settings);
    }
    if (others != null) {
        builder.put(others);
    }
    builder.put(ClusterName.CLUSTER_NAME_SETTING.getKey(), clusterName);
    return builder.build();
}
 

@Test
public void completetlyInvalidConfigurationTest() throws Exception {
	Builder settingsBuilder = Settings.builder();
	settingsBuilder.put("opendistro_security.audit.type", TestAuditlogImpl.class.getName());
	settingsBuilder.put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_TRANSPORT_CATEGORIES, "nonexistent");
       settingsBuilder.put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_REST_CATEGORIES, "nonexistent");
	AuditLogImpl auditLog = new AuditLogImpl(settingsBuilder.build(), null, null, AbstractSecurityUnitTest.MOCK_POOL, null, cs);
	logAll(auditLog);

	auditLog.close();

	String result = TestAuditlogImpl.sb.toString();
	Assert.assertTrue(categoriesPresentInLog(result, filterComplianceCategories(Category.values())));

}
 

@Test
public void invalidConfigurationTest() {
	Builder settingsBuilder  = Settings.builder();
	settingsBuilder.put("opendistro_security.audit.type", "debug");
	settingsBuilder.put("opendistro_security.audit.config.disabled_categories", "nonexistant, bad_headers");
	AuditLog auditLog = new AuditLogImpl(settingsBuilder.build(), null, null, AbstractSecurityUnitTest.MOCK_POOL, null, cs);
	logAll(auditLog);
	String result = TestAuditlogImpl.sb.toString();
	Assert.assertFalse(categoriesPresentInLog(result, Category.BAD_HEADERS));
}
 

@Test
public void enableAllCategoryTest() throws Exception {
	final Builder settingsBuilder  = Settings.builder();

	settingsBuilder.put("opendistro_security.audit.type", TestAuditlogImpl.class.getName());
	settingsBuilder.put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_TRANSPORT_CATEGORIES, "NONE");
       settingsBuilder.put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_REST_CATEGORIES, "NONE");

	// we use the debug output, no ES client is needed. Also, we
	// do not need to close.
	AuditLogImpl auditLog = new AuditLogImpl(settingsBuilder.build(), null, null, AbstractSecurityUnitTest.MOCK_POOL, null, cs);

	logAll(auditLog);

	// we're using the ExecutorService in AuditLogImpl, so we need to wait
	// until all tasks are finished before we can check the result
	auditLog.close();

	String result = TestAuditlogImpl.sb.toString();

	Assert.assertTrue(Category.values()+"#"+result, categoriesPresentInLog(result, filterComplianceCategories(Category.values())));

	Assert.assertThat(result, containsString("testuser.transport.succeededlogin"));
	Assert.assertThat(result, containsString("testuser.rest.succeededlogin"));
	Assert.assertThat(result, containsString("testuser.rest.failedlogin"));
	Assert.assertThat(result, containsString("testuser.transport.failedlogin"));
	Assert.assertThat(result, containsString("privilege.missing"));
	Assert.assertThat(result, containsString("action.indexattempt"));
	Assert.assertThat(result, containsString("action.transport.ssl"));
	Assert.assertThat(result, containsString("action.success"));
	Assert.assertThat(result, containsString("Empty"));
}
 

protected void checkCategoriesDisabled(Category ... disabledCategories) throws Exception {

		List<String> categoryNames = new LinkedList<>();
		for (Category category : disabledCategories) {
			categoryNames.add(category.name().toLowerCase());
		}
		String disabledCategoriesString = Joiner.on(",").join(categoryNames);

		Builder settingsBuilder  = Settings.builder();
		settingsBuilder.put("opendistro_security.audit.type", TestAuditlogImpl.class.getName());
		settingsBuilder.put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_TRANSPORT_CATEGORIES, disabledCategoriesString);
        settingsBuilder.put(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_CONFIG_DISABLED_REST_CATEGORIES, disabledCategoriesString);


		// we use the debug output, no ES client is needed. Also, we
		// do not need to close.
		AuditLog auditLog = new AuditLogImpl(settingsBuilder.build(), null, null, AbstractSecurityUnitTest.MOCK_POOL, null, cs);

		logAll(auditLog);

		auditLog.close();

		String result = TestAuditlogImpl.sb.toString();

		List<Category> allButDisablesCategories = new LinkedList<>(Arrays.asList(Category.values()));
		allButDisablesCategories.removeAll(Arrays.asList(disabledCategories));

		System.out.println(result+"###"+disabledCategoriesString);
		Assert.assertFalse(categoriesPresentInLog(result, disabledCategories));
		Assert.assertTrue(categoriesPresentInLog(result, filterComplianceCategories(allButDisablesCategories.toArray(new Category[] {}))));
	}
 

private void testAuditType(String type, Class<? extends AuditLogSink> expectedClass) throws Exception {
		Builder settingsBuilder  = Settings.builder();
		settingsBuilder.put("opendistro_security.audit.type", type);
		settingsBuilder.put("path.home", ".");
		AuditLogImpl auditLog = new AuditLogImpl(settingsBuilder.build(), null, null, null, null, null);
		auditLog.close();
//		if (expectedClass != null) {
//		    Assert.assertNotNull("delegate is null for type: "+type,auditLog.delegate);
//			Assert.assertEquals(expectedClass, auditLog.delegate.getClass());
//		} else {
//			Assert.assertNull(auditLog.delegate);
//		}

	}
 

public void testAutoShrinking() {
    final String[] allNodes = new String[]{"a", "b", "c", "d", "e", "f", "g"};

    final String[] liveNodes = new String[randomIntBetween(1, allNodes.length)];
    randomSubsetOf(liveNodes.length, allNodes).toArray(liveNodes);

    final String[] initialVotingNodes = new String[randomIntBetween(1, allNodes.length)];
    randomSubsetOf(initialVotingNodes.length, allNodes).toArray(initialVotingNodes);

    final Builder settingsBuilder = Settings.builder();
    if (randomBoolean()) {
        settingsBuilder.put(CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION.getKey(), true);
    }
    final Reconfigurator reconfigurator = makeReconfigurator(settingsBuilder.build());
    final Set<DiscoveryNode> liveNodesSet = nodes(liveNodes);
    final VotingConfiguration initialConfig = conf(initialVotingNodes);

    final int quorumSize = Math.max(liveNodes.length / 2 + 1, initialVotingNodes.length < 3 ? 1 : 2);

    final VotingConfiguration finalConfig = reconfigurator.reconfigure(liveNodesSet, emptySet(),
        randomFrom(liveNodesSet), initialConfig);

    final String description = "reconfigure " + liveNodesSet + " from " + initialConfig + " yielded " + finalConfig;

    if (quorumSize > liveNodes.length) {
        assertFalse(description + " without a live quorum", finalConfig.hasQuorum(Arrays.asList(liveNodes)));
    } else {
        final List<String> expectedQuorum = randomSubsetOf(quorumSize, liveNodes);
        assertTrue(description + " with quorum[" + quorumSize + "] of " + expectedQuorum, finalConfig.hasQuorum(expectedQuorum));
    }
}
 

public void testFailureCounterResetsOnSuccess() {
    final Builder settingsBuilder = Settings.builder();
    if (randomBoolean()) {
        settingsBuilder.put(FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), randomIntBetween(2, 10));
    }
    if (randomBoolean()) {
        settingsBuilder.put(FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), randomIntBetween(100, 100000) + "ms");
    }
    final Settings settings = settingsBuilder.build();

    final int retryCount = FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings);
    final int maxRecoveries = randomIntBetween(3, 10);

    // passes just enough checks to keep it alive, up to maxRecoveries, and then fails completely
    testBehaviourOfFailingNode(settings, new Supplier<Empty>() {
            private int checkIndex;
            private int recoveries;

            @Override
            public Empty get() {
                checkIndex++;
                if (checkIndex % retryCount == 0 && recoveries < maxRecoveries) {
                    recoveries++;
                    return Empty.INSTANCE;
                }
                throw new ElasticsearchException("simulated exception");
            }
        },
        "followers check retry count exceeded",
        (FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings) * (maxRecoveries + 1) - 1)
            * FOLLOWER_CHECK_INTERVAL_SETTING.get(settings).millis());
}
 

@Test
public void testTlsConfigurationNoFallback() throws Exception {

	TestHttpHandler handler = new TestHttpHandler();

	server = ServerBootstrap.bootstrap().setListenerPort(8083).setServerInfo("Test/1.1").setSslContext(createSSLContext()).registerHandler("*", handler).create();

	server.start();

	Builder builder = Settings.builder().loadFromPath(FileHelper.getAbsoluteFilePathFromClassPath("auditlog/endpoints/sink/configuration_tls.yml"));
	builder.put("path.home", "/");

	// replace some values with absolute paths for unit tests
	builder.put("opendistro_security.audit.config.webhook.ssl.pemtrustedcas_filepath", FileHelper.getAbsoluteFilePathFromClassPath("auditlog/root-ca.pem"));
	builder.put("opendistro_security.audit.endpoints.endpoint1.config.webhook.ssl.pemtrustedcas_filepath", FileHelper.getAbsoluteFilePathFromClassPath("auditlog/root-ca.pem"));
	builder.put("opendistro_security.audit.endpoints.endpoint2.config.webhook.ssl.pemtrustedcas_content", FileHelper.loadFile("auditlog/root-ca.pem"));

	SinkProvider provider = new SinkProvider(builder.build(), null, null, null);
	WebhookSink defaultSink = (WebhookSink) provider.defaultSink;
	Assert.assertEquals(true, defaultSink.verifySSL);

	AuditMessage msg = MockAuditMessageFactory.validAuditMessage();
	provider.allSinks.get("endpoint1").store(msg);

	Assert.assertTrue(handler.method.equals("POST"));
       Assert.assertTrue(handler.body != null);
       Assert.assertTrue(handler.body.contains("{"));
       assertStringContainsAllKeysAndValues(handler.body);

	handler.reset();

	provider.allSinks.get("endpoint2").store(msg);

	Assert.assertTrue(handler.method.equals("POST"));
       Assert.assertTrue(handler.body != null);
       Assert.assertTrue(handler.body.contains("{"));
       assertStringContainsAllKeysAndValues(handler.body);

	handler.reset();

	provider.defaultSink.store(msg);

	Assert.assertTrue(handler.method.equals("POST"));
       Assert.assertTrue(handler.body != null);
       Assert.assertTrue(handler.body.contains("{"));
       assertStringContainsAllKeysAndValues(handler.body);

       server.stop();
}
 

public static void main(String args[]) throws Exception {
    if (args.length != 2) {
        System.out.printf("Usage: java KibanaExporter.jar %s %s%n", "localhost:9300", "my-cluster-name");
        System.exit(0);
    }
    if (!args[0].contains(":")) {
        throw new IllegalArgumentException(
                String.format("%s not a valid format. Expected <hostname>:<port>.", args[0]));
    }

    // set ES address
    String split[] = args[0].split(":");

    // set cluster name
    Builder tcSettings = Settings.builder();
    tcSettings.put("cluster.name", args[1]);
    System.out.println("Connection to " + args[1]);

    client = new RestHighLevelClient(RestClient
            .builder(new HttpHost(InetAddress.getByName(split[0]), Integer.parseInt(split[1], 10), "http")));

    // search index pattern for needle
    searchIndexPattern();

    KibanaConfigHolderDto holder = new KibanaConfigHolderDto();
    System.out.println("Reading .kibana index");

    SearchResponse resp = client.search(new SearchRequest(".kibana").source(new SearchSourceBuilder().size(1000)),
            RequestOptions.DEFAULT);
    Arrays.asList(resp.getHits().getHits()).stream().map(KibanaExporter::parseSearchHit).forEach(holder::add);
    System.out.println("Reading finished");

    ObjectMapper mapper = new ObjectMapper();
    // we love pretty things
    mapper.enable(SerializationFeature.INDENT_OUTPUT);
    File f = new File("kibana_config.json");

    try (FileOutputStream out = new FileOutputStream(f, false)) {
        mapper.writeValue(out, holder);
    }

    System.out.println("File outputted to: " + f.getAbsolutePath());

    client.close();

}
 

public void testRetriesOnCorrectSchedule() {
    final Builder settingsBuilder = Settings.builder();

    final long initialTimeoutMillis;
    if (randomBoolean()) {
        initialTimeoutMillis = randomLongBetween(1, 10000);
        settingsBuilder.put(ELECTION_INITIAL_TIMEOUT_SETTING.getKey(), initialTimeoutMillis + "ms");
    } else {
        initialTimeoutMillis = ELECTION_INITIAL_TIMEOUT_SETTING.get(Settings.EMPTY).millis();
    }

    if (randomBoolean()) {
        settingsBuilder.put(ELECTION_BACK_OFF_TIME_SETTING.getKey(), randomLongBetween(1, 60000) + "ms");
    }

    if (ELECTION_MAX_TIMEOUT_SETTING.get(Settings.EMPTY).millis() < initialTimeoutMillis || randomBoolean()) {
        settingsBuilder.put(ELECTION_MAX_TIMEOUT_SETTING.getKey(),
            randomLongBetween(Math.max(200, initialTimeoutMillis), 180000) + "ms");
    }

    final long electionDurationMillis;
    if (randomBoolean()) {
        electionDurationMillis = randomLongBetween(1, 300000);
        settingsBuilder.put(ELECTION_DURATION_SETTING.getKey(), electionDurationMillis + "ms");
    } else {
        electionDurationMillis = ELECTION_DURATION_SETTING.get(Settings.EMPTY).millis();
    }

    final Settings settings = settingsBuilder.put(NODE_NAME_SETTING.getKey(), "node").build();
    final long initialTimeout = ELECTION_INITIAL_TIMEOUT_SETTING.get(settings).millis();
    final long backOffTime = ELECTION_BACK_OFF_TIME_SETTING.get(settings).millis();
    final long maxTimeout = ELECTION_MAX_TIMEOUT_SETTING.get(settings).millis();
    final long duration = ELECTION_DURATION_SETTING.get(settings).millis();

    final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(settings, random());
    final ElectionSchedulerFactory electionSchedulerFactory
        = new ElectionSchedulerFactory(settings, random(), deterministicTaskQueue.getThreadPool());

    assertElectionSchedule(deterministicTaskQueue, electionSchedulerFactory, initialTimeout, backOffTime, maxTimeout, duration);

    // do it again to show that the max is reset when the scheduler is restarted
    assertElectionSchedule(deterministicTaskQueue, electionSchedulerFactory, initialTimeout, backOffTime, maxTimeout, duration);
}
 

private static Settings getRandomNodeSettings(long seed) {
    Random random = new Random(seed);
    Builder builder = Settings.builder();
    builder.put(TransportSettings.TRANSPORT_COMPRESS.getKey(), rarely(random));
    if (random.nextBoolean()) {
        builder.put("cache.recycler.page.type", RandomPicks.randomFrom(random, PageCacheRecycler.Type.values()));
    }

    builder.put(EsExecutors.PROCESSORS_SETTING.getKey(), 1 + random.nextInt(3));

    // randomize tcp settings
    if (random.nextBoolean()) {
        builder.put(TransportSettings.CONNECTIONS_PER_NODE_RECOVERY.getKey(), random.nextInt(2) + 1);
        builder.put(TransportSettings.CONNECTIONS_PER_NODE_BULK.getKey(), random.nextInt(3) + 1);
        builder.put(TransportSettings.CONNECTIONS_PER_NODE_REG.getKey(), random.nextInt(6) + 1);
    }

    if (random.nextBoolean()) {
        builder.put(MappingUpdatedAction.INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING.getKey(),
                timeValueSeconds(RandomNumbers.randomIntBetween(random, 10, 30)).getStringRep());
    }

    if (random.nextInt(10) == 0) {
        builder.put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_TYPE_SETTING.getKey(), "noop");
        builder.put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_TYPE_SETTING.getKey(), "noop");
    }

    if (random.nextBoolean()) {
        if (random.nextInt(10) == 0) { // do something crazy slow here
            builder.put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(),
                    new ByteSizeValue(RandomNumbers.randomIntBetween(random, 1, 10), ByteSizeUnit.MB));
        } else {
            builder.put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(),
                    new ByteSizeValue(RandomNumbers.randomIntBetween(random, 10, 200), ByteSizeUnit.MB));
        }
    }

    if (random.nextBoolean()) {
        builder.put(TransportSettings.PING_SCHEDULE.getKey(), RandomNumbers.randomIntBetween(random, 100, 2000) + "ms");
    }

    return builder.build();
}