com.containersol.minimesos.cluster.MesosCluster Java Examples

@Test
public void testInheritedLoggingLevel() {
    String config = "minimesos { \n" +
            "loggingLevel = \"warning\" \n" +
            "agent { loggingLevel = \"ERROR\"} \n" +
            "agent {} \n" +
            "}";

    ConfigParser parser = new ConfigParser();
    ClusterConfig dsl = parser.parse(config);

    MesosCluster cluster = new MesosClusterContainersFactory().createMesosCluster(dsl);

    List<MesosAgentContainer> agents = CollectionsUtils.typedList(cluster.getAgents(), MesosAgentContainer.class);
    assertEquals(2, agents.size());

    assertEquals("ERROR", agents.get(0).getLoggingLevel());
    assertEquals("WARNING", agents.get(1).getLoggingLevel());
}
 

@BeforeClass
public static void startCluster() {
    new DockerUtil(dockerClient).killAllSchedulers();
    new DockerUtil(dockerClient).killAllExecutors();

    final AlpineContainer ymlWrite = new AlpineContainer(dockerClient, CUSTOM_CONFIG_PATH, CUSTOM_CONFIG_PATH,
            "sh", "-c", "echo \"index.auto_expand_replicas: " + TEST_AUTO_EXPAND_REPLICAS + "\npath.plugins: " + TEST_PATH_PLUGINS + "\" > " + CUSTOM_CONFIG_FILE);
    ymlWrite.start(TEST_CONFIG.getClusterTimeout());
    ymlWrite.remove();

    MesosCluster mesosCluster = factory.createMesosCluster("src/main/resources/testMinimesosFile");
    mesosCluster.setMapPortsToHost(true);

    cluster.start(TEST_CONFIG.getClusterTimeout());

    // Make sure all tasks are running before we test.
    esTasks = new ESTasks(TEST_CONFIG, scheduler.getIpAddress());
    new TasksResponse(esTasks, NUMBER_OF_TEST_TASKS);

    nodesResponse = new ElasticsearchNodesResponse(esTasks, NUMBER_OF_TEST_TASKS);
}
 

/**
 * Replaces ${MINIMESOS_[ROLE]}, ${MINIMESOS_[ROLE]_IP} and ${MINIMESOS_[ROLE]_PORT} tokens in the given string with actual values.
 * Also supports ${NETWORK_GATEWAY}
 *
 * @param source string to replace values in
 * @return updated string
 */
public String replaceTokens(String source) {
    MesosCluster cluster = getCluster();
    // received JSON might contain tokens, which should be replaced before the installation
    List<ClusterProcess> uniqueRoles = ClusterUtil.getDistinctRoleProcesses(cluster.getMemberProcesses());
    String updatedJson = source;
    for (ClusterProcess process : uniqueRoles) {
        URI serviceUri = process.getServiceUrl();
        if (serviceUri != null) {
            updatedJson = replaceToken(updatedJson, MesosCluster.MINIMESOS_TOKEN_PREFIX + process.getRole().toUpperCase(), serviceUri.toString());
            updatedJson = replaceToken(updatedJson, MesosCluster.MINIMESOS_TOKEN_PREFIX + process.getRole().toUpperCase() + "_IP", serviceUri.getHost());
            updatedJson = replaceToken(updatedJson, MesosCluster.MINIMESOS_TOKEN_PREFIX + process.getRole().toUpperCase() + "_PORT", Integer.toString(serviceUri.getPort()));
        }
    }

    // replace independent from roles tokens
    String masterContainer = cluster.getMaster().getContainerId();
    updatedJson = replaceToken(updatedJson, MesosCluster.TOKEN_NETWORK_GATEWAY, DockerContainersUtil.getGatewayIpAddress(masterContainer));
    updatedJson = replaceToken(updatedJson, TOKEN_HOST_DIR, MesosCluster.getClusterHostDir().getAbsolutePath());

    return updatedJson;
}
 

@Test
public void testState() throws IOException {
    CommandUp commandUp = new CommandUp();
    commandUp.setClusterConfigPath("src/integration-test/resources/configFiles/complete-minimesosFile");
    commandUp.execute();
    MesosCluster cluster = commandUp.getCluster();

    CommandState commandState = new CommandState(ps);
    commandState.execute();

    JSONObject state = new JSONObject(outputStream.toString("UTF-8"));

    assertEquals("master@" + cluster.getMaster().getIpAddress() + ":5050", state.getString("leader"));

    cluster.destroy(new MesosClusterContainersFactory());
}
 

@Test
public void testUp_invalidMinimesosFile() throws IOException {
    FileUtils.write(repository.getMinimesosFile(), "invalid", "UTF-8");

    CommandUp commandUp = new CommandUp();
    commandUp.setClusterConfigPath("src/integration-test/resources/configFiles/complete-minimesosFile");
    commandUp.execute();
    MesosCluster cluster = commandUp.getCluster();

    String fileContent = FileUtils.readFileToString(repository.getMinimesosFile(), "UTF-8");
    assertEquals("Invalid state file has not been overwritten", cluster.getClusterId(), fileContent);

    cluster.destroy(new MesosClusterContainersFactory());

    File minimesosFile = repository.getMinimesosFile();

    assertFalse("Minimesos file at " + minimesosFile + " should be removed", minimesosFile.exists());
}
 

@Before
public void initTest() {
    outputStream = new ByteArrayOutputStream();
    ps = new PrintStream(outputStream, true);

    marathon = Mockito.mock(Marathon.class);

    mesosCluster = Mockito.mock(MesosCluster.class);
    when(mesosCluster.getMarathon()).thenReturn(marathon);

    repository = Mockito.mock(ClusterRepository.class);
    when(repository.loadCluster(Matchers.any(MesosClusterFactory.class))).thenReturn(mesosCluster);

    commandUninstall = new CommandUninstall(ps);
    commandUninstall.setRepository(repository);
}
 

@Before
public void initTest() throws URISyntaxException {
    outputStream = new ByteArrayOutputStream();
    ps = new PrintStream(outputStream, true);

    masterState = generateMasterState();
    agentState = generateAgentState();

    MesosMasterContainer master = mock(MesosMasterContainer.class);
    when(master.getState()).thenReturn(masterState);

    MesosAgentContainer agent = mock(MesosAgentContainer.class);
    when(agent.getState()).thenReturn(agentState);
    when(agent.getServiceUrl()).thenReturn(new URI(agentServiceURL));

    MesosCluster mesosCluster = mock(MesosCluster.class);
    when(mesosCluster.getMaster()).thenReturn(master);
    when(mesosCluster.getAgents()).thenReturn(Collections.singletonList(agent));

    repository = mock(ClusterRepository.class);
    when(repository.loadCluster(any(MesosClusterFactory.class))).thenReturn(mesosCluster);

    downloader = mock(Downloader.class);
    when(downloader.getFileContentAsString(STDOUT_URL)).thenReturn("stdout file content");
    when(downloader.getFileContentAsString(STDERR_URL)).thenReturn("stderr file content");
}
 

@Test(expected = MinimesosException.class)
public void testExecute_existingMiniMesosFile() throws IOException {
    String oldHostDir = System.getProperty(MesosCluster.MINIMESOS_HOST_DIR_PROPERTY);
    File dir = File.createTempFile("mimimesos-test", "dir");
    assertTrue("Failed to delete temp file", dir.delete());
    assertTrue("Failed to create temp directory", dir.mkdir());
    System.setProperty(MesosCluster.MINIMESOS_HOST_DIR_PROPERTY, dir.getAbsolutePath());


    File minimesosFile = new File(dir, ClusterConfig.DEFAULT_CONFIG_FILE);
    Files.write(Paths.get(minimesosFile.getAbsolutePath()), "minimesos { }".getBytes());

    try {
        commandInit.execute();
    } finally {
        if (oldHostDir == null) {
            System.getProperties().remove(MesosCluster.MINIMESOS_HOST_DIR_PROPERTY);
        } else {
            System.setProperty(MesosCluster.MINIMESOS_HOST_DIR_PROPERTY, oldHostDir);
        }
        FileUtils.forceDelete(dir);
    }
}
 

/**
 * Prints cluster services URLs and IPs
 *
 * @param cluster to examine
 */
private void printServiceUrls(MesosCluster cluster) {

    // print independent from roles variables
    String masterContainer = cluster.getMaster().getContainerId();
    String gateway = String.format("export %s=%s", MesosCluster.TOKEN_NETWORK_GATEWAY, DockerContainersUtil.getGatewayIpAddress(masterContainer));
    output.println(gateway);

    List<ClusterProcess> uniqueMembers = ClusterUtil.getDistinctRoleProcesses(cluster.getMemberProcesses());
    for (ClusterProcess process : uniqueMembers) {

        URI serviceUrl = process.getServiceUrl();
        if (serviceUrl != null) {
            String service = String.format("export %s%s=%s", MesosCluster.MINIMESOS_TOKEN_PREFIX, process.getRole().toUpperCase(), serviceUrl.toString());
            String serviceIp = String.format("export %s%s_IP=%s", MesosCluster.MINIMESOS_TOKEN_PREFIX, process.getRole().toUpperCase(), serviceUrl.getHost());

            output.println(String.format("%s; %s", service, serviceIp));
        }

    }

    if (Environment.isRunningInDockerOnMac()) {
        output.println("You are running Docker on Mac so use localhost instead of container IPs for Master, Marathon, Zookeepr and Consul");
    }
}
 

@Override
public void execute() {
    String clusterId = repository.readClusterId();
    if (clusterId != null) {
        MesosCluster cluster = repository.loadCluster(new MesosClusterContainersFactory());
        if (cluster != null) {
            output.println("Minimesos cluster is running: " + cluster.getClusterId());
            output.println("Mesos version: " + cluster.getMaster().getState().getVersion());
            printServiceUrls(cluster);

            MesosDns mesosDns = cluster.getMesosDns();
            if (mesosDns != null) {
                output.println("Running dnsmasq? Add 'server=/mm/" + mesosDns.getIpAddress() + "#53' to /etc/dnsmasq.d/10-minimesos to resolve master.mm, zookeeper.mm and Marathon apps on app.marathon.mm.");
            }
        } else {
            output.println(String.format("Minimesos cluster %s is not running. %s is removed", clusterId, repository.getMinimesosFile().getAbsolutePath()));
        }
    } else {
        output.println("Cluster ID is not found in " + repository.getMinimesosFile().getAbsolutePath());
    }
}
 

@Override
public void execute() {
    LOGGER.debug("Executing up command");

    MesosCluster cluster = getCluster();
    if (cluster != null) {
        output.println("Cluster " + cluster.getClusterId() + " is already running");
        return;
    }
    ClusterConfig clusterConfig = readClusterConfigFromMinimesosFile();
    updateWithParameters(clusterConfig);

    startedCluster = mesosClusterFactory.createMesosCluster(clusterConfig);
    // save cluster ID first, so it becomes available for 'destroy' even if its part failed to start
    repository.saveClusterFile(startedCluster);

    startedCluster.start();
    startedCluster.waitForState(state -> state != null);

    new CommandInfo(output).execute();
}
 

@Test
public void testInheritedImageTag() {
    String config = "minimesos { \n" +
            "mesosVersion = \"0.26\" \n" +
            "agent { imageTag = \"0.27.0-0.1.0\"} \n" +
            "agent {} \n" +
            "}";

    ConfigParser parser = new ConfigParser();
    ClusterConfig dsl = parser.parse(config);

    MesosCluster cluster = new MesosClusterContainersFactory().createMesosCluster(dsl);

    List<MesosAgentContainer> agents = CollectionsUtils.typedList(cluster.getAgents(), MesosAgentContainer.class);
    assertEquals(2, agents.size());

    assertEquals("0.27.0-" + ClusterConfig.DEFAULT_MINIMESOS_DOCKER_VERSION, agents.get(0).getImageTag());
    assertEquals("0.26-" + ClusterConfig.DEFAULT_MINIMESOS_DOCKER_VERSION, agents.get(1).getImageTag());
}
 

@Override
public void execute() {
    MesosCluster cluster = repository.loadCluster(new MesosClusterContainersFactory());

    if (cluster == null) {
        output.println("Minimesos cluster is not running");
        return;
    }

    output.printf(FORMAT, COLUMNS);
    State state = cluster.getMaster().getState();
    for (Framework framework : state.getFrameworks()) {
        for (Task task : framework.getTasks()) {
            output.printf(FORMAT, framework.getName(), task.getName(), task.getState(), task.getDiscovery().getPorts().getPorts().get(0).getNumber());
        }
    }
}
 

@Test
public void testDefaultInAgentLoggingLevel() {
    String config = "minimesos { \n" +
            "loggingLevel = \"warning\" \n" +
            "agent { loggingLevel = \"ERROR\" } \n" +
            "agent { loggingLevel = \"INFO\" } \n" +
            "}";

    ConfigParser parser = new ConfigParser();
    ClusterConfig dsl = parser.parse(config);

    MesosCluster cluster = new MesosClusterContainersFactory().createMesosCluster(dsl);

    List<MesosAgentContainer> agents = CollectionsUtils.typedList(cluster.getAgents(), MesosAgentContainer.class);
    assertEquals(2, agents.size());

    assertEquals("ERROR", agents.get(0).getLoggingLevel());
    assertEquals("INFO", agents.get(1).getLoggingLevel());
}
 

public static void main(String[] args) throws InterruptedException {
        DockerClient dockerClient = DockerClientFactory.build();

        MesosCluster cluster = new MesosCluster(ClusterUtil.withAgent(1, zooKeeper -> new LogstashMesosSlave(dockerClient, zooKeeper)).withMaster().build());

        cluster.start();

/*
        LOGGER.info("Starting scheduler");
        LogstashSchedulerContainer scheduler = new LogstashSchedulerContainer(dockerClient, cluster.getZkContainer().getIpAddress());
        scheduler.start();
        LOGGER.info("Scheduler started at http://" + scheduler.getIpAddress() + ":9092");
*/

        LOGGER.info("Type CTRL-C to quit");
        while (true) {
            Thread.sleep(1000);
        }
    }
 

@Before
public void before() {
    dockerUtil.killAllSchedulers();
    dockerUtil.killAllExecutors();

    MesosCluster mesosCluster = factory.createMesosCluster("src/main/resources/testMinimesosFile");
    mesosCluster.setMapPortsToHost(true);

    cluster.start(TEST_CONFIG.getClusterTimeout());
}
 

public static void apply(DockerClient client, MesosCluster cluster, Configuration config) {
    // Install IP tables and reroute traffic from slaves to ports exposed on host.
    for (MesosAgent slave : cluster.getAgents()) {
        LOGGER.debug("Applying iptable redirect to " + slave.getIpAddress());
        Awaitility.await().pollInterval(1L, TimeUnit.SECONDS).atMost(30, TimeUnit.SECONDS).until(new IpTables(client, config, slave.getContainerId()));
    }
}
 

@Before
public void before() {
    marathon = mock(Marathon.class);

    mesosCluster = mock(MesosCluster.class);
    when(mesosCluster.getMarathon()).thenReturn(marathon);

    repository = mock(ClusterRepository.class);
    when(repository.loadCluster(any())).thenReturn(mesosCluster);

    command = new CommandInstall();
    command.repository = repository;
}
 

public MesosCluster createMesosCluster(String path) {
    try (InputStream is = new FileInputStream(path)) {
        return createMesosCluster(is);
    } catch (IOException e) {
        LOGGER.debug("Could not read minimesos config: ", e.getMessage());
        throw new MinimesosException("Could not read minimesos config: " + e.getMessage());
    }
}
 

public MesosCluster createMesosCluster(InputStream inputStream) {
    try {
        ClusterConfig clusterConfig = new ConfigParser().parse(IOUtils.toString(inputStream, "UTF-8"));
        return createMesosCluster(clusterConfig);
    } catch (IOException e) {
        throw new MinimesosException("Could not read minimesos config:" + e.getCause());
    }
}
 

@Override
public void execute() {
    MesosCluster cluster = repository.loadCluster(new MesosClusterContainersFactory());
    if (cluster != null) {
        cluster.state(output);
    } else {
        output.println("Minimesos cluster is not running");
    }
}
 

/**
 * Reads ClusterConfig from minimesosFile.
 *
 * @return configuration of the cluster from the minimesosFile
 * @throws MinimesosException if minimesosFile is not found or malformed
 */
public ClusterConfig readClusterConfigFromMinimesosFile() {
    InputStream clusterConfigFile = MesosCluster.getInputStream(getClusterConfigPath());
    if (clusterConfigFile != null) {
        ConfigParser configParser = new ConfigParser();
        try {
            return configParser.parse(IOUtils.toString(clusterConfigFile, "UTF-8"));
        } catch (Exception e) {
            String msg = String.format("Failed to load cluster configuration from %s: %s", getClusterConfigPath(), e.getMessage());
            throw new MinimesosException(msg, e);
        }
    }
    throw new MinimesosException("No minimesosFile found in current directory. Please generate one with 'minimesos init'");
}
 

public static MesosClusterTestRule fromFile(String minimesosFilePath) {
    try {
        MesosCluster cluster = new MesosClusterContainersFactory().createMesosCluster(new FileInputStream(minimesosFilePath));
        return new MesosClusterTestRule(cluster);
    } catch (FileNotFoundException e) {
        throw new MinimesosException("Could not read minimesosFile at " + minimesosFilePath, e);
    }
}
 

private int handleNoCommand() {
    MesosCluster cluster = repository.loadCluster(new MesosClusterContainersFactory());
    if (cluster != null) {
        new CommandInfo().execute();
        return EXIT_CODE_OK;
    } else {
        printUsage(null);
        return EXIT_CODE_ERR;
    }
}
 

private MesosAgent findAgent(MesosCluster cluster, String slaveId) {
    for (MesosAgent agent : cluster.getAgents()) {
        State agentState = agent.getState();
        if (agentState.getId().equals(slaveId)) {
            return agent;
        }
    }
    return null;
}
 

@Test
public void testMesosVersionRestored() {
    String clusterId = CLUSTER.getClusterId();
    MesosCluster cluster = MesosCluster.loadCluster(clusterId, new MesosClusterContainersFactory());

    Assert.assertEquals("1.0.0", cluster.getConfiguredMesosVersion());
}
 

@Test
public void testCreateMesosCluster() throws FileNotFoundException {
    MesosCluster mesosCluster = new MesosClusterContainersFactory().createMesosCluster(new FileInputStream("src/test/resources/configFiles/minimesosFile-mesosClusterTest"));
    assertEquals(3 , mesosCluster.getAgents().size());
    assertNotNull(mesosCluster.getZooKeeper());
    assertNotNull(mesosCluster.getMaster());
}
 

public static MesosClusterTestRule fromClassPath(String path) {
    try (InputStream is = MesosClusterTestRule.class.getResourceAsStream(path)) {
        MesosCluster cluster = new MesosClusterContainersFactory().createMesosCluster(is);
        return new MesosClusterTestRule(cluster);
    } catch (IOException e) {
        throw new MinimesosException("Could not read minimesosFile on classpath " + path, e);
    }
}
 

@Override
public void execute() {
    MesosCluster cluster = repository.loadCluster(new MesosClusterContainersFactory());
    if (cluster == null) {
        output.println("Minimesos cluster is not running");
        return;
    }

    State masterState = cluster.getMaster().getState();
    Task task = findTask(masterState, taskId);
    if (task == null) {
        output.println(String.format("Cannot find task: '%s'", taskId));
        return;
    }

    MesosAgent agent = findAgent(cluster, task.getSlaveId());
    if (agent == null) {
        output.println(String.format("Cannot find agent: '%s'", task.getSlaveId()));
        return;
    }


    String filename = stderr ? "stderr" : "stdout";
    output.println(String.format("[minimesos] Fetching '%s' of task '%s'\n", filename, task.getId()));
    URI fileUrl = getFileUrl(agent, task, filename);
    String content = downloader.getFileContentAsString(fileUrl.toString());
    output.println(content);
}
 

private void restoreMapToPorts(MesosCluster cluster, Container container) {
    // Restore "map ports to host" attribute
    ContainerPort[] ports = container.getPorts();
    if (ports != null) {
        for (ContainerPort port : ports) {
            if (port.getIp() != null && port.getPrivatePort() == MesosMasterConfig.MESOS_MASTER_PORT) {
                cluster.setMapPortsToHost(true);
            }
        }
    }
}