org.apache.accumulo.minicluster.MiniAccumuloCluster Java Examples

/**
 * Gets the AccumuloConnector singleton, starting the MiniAccumuloCluster on initialization.
 * This singleton instance is required so all test cases access the same MiniAccumuloCluster.
 *
 * @return Accumulo connector
 */
public static Connector getAccumuloConnector()
{
    if (connector != null) {
        return connector;
    }

    try {
        MiniAccumuloCluster accumulo = createMiniAccumuloCluster();
        Instance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
        connector = instance.getConnector(MAC_USER, new PasswordToken(MAC_PASSWORD));
        LOG.info("Connection to MAC instance %s at %s established, user %s password %s", accumulo.getInstanceName(), accumulo.getZooKeepers(), MAC_USER, MAC_PASSWORD);
        return connector;
    }
    catch (AccumuloException | AccumuloSecurityException | InterruptedException | IOException e) {
        throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to get connector to Accumulo", e);
    }
}
 

private void installRyaInstance() throws Exception {
    final MiniAccumuloCluster cluster = super.getMiniAccumuloCluster();
    final String instanceName = cluster.getInstanceName();
    final String zookeepers = cluster.getZooKeepers();

    // Install the Rya instance to the mini accumulo cluster.
    final RyaClient ryaClient = AccumuloRyaClientFactory.build(
            new AccumuloConnectionDetails(ACCUMULO_USER, ACCUMULO_PASSWORD.toCharArray(), instanceName, zookeepers),
            super.getAccumuloConnector());

    ryaClient.getInstall().install(RYA_INSTANCE_NAME,
            InstallConfiguration.builder().setEnableTableHashPrefix(false).setEnableFreeTextIndex(false)
                    .setEnableEntityCentricIndex(false).setEnableGeoIndex(false).setEnableTemporalIndex(false).setEnablePcjIndex(true)
                    .setFluoPcjAppName(super.getFluoConfiguration().getApplicationName()).build());

    // Connect to the Rya instance that was just installed.
    final AccumuloRdfConfiguration conf = makeConfig(instanceName, zookeepers);
    final Sail sail = RyaSailFactory.getInstance(conf);
    dao = RyaSailFactory.getAccumuloDAOWithUpdatedConfig(conf);
    ryaSailRepo = new RyaSailRepository(sail);
}
 

@After
public void teardownRya() {
    final MiniAccumuloCluster cluster = getMiniAccumuloCluster();
    final String instanceName = cluster.getInstanceName();
    final String zookeepers = cluster.getZooKeepers();

    // Uninstall the instance of Rya.
    final RyaClient ryaClient = AccumuloRyaClientFactory.build(
            new AccumuloConnectionDetails(ACCUMULO_USER, ACCUMULO_PASSWORD.toCharArray(), instanceName, zookeepers),
            super.getAccumuloConnector());

    try {
        ryaClient.getUninstall().uninstall(RYA_INSTANCE_NAME);
        // Shutdown the repo.
        if(ryaSailRepo != null) {ryaSailRepo.shutDown();}
        if(dao != null ) {dao.destroy();}
    } catch (final Exception e) {
        System.out.println("Encountered the following Exception when shutting down Rya: " + e.getMessage());
    }
}
 

@Test
public void connectAccumulo() throws IOException {
    final MiniAccumuloCluster cluster = getCluster();
    final Bootstrap bootstrap = getTestBootstrap();
    final JLineShellComponent shell = getTestShell();

    // Mock the user entering the correct password.
    final ApplicationContext context = bootstrap.getApplicationContext();
    final PasswordPrompt mockPrompt = context.getBean( PasswordPrompt.class );
    when(mockPrompt.getPassword()).thenReturn("password".toCharArray());

    // Execute the connect command.
    final String cmd =
            RyaConnectionCommands.CONNECT_ACCUMULO_CMD + " " +
                    "--username root " +
                    "--instanceName " + cluster.getInstanceName() + " "+
                    "--zookeepers " + cluster.getZooKeepers();

    final CommandResult connectResult = shell.executeCommand(cmd);

    // Ensure the connection was successful.
    assertTrue( connectResult.isSuccess() );
}
 

@Test
public void connectAccumulo_wrongCredentials() throws IOException {
    final MiniAccumuloCluster cluster = getCluster();
    final Bootstrap bootstrap = getTestBootstrap();
    final JLineShellComponent shell = getTestShell();

    // Mock the user entering the wrong password.
    final ApplicationContext context = bootstrap.getApplicationContext();
    final PasswordPrompt mockPrompt = context.getBean( PasswordPrompt.class );
    when(mockPrompt.getPassword()).thenReturn("asjifo[ijwa".toCharArray());

    // Execute the command
    final String cmd =
            RyaConnectionCommands.CONNECT_ACCUMULO_CMD + " " +
                    "--username root " +
                    "--instanceName " + cluster.getInstanceName() + " "+
                    "--zookeepers " + cluster.getZooKeepers();

    final CommandResult connectResult = shell.executeCommand(cmd);

    // Ensure the command failed.
    assertFalse( connectResult.isSuccess() );
}
 

@Test
public void connectToInstance_instanceDoesNotExist() throws IOException {
    final MiniAccumuloCluster cluster = getCluster();
    final Bootstrap bootstrap = getTestBootstrap();
    final JLineShellComponent shell = getTestShell();

    // Mock the user entering the correct password.
    final ApplicationContext context = bootstrap.getApplicationContext();
    final PasswordPrompt mockPrompt = context.getBean( PasswordPrompt.class );
    when(mockPrompt.getPassword()).thenReturn("password".toCharArray());

    // Connect to the mini accumulo instance.
    String cmd =
            RyaConnectionCommands.CONNECT_ACCUMULO_CMD + " " +
                    "--username root " +
                    "--instanceName " + cluster.getInstanceName() + " "+
                    "--zookeepers " + cluster.getZooKeepers();
    shell.executeCommand(cmd);

    // Try to connect to a non-existing instance.
    cmd = RyaConnectionCommands.CONNECT_INSTANCE_CMD + " --instance doesNotExist";
    final CommandResult result = shell.executeCommand(cmd);
    assertFalse( result.isSuccess() );
}
 

@Test
public void disconnect() throws IOException {
    final MiniAccumuloCluster cluster = getCluster();
    final Bootstrap bootstrap = getTestBootstrap();
    final JLineShellComponent shell = getTestShell();

    // Mock the user entering the correct password.
    final ApplicationContext context = bootstrap.getApplicationContext();
    final PasswordPrompt mockPrompt = context.getBean( PasswordPrompt.class );
    when(mockPrompt.getPassword()).thenReturn("password".toCharArray());

    // Connect to the mini accumulo instance.
    final String cmd =
            RyaConnectionCommands.CONNECT_ACCUMULO_CMD + " " +
                    "--username root " +
                    "--instanceName " + cluster.getInstanceName() + " "+
                    "--zookeepers " + cluster.getZooKeepers();
    shell.executeCommand(cmd);

    // Disconnect from it.
    final CommandResult disconnectResult = shell.executeCommand( RyaConnectionCommands.DISCONNECT_COMMAND_NAME_CMD );
    assertTrue( disconnectResult.isSuccess() );
}
 

@BeforeClass
public static void setUpAccumulo() throws Exception {
  instanceName = System.getProperty(IT_INSTANCE_NAME_PROP, "it-instance-default");
  File instanceDir = new File("target/accumulo2-maven-plugin/" + instanceName);
  boolean instanceClear =
      System.getProperty(IT_INSTANCE_CLEAR_PROP, "true").equalsIgnoreCase("true");
  if (instanceDir.exists() && instanceClear) {
    FileUtils.deleteDirectory(instanceDir);
  }
  if (!instanceDir.exists()) {
    MiniAccumuloConfig cfg = new MiniAccumuloConfig(instanceDir, PASSWORD);
    cfg.setInstanceName(instanceName);
    cluster = new MiniAccumuloCluster(cfg);
    cluster.start();
    startedCluster = true;
  }
  Properties props = MiniAccumuloCluster.getClientProperties(instanceDir);
  aClient = Accumulo.newClient().from(props).build();
}
 

@BeforeClass
public static void setupClusters() throws Exception {
    MiniAccumuloConfig macConf = new MiniAccumuloConfig(tmpdir, "password");
    macConf.setNumTservers(1);

    accumuloCluster = new MiniAccumuloCluster(macConf);
    accumuloCluster.start();
}
 

@BeforeClass
public static void setupInstance() throws IOException, InterruptedException, AccumuloSecurityException, AccumuloException, TableExistsException {
    Assume.assumeTrue("Test only runs on *nix", !SystemUtils.IS_OS_WINDOWS);
    Path tempDirectory = Files.createTempDirectory("acc"); // JUnit and Guava supply mechanisms for creating temp directories
    accumulo = new MiniAccumuloCluster(tempDirectory.toFile(), "password");
    accumulo.start();
}
 

@BeforeClass
public static void setupInstance() throws IOException, InterruptedException, AccumuloSecurityException, AccumuloException, TableExistsException {
    Assume.assumeTrue("Test only runs on *nix", !SystemUtils.IS_OS_WINDOWS);
    Path tempDirectory = Files.createTempDirectory("acc"); // JUnit and Guava supply mechanisms for creating temp directories
    accumulo = new MiniAccumuloCluster(tempDirectory.toFile(), "password");
    accumulo.start();
}
 

@Test
public void printConnectionDetails_connectedToAccumulo() throws IOException {
    final MiniAccumuloCluster cluster = getCluster();
    final Bootstrap bootstrap = getTestBootstrap();
    final JLineShellComponent shell = getTestShell();

    // Mock the user entering the correct password.
    final ApplicationContext context = bootstrap.getApplicationContext();
    final PasswordPrompt mockPrompt = context.getBean( PasswordPrompt.class );
    when(mockPrompt.getPassword()).thenReturn("password".toCharArray());

    // Connect to the mini accumulo instance.
    final String cmd =
            RyaConnectionCommands.CONNECT_ACCUMULO_CMD + " " +
                    "--username root " +
                    "--instanceName " + cluster.getInstanceName() + " "+
                    "--zookeepers " + cluster.getZooKeepers();
    shell.executeCommand(cmd);

    // Run the print connection details command.
    final CommandResult printResult = shell.executeCommand( RyaConnectionCommands.PRINT_CONNECTION_DETAILS_CMD );
    final String msg = (String) printResult.getResult();

    final String expected =
            "The shell is connected to an instance of Accumulo using the following parameters:\n" +
            "    Username: root\n" +
            "    Instance Name: " + cluster.getInstanceName() + "\n" +
            "    Zookeepers: " + cluster.getZooKeepers();
    assertEquals(expected, msg);
}
 

private void startMiniAccumulo() {
  try {
    // start mini accumulo cluster
    MiniAccumuloConfig cfg = new MiniAccumuloConfig(new File(config.getMiniDataDir()), PASSWORD);
    cluster = new MiniAccumuloCluster(cfg);
    cluster.start();

    log.debug("Started MiniAccumulo(accumulo=" + cluster.getInstanceName() + " zk="
        + cluster.getZooKeepers() + ")");

    // configuration that must overridden
    config.setAccumuloInstance(cluster.getInstanceName());
    config.setAccumuloUser(USER);
    config.setAccumuloPassword(PASSWORD);
    config.setAccumuloZookeepers(cluster.getZooKeepers());
    config.setInstanceZookeepers(cluster.getZooKeepers() + "/fluo");

    // configuration that only needs to be set if not by user
    if ((config.containsKey(FluoConfiguration.ACCUMULO_TABLE_PROP) == false)
        || config.getAccumuloTable().trim().isEmpty()) {
      config.setAccumuloTable("fluo");
    }

    InitializationOptions opts = new InitializationOptions();
    try (FluoAdmin admin = FluoFactory.newAdmin(config)) {
      admin.initialize(opts);
    }

    File miniProps = new File(clientPropsPath(config));
    config.getClientConfiguration().save(miniProps);

    log.debug("Wrote MiniFluo client properties to {}", miniProps.getAbsolutePath());

  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

private static MiniAccumuloCluster startMiniAccumulo() throws IOException, InterruptedException, AccumuloException, AccumuloSecurityException {
    final File miniDataDir = Files.createTempDir();

    // Setup and start the Mini Accumulo.
    final MiniAccumuloCluster accumulo = new MiniAccumuloCluster(
            miniDataDir, "password");
    accumulo.start();

    // Store a connector to the Mini Accumulo.
    final Instance instance = new ZooKeeperInstance(
            accumulo.getInstanceName(), accumulo.getZooKeepers());
    accCon = instance.getConnector("root", new PasswordToken("password"));

    return accumulo;
}
 

/**
 * Sets up the {@link MiniAccumuloCluster} or the {@link MockInstance}.
 * @throws Exception
 */
public void setUpInstance() throws Exception {
    if (!isMock) {
        log.info("Setting up " + driverName + " MiniAccumulo cluster...");
        // Create and Run MiniAccumulo Cluster
        tempDir = Files.createTempDir();
        tempDir.deleteOnExit();
        miniAccumuloCluster = new MiniAccumuloCluster(tempDir, userpwd);
        copyHadoopHomeToTemp();
        miniAccumuloCluster.getConfig().setInstanceName(instanceName);
        log.info(driverName + " MiniAccumulo instance starting up...");
        miniAccumuloCluster.start();
        Thread.sleep(1000);
        log.info(driverName + " MiniAccumulo instance started");
        log.info("Creating connector to " + driverName + " MiniAccumulo instance...");
        zooKeeperInstance = new ZooKeeperInstance(miniAccumuloCluster.getClientConfig());
        instance = zooKeeperInstance;
        connector = zooKeeperInstance.getConnector(user, new PasswordToken(userpwd));
        log.info("Created connector to " + driverName + " MiniAccumulo instance");
    } else {
        log.info("Setting up " + driverName + " mock instance...");
        mockInstance = new MockInstance(instanceName);
        instance = mockInstance;
        connector = mockInstance.getConnector(user, new PasswordToken(userpwd));
        log.info("Created connector to " + driverName + " mock instance");
    }
    zooKeepers = instance.getZooKeepers();
}
 

public void start() throws IOException {
    try {
        initInstanceAndConnector(300);
    } catch (Throwable e) {
        try {
            initConfig();
            miniAccumuloCluster = new MiniAccumuloCluster(miniAccumuloConfig);
            miniAccumuloCluster.start();
            log.info("started minicluster");
            initInstanceAndConnector(null);
        } catch (Throwable e1) {
            throw new IOException(e1);
        }
    }
}
 

@Test
public void testWriteSplitsToAccumuloAndReadThem() throws Exception {
    Configuration conf = new Configuration();
    conf.setInt(ShardIdFactory.NUM_SHARDS, 1);
    conf.setInt(ShardedTableMapFile.SHARDS_BALANCED_DAYS_TO_VERIFY, 1);
    String today = formatDay(0) + "_1";
    
    MiniAccumuloCluster accumuloCluster = null;
    try {
        SortedSet<Text> sortedSet = new TreeSet<>();
        sortedSet.add(new Text(today));
        accumuloCluster = createMiniAccumuloWithTestTableAndSplits(sortedSet);
        
        configureAccumuloHelper(conf, accumuloCluster);
        
        conf.set(ShardedDataTypeHandler.SHARDED_TNAMES, TABLE_NAME
                        + ",shard_ingest_unit_test_table_1,shard_ingest_unit_test_table_2,shard_ingest_unit_test_table_3");
        conf.set(ShardedTableMapFile.TABLE_NAMES, TABLE_NAME);
        setWorkingDirectory(conf);
        ShardedTableMapFile.setupFile(conf);
    } finally {
        if (null != accumuloCluster) {
            accumuloCluster.stop();
        }
    }
    
    TreeMap<Text,String> result = ShardedTableMapFile.getShardIdToLocations(conf, TABLE_NAME);
    Assert.assertNotNull(result.get(new Text(today)).toString());
    Assert.assertEquals(1, result.size());
}
 

/**
 * Setup a Mini Accumulo cluster that uses a temporary directory to store its data.
 *
 * @return A Mini Accumulo cluster.
 */
private static MiniAccumuloCluster startMiniAccumulo() throws IOException, InterruptedException, AccumuloException, AccumuloSecurityException {
    final File miniDataDir = Files.createTempDir();

    // Setup and start the Mini Accumulo.
    final MiniAccumuloCluster accumulo = new MiniAccumuloCluster(miniDataDir, "password");
    accumulo.start();

    // Store a connector to the Mini Accumulo.
    final Instance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
    accumuloConn = instance.getConnector("root", new PasswordToken("password"));

    return accumulo;
}
 

@BeforeClass
public static void startCluster() throws Exception {
    File macDir = new File(System.getProperty("user.dir") + "/target/mac/" + TableConfigHelperFactoryTest.class.getName());
    if (macDir.exists())
        FileUtils.deleteDirectory(macDir);
    macDir.mkdirs();
    mac = new MiniAccumuloCluster(new MiniAccumuloConfig(macDir, "pass"));
    mac.start();
}
 

@BeforeClass
public static void beforeClass() throws Exception {
    temp.create();
    final MiniAccumuloConfig macConfig = new MiniAccumuloConfig(temp.newFolder("mac"), "secret");
    mac = new MiniAccumuloCluster(macConfig);
    mac.start();
    conf = TestConfiguration.createMinimalConfigurationForTest();
    conf.getAccumulo().setInstanceName(mac.getInstanceName());
    conf.getAccumulo().setZookeepers(mac.getZooKeepers());
    setupSSL(conf);
}
 

@BeforeClass
public static void beforeClass() throws Exception {
    final MiniAccumuloConfig macConfig = new MiniAccumuloConfig(temp.newFolder("mac"), "secret");
    mac = new MiniAccumuloCluster(macConfig);
    mac.start();
    conf = TestConfiguration.createMinimalConfigurationForTest();
    conf.getAccumulo().setInstanceName(mac.getInstanceName());
    conf.getAccumulo().setZookeepers(mac.getZooKeepers());
    setupSSL(conf);
}
 

protected static MiniAccumuloCluster createMiniAccumuloCluster(File tempDir, String rootPassword) throws Exception {
  final String configImplClassName = "org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl", clusterImplClassName = "org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl";
  try {
    // Get the MiniAccumuloConfigImpl class
    Class<?> configImplClz = Class.forName(configImplClassName);
    // Get the (File,String) constructor
    Constructor<?> cfgConstructor = configImplClz.getConstructor(new Class[] {File.class, String.class});
    Object configImpl = cfgConstructor.newInstance(tempDir, rootPassword);
    // Get setClasspathItems(String...)
    Method setClasspathItemsMethod = configImplClz.getDeclaredMethod("setClasspathItems", String[].class);
    // Get the classpath, removing problematic jars
    String classpath = getClasspath(new File(tempDir, "conf"));
    // Call the method
    setClasspathItemsMethod.invoke(configImpl, (Object) new String[] {classpath});

    // Get the private MiniAccumuloCluster(MiniAccumuloConfigImpl constructor)
    Constructor<?> clusterConstructor = MiniAccumuloCluster.class.getDeclaredConstructor(configImplClz);
    // Make it accessible (since its private)
    clusterConstructor.setAccessible(true);
    Object clusterImpl = clusterConstructor.newInstance(configImpl);
    return MiniAccumuloCluster.class.cast(clusterImpl);
  } catch (Exception e) {
    // Couldn't load the 1.6 MiniAccumuloConfigImpl which has
    // the classpath control
    LOG.warn("Could not load 1.6 minicluster classes", e);

    return new MiniAccumuloCluster(tempDir, ACCUMULO_PASSWORD);
  }
}
 

@BeforeClass
public static void beforeClass() throws Exception {
    temp.create();
    final MiniAccumuloConfig macConfig = new MiniAccumuloConfig(temp.newFolder("mac"), "secret");
    mac = new MiniAccumuloCluster(macConfig);
    mac.start();
    conf = TestConfiguration.createMinimalConfigurationForTest();
    conf.getAccumulo().setInstanceName(mac.getInstanceName());
    conf.getAccumulo().setZookeepers(mac.getZooKeepers());
    setupSSL(conf);
}
 

/**
 * Start the {@link MiniAccumuloCluster}.
 */
public void startMiniAccumulo() throws IOException, InterruptedException, AccumuloException, AccumuloSecurityException {
    final File miniDataDir = Files.createTempDir();

    // Setup and start the Mini Accumulo.
    final MiniAccumuloConfig cfg = new MiniAccumuloConfig(miniDataDir, PASSWORD);
    cluster = new MiniAccumuloCluster(cfg);
    cluster.start();
}
 

@BeforeClass
public static void beforeClass() throws Exception {
    final MiniAccumuloConfig macConfig = new MiniAccumuloConfig(temp.newFolder("mac"), "secret");
    mac = new MiniAccumuloCluster(macConfig);
    mac.start();
    conf = TestConfiguration.createMinimalConfigurationForTest();
    conf.getAccumulo().setInstanceName(mac.getInstanceName());
    conf.getAccumulo().setZookeepers(mac.getZooKeepers());
    setupSSL(conf);
}
 

public MiniAccumuloCluster startMiniCluster(File tempFile) {

        try {
            amc = new MiniAccumuloCluster(tempFile, "password");
            amc.start();

            return amc;
        }catch (IOException | InterruptedException e){
            e.printStackTrace();
        }

        return null;
    }
 

/**
 * Run the demo.
 */
public void execute(
        MiniAccumuloCluster accumulo,
        Connector accumuloConn,
        String ryaTablePrefix,
        RyaSailRepository ryaRepo,
        RepositoryConnection ryaConn,
        MiniFluo fluo,
        FluoClient fluoClient) throws DemoExecutionException;
 

/**
 * @return The cluster that is hosting the test.
 */
public MiniAccumuloCluster getCluster() {
    return MiniAccumuloSingleton.getInstance().getCluster();
}
 

private void configureAccumuloHelper(Configuration conf, MiniAccumuloCluster accumuloCluster) {
    AccumuloHelper.setInstanceName(conf, accumuloCluster.getInstanceName());
    AccumuloHelper.setPassword(conf, PASSWORD.getBytes());
    AccumuloHelper.setUsername(conf, USERNAME);
    AccumuloHelper.setZooKeepers(conf, accumuloCluster.getZooKeepers());
}
 

public MiniAccumuloCluster startMiniCluster(String tempStr){
    return startMiniCluster(new File(tempStr));
}