Java Code Examples for org.apache.hadoop.hbase.util.FSUtils#setRootDir()

/**
 * Creates an hbase rootdir in user home directory.  Also creates hbase
 * version file.  Normally you won't make use of this method.  Root hbasedir
 * is created for you as part of mini cluster startup.  You'd only use this
 * method if you were doing manual operation.
 * @return Fully qualified path to hbase root dir
 * @throws java.io.IOException
 */
public Path createRootDir() throws IOException {
  FileSystem fs = FileSystem.get(this.conf);
  Path hbaseRootdir = getDefaultRootDirPath();
  FSUtils.setRootDir(this.conf, hbaseRootdir);
  fs.mkdirs(hbaseRootdir);
  FSUtils.setVersion(fs, hbaseRootdir);
  return hbaseRootdir;
}
 

public Job createSubmittableJob(String schemaName, String indexTable, String dataTable,
        SourceTable sourceTable, Class<IndexScrutinyMapperForTest> mapperClass) throws Exception {
    Preconditions.checkArgument(SourceTable.DATA_TABLE_SOURCE.equals(sourceTable)
            || SourceTable.INDEX_TABLE_SOURCE.equals(sourceTable));

    final String qDataTable = SchemaUtil.getQualifiedTableName(schemaName, dataTable);
    final String qIndexTable;
    if (schemaName != null && !schemaName.isEmpty()) {
        qIndexTable = SchemaUtil.getQualifiedTableName(schemaName, indexTable);
    } else {
        qIndexTable = indexTable;
    }
    PhoenixConfigurationUtil.setScrutinyDataTable(configuration, qDataTable);
    PhoenixConfigurationUtil.setScrutinyIndexTable(configuration, qIndexTable);
    PhoenixConfigurationUtil.setScrutinySourceTable(configuration, sourceTable);
    PhoenixConfigurationUtil.setScrutinyOutputInvalidRows(configuration, outputInvalidRows);
    PhoenixConfigurationUtil.setScrutinyOutputMax(configuration, outputMaxRows);

    final PTable pdataTable = PhoenixRuntime.getTable(connection, qDataTable);
    final PTable pindexTable = PhoenixRuntime.getTable(connection, qIndexTable);

    // set CURRENT_SCN for our scan so that incoming writes don't throw off scrutiny
    configuration.set(PhoenixConfigurationUtil.CURRENT_SCN_VALUE, Long.toString(ts));

    // set the source table to either data or index table
    SourceTargetColumnNames columnNames =
            SourceTable.DATA_TABLE_SOURCE.equals(sourceTable)
                    ? new SourceTargetColumnNames.DataSourceColNames(pdataTable,
                            pindexTable)
                    : new SourceTargetColumnNames.IndexSourceColNames(pdataTable,
                            pindexTable);
    String qSourceTable = columnNames.getQualifiedSourceTableName();
    List<String> sourceColumnNames = columnNames.getSourceColNames();
    List<String> sourceDynamicCols = columnNames.getSourceDynamicCols();
    List<String> targetDynamicCols = columnNames.getTargetDynamicCols();

    // Setup the select query against source - we either select the index columns from the
    // index table,
    // or select the data table equivalents of the index columns from the data table
    final String selectQuery =
            QueryUtil.constructSelectStatement(qSourceTable, sourceColumnNames, null,
                Hint.NO_INDEX, true);
    LOGGER.info("Query used on source table to feed the mapper: " + selectQuery);

    PhoenixConfigurationUtil.setScrutinyOutputFormat(configuration, outputFormat);
    // if outputting to table, setup the upsert to the output table
    if (outputInvalidRows && OutputFormat.TABLE.equals(outputFormat)) {
        String upsertStmt =
                IndexScrutinyTableOutput.constructOutputTableUpsert(sourceDynamicCols,
                    targetDynamicCols, connection);
        PhoenixConfigurationUtil.setUpsertStatement(configuration, upsertStmt);
        LOGGER.info("Upsert statement used for output table: " + upsertStmt);
    }

    final String jobName =
            String.format(INDEX_JOB_NAME_TEMPLATE, qSourceTable,
                columnNames.getQualifiedTargetTableName());
    final Job job = Job.getInstance(configuration, jobName);

    if (!useSnapshot) {
        PhoenixMapReduceUtil.setInput(job, PhoenixIndexDBWritable.class, qDataTable,
            selectQuery);
    } else { // TODO check if using a snapshot works
        Admin admin = null;
        String snapshotName;
        try {
            final PhoenixConnection pConnection =
                    connection.unwrap(PhoenixConnection.class);
            admin = pConnection.getQueryServices().getAdmin();
            String pdataTableName = pdataTable.getName().getString();
            snapshotName = new StringBuilder(pdataTableName).append("-Snapshot").toString();
            admin.snapshot(snapshotName, TableName.valueOf(pdataTableName));
        } finally {
            if (admin != null) {
                admin.close();
            }
        }
        // root dir not a subdirectory of hbase dir
        Path rootDir = new Path("hdfs:///index-snapshot-dir");
        FSUtils.setRootDir(configuration, rootDir);

        // set input for map reduce job using hbase snapshots
        //PhoenixMapReduceUtil.setInput(job, PhoenixIndexDBWritable.class, snapshotName,
        //    qDataTable, restoreDir, selectQuery);
    }
    TableMapReduceUtil.initCredentials(job);
    Path outputPath =
            getOutputPath(configuration, basePath,
                SourceTable.DATA_TABLE_SOURCE.equals(sourceTable) ? pdataTable
                        : pindexTable);

    return configureSubmittableJob(job, outputPath, mapperClass);
}
 

private Job configureJobForAsyncIndex() throws Exception {
    String physicalIndexTable = pIndexTable.getPhysicalName().getString();
    final PhoenixConnection pConnection = connection.unwrap(PhoenixConnection.class);
    final PostIndexDDLCompiler ddlCompiler =
            new PostIndexDDLCompiler(pConnection, new TableRef(pDataTable));
    ddlCompiler.compile(pIndexTable);
    final List<String> indexColumns = ddlCompiler.getIndexColumnNames();
    final String selectQuery = ddlCompiler.getSelectQuery();
    final String upsertQuery =
            QueryUtil.constructUpsertStatement(qIndexTable, indexColumns, Hint.NO_INDEX);

    configuration.set(PhoenixConfigurationUtil.UPSERT_STATEMENT, upsertQuery);
    PhoenixConfigurationUtil.setPhysicalTableName(configuration, physicalIndexTable);
    PhoenixConfigurationUtil.setDisableIndexes(configuration, indexTable);

    PhoenixConfigurationUtil.setUpsertColumnNames(configuration,
        indexColumns.toArray(new String[indexColumns.size()]));
    if (tenantId != null) {
        PhoenixConfigurationUtil.setTenantId(configuration, tenantId);
    }
    final List<ColumnInfo> columnMetadataList =
            PhoenixRuntime.generateColumnInfo(connection, qIndexTable, indexColumns);
    ColumnInfoToStringEncoderDecoder.encode(configuration, columnMetadataList);

    if (outputPath != null) {
        fs = outputPath.getFileSystem(configuration);
        fs.delete(outputPath, true);
    }
    final String jobName = String.format(INDEX_JOB_NAME_TEMPLATE, schemaName, dataTable, indexTable);
    final Job job = Job.getInstance(configuration, jobName);
    job.setJarByClass(IndexTool.class);
    job.setMapOutputKeyClass(ImmutableBytesWritable.class);
    if (outputPath != null) {
        FileOutputFormat.setOutputPath(job, outputPath);
    }

    if (!useSnapshot) {
        PhoenixMapReduceUtil.setInput(job, PhoenixIndexDBWritable.class, qDataTable, selectQuery);
    } else {
        Admin admin = null;
        String snapshotName;
        try {
            admin = pConnection.getQueryServices().getAdmin();
            String pdataTableName = pDataTable.getName().getString();
            snapshotName = new StringBuilder(pdataTableName).append("-Snapshot").toString();
            admin.snapshot(snapshotName, TableName.valueOf(pdataTableName));
        } finally {
            if (admin != null) {
                admin.close();
            }
        }
        // root dir not a subdirectory of hbase dir
        Path rootDir = new Path("hdfs:///index-snapshot-dir");
        FSUtils.setRootDir(configuration, rootDir);
        Path restoreDir = new Path(FSUtils.getRootDir(configuration), "restore-dir");

        // set input for map reduce job using hbase snapshots
        PhoenixMapReduceUtil
                    .setInput(job, PhoenixIndexDBWritable.class, snapshotName, qDataTable, restoreDir, selectQuery);
    }
    TableMapReduceUtil.initCredentials(job);
    
    job.setMapperClass(PhoenixIndexImportDirectMapper.class);
    return configureSubmittableJobUsingDirectApi(job);
}
 

/**
 * Configure the HBase cluster before launching it
 * 
 * @param config
 *          already created Hadoop configuration we'll further configure for
 *          HDFS
 * @param zkClientPort
 *          The client port zookeeper is listening on
 * @param hdfsFs
 *          The HDFS FileSystem this HBase cluster will run on top of
 * @param bindIP
 *          The IP Address to force bind all sockets on. If null, will use
 *          defaults
 * @param masterPort
 *          The port the master listens on
 * @param regionserverPort
 *          The port the regionserver listens on
 * @return The updated Configuration object.
 * @throws IOException
 */
private static Configuration configureHBaseCluster(Configuration config,
    int zkClientPort, FileSystem hdfsFs, String bindIP, int masterPort,
    int regionserverPort) throws IOException {
  // Configure the zookeeper port
  config
      .set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(zkClientPort));
  // Initialize HDFS path configurations required by HBase
  Path hbaseDir = new Path(hdfsFs.makeQualified(hdfsFs.getHomeDirectory()),
      "hbase");
  FSUtils.setRootDir(config, hbaseDir);
  hdfsFs.mkdirs(hbaseDir);
  config.set("fs.defaultFS", hdfsFs.getUri().toString());
  config.set("fs.default.name", hdfsFs.getUri().toString());
  FSUtils.setVersion(hdfsFs, hbaseDir);

  // Configure the bind addresses and ports. If running in Openshift, we only
  // have permission to bind to the private IP address, accessible through an
  // environment variable.
  logger.info("HBase force binding to ip: " + bindIP);
  config.set("hbase.master.ipc.address", bindIP);
  config.set(HConstants.MASTER_PORT, Integer.toString(masterPort));
  config.set("hbase.regionserver.ipc.address", bindIP);
  config
      .set(HConstants.REGIONSERVER_PORT, Integer.toString(regionserverPort));
  config.set(HConstants.ZOOKEEPER_QUORUM, bindIP);

  // By default, the HBase master and regionservers will report to zookeeper
  // that its hostname is what it determines by reverse DNS lookup, and not
  // what we use as the bind address. This means when we set the bind
  // address, daemons won't actually be able to connect to eachother if they
  // are different. Here, we do something that's illegal in 48 states - use
  // reflection to override a private static final field in the DNS class
  // that is a cachedHostname. This way, we are forcing the hostname that
  // reverse dns finds. This may not be compatible with newer versions of
  // Hadoop.
  try {
    Field cachedHostname = DNS.class.getDeclaredField("cachedHostname");
    cachedHostname.setAccessible(true);
    Field modifiersField = Field.class.getDeclaredField("modifiers");
    modifiersField.setAccessible(true);
    modifiersField.setInt(cachedHostname, cachedHostname.getModifiers()
        & ~Modifier.FINAL);
    cachedHostname.set(null, bindIP);
  } catch (Exception e) {
    // Reflection can throw so many checked exceptions. Let's wrap in an
    // IOException.
    throw new IOException(e);
  }

  // By setting the info ports to -1 for, we won't launch the master or
  // regionserver info web interfaces
  config.set(HConstants.MASTER_INFO_PORT, "-1");
  config.set(HConstants.REGIONSERVER_INFO_PORT, "-1");
  return config;
}