Java Code Examples for org.apache.hadoop.hive.ql.plan.TableDesc#getProperties()

private void configureJobProperties(TableDesc tableDesc, Map<String, String> jobProperties) {
    if(LOG.isDebugEnabled()) {
        LOG.debug("tabelDesc: " + tableDesc);
        LOG.debug("jobProperties: " + jobProperties);
    }

    String tblName = tableDesc.getTableName();
    Properties tblProps = tableDesc.getProperties();
    String columnNames = tblProps.getProperty(Constants.LIST_COLUMNS);
    jobProperties.put(DBConfiguration.INPUT_CLASS_PROPERTY, DbRecordWritable.class.getName());
    jobProperties.put(DBConfiguration.INPUT_TABLE_NAME_PROPERTY, tblName);
    jobProperties.put(DBConfiguration.OUTPUT_TABLE_NAME_PROPERTY, tblName);
    jobProperties.put(DBConfiguration.INPUT_FIELD_NAMES_PROPERTY, columnNames);
    jobProperties.put(DBConfiguration.OUTPUT_FIELD_NAMES_PROPERTY, columnNames);

    for(String key : tblProps.stringPropertyNames()) {
        if(key.startsWith("mapred.jdbc.")) {
            String value = tblProps.getProperty(key);
            jobProperties.put(key, value);
        }
    }
}
 

@Override
public void configureOutputJobProperties(TableDesc tbl, Map<String, String> jobProperties) {
    final Properties properties = tbl.getProperties();

    //设置属性到运行时的jobconf里面
    Conf.copyProperties(properties,jobProperties);

}
 

@Override
public void configureInputJobProperties(TableDesc tableDesc, Map<String,
    String> jobProperties) {
  Properties tableProperties = tableDesc.getProperties();
  new KafkaBackedTableProperties().initialize(tableProperties, jobProperties, tableDesc);

}
 

@Override
public void configureOutputJobProperties(TableDesc tableDesc, Map<String,
    String> jobProperties) {
  Properties tableProperties = tableDesc.getProperties();
  new KafkaBackedTableProperties().initialize(tableProperties, jobProperties, tableDesc);

}
 

/**
 *
 * @param desc table description
 * @param jobProps
 */
@Override
public void configureTableJobProperties(TableDesc desc,
                                        Map<String, String> jobProps) {
    Properties tblProperties = desc.getProperties();
    jobProps.put(AccumuloSerde.COLUMN_MAPPINGS,
            tblProperties.getProperty(AccumuloSerde.COLUMN_MAPPINGS));
    String tableName = tblProperties.getProperty(AccumuloSerde.TABLE_NAME);
    jobProps.put(AccumuloSerde.TABLE_NAME, tableName);
    String useIterators = tblProperties.getProperty(AccumuloSerde.NO_ITERATOR_PUSHDOWN);
    if(useIterators != null) {
        jobProps.put(AccumuloSerde.NO_ITERATOR_PUSHDOWN, useIterators);
    }

}
 

@Override
public void configureInputJobProperties(TableDesc tableDesc, Map<String, String> properties) {
    Properties props = tableDesc.getProperties();
    properties.put(AccumuloSerde.COLUMN_MAPPINGS,
            props.getProperty(AccumuloSerde.COLUMN_MAPPINGS));
    properties.put(AccumuloSerde.TABLE_NAME,
            props.getProperty(AccumuloSerde.TABLE_NAME));
    String useIterators = props.getProperty(AccumuloSerde.NO_ITERATOR_PUSHDOWN);
    if(useIterators != null) {
        properties.put(AccumuloSerde.NO_ITERATOR_PUSHDOWN, useIterators);
    }
}
 

public void configureTableJobProperties(TableDesc tableDesc,
                                        Map<String, String> jobProperties, boolean isInputJob) throws Exception {
    Properties tableProperties = tableDesc.getProperties();
    String tableName = null;
    String connStr = tableProperties.getProperty(MRConstants.SPLICE_JDBC_STR);
    if (connStr == null)
        throw new Exception("Error: wrong param. Did you mean '" + MRConstants.SPLICE_JDBC_STR + "'?"); // TODO JL
    if (sqlUtil == null)
        sqlUtil = SMSQLUtil.getInstance(connStr);
    if (isInputJob) {
        tableName = tableProperties.getProperty(MRConstants.SPLICE_TABLE_NAME);
        if (tableName == null)
            throw new Exception("Error: wrong param. Did you mean '" + MRConstants.SPLICE_TABLE_NAME + "'?");
    } else {
        tableName = tableProperties.getProperty(MRConstants.SPLICE_TABLE_NAME);
        if (tableName == null)
            throw new Exception("Error: wrong param. Did you mean '" + MRConstants.SPLICE_TABLE_NAME + "'?");
    }

    tableName = tableName.trim();

    if (parentConn == null) {
        parentTxnId = startWriteJobParentTxn(connStr, tableName);
    }
    jobProperties.put(MRConstants.SPLICE_TRANSACTION_ID, parentTxnId);
    jobProperties.put(MRConstants.SPLICE_TABLE_NAME, tableName);
    jobProperties.put(MRConstants.SPLICE_JDBC_STR, connStr);
}
 

@Override
public void configureInputJobProperties(TableDesc tbl, Map<String, String> jobProperties) {
    final Properties properties = tbl.getProperties();
    //设置属性到运行时的jobconf里面
    Conf.copyProperties(properties,jobProperties);
}
 

private void configureJobProperties(TableDesc tableDesc,
                                    Map<String, String> jobProperties) {

    //This will always have the DB Name qualifier of Hive. Dont use this to set Kudu Tablename.
    String tblName = tableDesc.getTableName();
    LOG.debug("Hive Table Name:" + tblName);
    Properties tblProps = tableDesc.getProperties();
    String columnNames = tblProps.getProperty(HiveKuduConstants.LIST_COLUMNS);
    String columnTypes = tblProps.getProperty(HiveKuduConstants.LIST_COLUMN_TYPES);
    LOG.debug("Columns names:" + columnNames);
    LOG.debug("Column types:" + columnTypes);

    if (columnNames.length() == 0) {
        //TODO: Place keeper to insert SerDeHelper code to connect to Kudu to extract column names.
        LOG.warn("SerDe currently doesn't support column names and types. Please provide it explicitly");
    }

    //set map reduce properties.
    jobProperties.put(HiveKuduConstants.MR_INPUT_TABLE_NAME,
            tblProps.getProperty(HiveKuduConstants.TABLE_NAME));
    jobProperties.put(HiveKuduConstants.MR_OUTPUT_TABLE_NAME,
            tblProps.getProperty(HiveKuduConstants.TABLE_NAME));
    jobProperties.put(HiveKuduConstants.MR_MASTER_ADDRESS_NAME,
            tblProps.getProperty(HiveKuduConstants.MASTER_ADDRESS_NAME));

    LOG.debug("Kudu Table Name: " + tblProps.getProperty(HiveKuduConstants.TABLE_NAME));
    LOG.debug("Kudu Master Addresses: " + tblProps.getProperty(HiveKuduConstants.MASTER_ADDRESS_NAME));


    //set configuration property
    conf.set(HiveKuduConstants.MR_INPUT_TABLE_NAME,
            tblProps.getProperty(HiveKuduConstants.TABLE_NAME));
    conf.set(HiveKuduConstants.MR_OUTPUT_TABLE_NAME,
            tblProps.getProperty(HiveKuduConstants.TABLE_NAME));
    conf.set(HiveKuduConstants.MR_MASTER_ADDRESS_NAME,
            tblProps.getProperty(HiveKuduConstants.MASTER_ADDRESS_NAME));

    conf.set(HiveKuduConstants.TABLE_NAME,
            tblProps.getProperty(HiveKuduConstants.TABLE_NAME));
    conf.set(HiveKuduConstants.MASTER_ADDRESS_NAME,
            tblProps.getProperty(HiveKuduConstants.MASTER_ADDRESS_NAME));

    //set class variables
    kuduMaster = conf.get(HiveKuduConstants.MASTER_ADDRESS_NAME);
    kuduTableName = conf.get(HiveKuduConstants.TABLE_NAME);

    for (String key : tblProps.stringPropertyNames()) {
        if (key.startsWith(HiveKuduConstants.MR_PROPERTY_PREFIX)) {
            String value = tblProps.getProperty(key);
            jobProperties.put(key, value);
            //Also set configuration for Non Map Reduce Hive calls to the Handler
            conf.set(key, value);
        }
    }
}