org.apache.hadoop.mapreduce.lib.db.DBInputFormat.NullDBWritable Java Examples

The following examples show how to use org.apache.hadoop.mapreduce.lib.db.DBInputFormat.NullDBWritable. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestDbClasses.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * test generate sql script for OracleDBRecordReader.
 */

@Test(timeout = 20000)
public void testOracleDBRecordReader() throws Exception {
  DBInputSplit splitter = new DBInputSplit(1, 10);
  Configuration configuration = new Configuration();
  Connection connect = DriverForTest.getConnection();

  DBConfiguration dbConfiguration = new DBConfiguration(configuration);
  dbConfiguration.setInputOrderBy("Order");
  String[] fields = { "f1", "f2" };

  OracleDBRecordReader<NullDBWritable> recorder = new OracleDBRecordReader<NullDBWritable>(
      splitter, NullDBWritable.class, configuration, connect,
      dbConfiguration, "condition", fields, "table");
  assertEquals(
      "SELECT * FROM (SELECT a.*,ROWNUM dbif_rno FROM ( SELECT f1, f2 FROM table WHERE condition ORDER BY Order ) a WHERE rownum <= 10 ) WHERE dbif_rno > 1",
      recorder.getSelectQuery());
}
Example #2
Source File: TestDbClasses.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * test generate sql script for OracleDBRecordReader.
 */

@Test(timeout = 20000)
public void testOracleDBRecordReader() throws Exception {
  DBInputSplit splitter = new DBInputSplit(1, 10);
  Configuration configuration = new Configuration();
  Connection connect = DriverForTest.getConnection();

  DBConfiguration dbConfiguration = new DBConfiguration(configuration);
  dbConfiguration.setInputOrderBy("Order");
  String[] fields = { "f1", "f2" };

  OracleDBRecordReader<NullDBWritable> recorder = new OracleDBRecordReader<NullDBWritable>(
      splitter, NullDBWritable.class, configuration, connect,
      dbConfiguration, "condition", fields, "table");
  assertEquals(
      "SELECT * FROM (SELECT a.*,ROWNUM dbif_rno FROM ( SELECT f1, f2 FROM table WHERE condition ORDER BY Order ) a WHERE rownum <= 10 ) WHERE dbif_rno > 1",
      recorder.getSelectQuery());
}
Example #3
Source File: TestDbClasses.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * test splitters from DataDrivenDBInputFormat. For different data types may
 * be different splitter
 */
@Test(timeout = 10000)
public void testDataDrivenDBInputFormatSplitter() {
  DataDrivenDBInputFormat<NullDBWritable> format = new DataDrivenDBInputFormat<NullDBWritable>();
  testCommonSplitterTypes(format);
  assertEquals(DateSplitter.class, format.getSplitter(Types.TIMESTAMP)
      .getClass());
  assertEquals(DateSplitter.class, format.getSplitter(Types.DATE).getClass());
  assertEquals(DateSplitter.class, format.getSplitter(Types.TIME).getClass());
}
Example #4
Source File: TestDbClasses.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 10000)
public void testDataDrivenDBInputFormat() throws Exception {
  JobContext jobContext = mock(JobContext.class);
  Configuration configuration = new Configuration();
  configuration.setInt(MRJobConfig.NUM_MAPS, 1);

  when(jobContext.getConfiguration()).thenReturn(configuration);
  DataDrivenDBInputFormat<NullDBWritable> format = new DataDrivenDBInputFormat<NullDBWritable>();
  List<InputSplit> splits = format.getSplits(jobContext);
  assertEquals(1, splits.size());
  DataDrivenDBInputSplit split = (DataDrivenDBInputSplit) splits.get(0);
  assertEquals("1=1", split.getLowerClause());
  assertEquals("1=1", split.getUpperClause());

  // 2
  configuration.setInt(MRJobConfig.NUM_MAPS, 2);

  DataDrivenDBInputFormat.setBoundingQuery(configuration, "query");
  assertEquals("query",
      configuration.get(DBConfiguration.INPUT_BOUNDING_QUERY));

  Job job = mock(Job.class);
  when(job.getConfiguration()).thenReturn(configuration);
  DataDrivenDBInputFormat.setInput(job, NullDBWritable.class, "query",
      "Bounding Query");

  assertEquals("Bounding Query",
      configuration.get(DBConfiguration.INPUT_BOUNDING_QUERY));
}
Example #5
Source File: TestDbClasses.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 10000)
public void testOracleDataDrivenDBInputFormat() throws Exception {
  OracleDataDrivenDBInputFormat<NullDBWritable> format = 
      new OracleDataDrivenDBInputFormatForTest();
  testCommonSplitterTypes(format);
  assertEquals(OracleDateSplitter.class, format.getSplitter(Types.TIMESTAMP)
      .getClass());
  assertEquals(OracleDateSplitter.class, 
      format.getSplitter(Types.DATE).getClass());
  assertEquals(OracleDateSplitter.class, 
      format.getSplitter(Types.TIME).getClass());
}
Example #6
Source File: TestDbClasses.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void testCommonSplitterTypes(
    DataDrivenDBInputFormat<NullDBWritable> format) {
  assertEquals(BigDecimalSplitter.class, format.getSplitter(Types.DECIMAL)
      .getClass());
  assertEquals(BigDecimalSplitter.class, format.getSplitter(Types.NUMERIC)
      .getClass());
  assertEquals(BooleanSplitter.class, format.getSplitter(Types.BOOLEAN)
      .getClass());
  assertEquals(BooleanSplitter.class, format.getSplitter(Types.BIT)
      .getClass());
  assertEquals(IntegerSplitter.class, format.getSplitter(Types.BIGINT)
      .getClass());
  assertEquals(IntegerSplitter.class, format.getSplitter(Types.TINYINT)
      .getClass());
  assertEquals(IntegerSplitter.class, format.getSplitter(Types.SMALLINT)
      .getClass());
  assertEquals(IntegerSplitter.class, format.getSplitter(Types.INTEGER)
      .getClass());
  assertEquals(FloatSplitter.class, format.getSplitter(Types.DOUBLE)
      .getClass());
  assertEquals(FloatSplitter.class, format.getSplitter(Types.REAL).getClass());
  assertEquals(FloatSplitter.class, format.getSplitter(Types.FLOAT)
      .getClass());
  assertEquals(TextSplitter.class, format.getSplitter(Types.LONGVARCHAR)
      .getClass());
  assertEquals(TextSplitter.class, format.getSplitter(Types.CHAR).getClass());
  assertEquals(TextSplitter.class, format.getSplitter(Types.VARCHAR)
      .getClass());
  // if unknown data type splitter is null
  assertNull(format.getSplitter(Types.BINARY));
}
Example #7
Source File: TestDbClasses.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * test splitters from DataDrivenDBInputFormat. For different data types may
 * be different splitter
 */
@Test(timeout = 10000)
public void testDataDrivenDBInputFormatSplitter() {
  DataDrivenDBInputFormat<NullDBWritable> format = new DataDrivenDBInputFormat<NullDBWritable>();
  testCommonSplitterTypes(format);
  assertEquals(DateSplitter.class, format.getSplitter(Types.TIMESTAMP)
      .getClass());
  assertEquals(DateSplitter.class, format.getSplitter(Types.DATE).getClass());
  assertEquals(DateSplitter.class, format.getSplitter(Types.TIME).getClass());
}
Example #8
Source File: TestDbClasses.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 10000)
public void testDataDrivenDBInputFormat() throws Exception {
  JobContext jobContext = mock(JobContext.class);
  Configuration configuration = new Configuration();
  configuration.setInt(MRJobConfig.NUM_MAPS, 1);

  when(jobContext.getConfiguration()).thenReturn(configuration);
  DataDrivenDBInputFormat<NullDBWritable> format = new DataDrivenDBInputFormat<NullDBWritable>();
  List<InputSplit> splits = format.getSplits(jobContext);
  assertEquals(1, splits.size());
  DataDrivenDBInputSplit split = (DataDrivenDBInputSplit) splits.get(0);
  assertEquals("1=1", split.getLowerClause());
  assertEquals("1=1", split.getUpperClause());

  // 2
  configuration.setInt(MRJobConfig.NUM_MAPS, 2);

  DataDrivenDBInputFormat.setBoundingQuery(configuration, "query");
  assertEquals("query",
      configuration.get(DBConfiguration.INPUT_BOUNDING_QUERY));

  Job job = mock(Job.class);
  when(job.getConfiguration()).thenReturn(configuration);
  DataDrivenDBInputFormat.setInput(job, NullDBWritable.class, "query",
      "Bounding Query");

  assertEquals("Bounding Query",
      configuration.get(DBConfiguration.INPUT_BOUNDING_QUERY));
}
Example #9
Source File: TestDbClasses.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 10000)
public void testOracleDataDrivenDBInputFormat() throws Exception {
  OracleDataDrivenDBInputFormat<NullDBWritable> format = 
      new OracleDataDrivenDBInputFormatForTest();
  testCommonSplitterTypes(format);
  assertEquals(OracleDateSplitter.class, format.getSplitter(Types.TIMESTAMP)
      .getClass());
  assertEquals(OracleDateSplitter.class, 
      format.getSplitter(Types.DATE).getClass());
  assertEquals(OracleDateSplitter.class, 
      format.getSplitter(Types.TIME).getClass());
}
Example #10
Source File: TestDbClasses.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void testCommonSplitterTypes(
    DataDrivenDBInputFormat<NullDBWritable> format) {
  assertEquals(BigDecimalSplitter.class, format.getSplitter(Types.DECIMAL)
      .getClass());
  assertEquals(BigDecimalSplitter.class, format.getSplitter(Types.NUMERIC)
      .getClass());
  assertEquals(BooleanSplitter.class, format.getSplitter(Types.BOOLEAN)
      .getClass());
  assertEquals(BooleanSplitter.class, format.getSplitter(Types.BIT)
      .getClass());
  assertEquals(IntegerSplitter.class, format.getSplitter(Types.BIGINT)
      .getClass());
  assertEquals(IntegerSplitter.class, format.getSplitter(Types.TINYINT)
      .getClass());
  assertEquals(IntegerSplitter.class, format.getSplitter(Types.SMALLINT)
      .getClass());
  assertEquals(IntegerSplitter.class, format.getSplitter(Types.INTEGER)
      .getClass());
  assertEquals(FloatSplitter.class, format.getSplitter(Types.DOUBLE)
      .getClass());
  assertEquals(FloatSplitter.class, format.getSplitter(Types.REAL).getClass());
  assertEquals(FloatSplitter.class, format.getSplitter(Types.FLOAT)
      .getClass());
  assertEquals(TextSplitter.class, format.getSplitter(Types.LONGVARCHAR)
      .getClass());
  assertEquals(TextSplitter.class, format.getSplitter(Types.CHAR).getClass());
  assertEquals(TextSplitter.class, format.getSplitter(Types.VARCHAR)
      .getClass());
  // if unknown data type splitter is null
  assertNull(format.getSplitter(Types.BINARY));
}
Example #11
Source File: UpdateStatisticsTool.java    From phoenix with Apache License 2.0 5 votes vote down vote up 
private void configureJob() throws Exception {
    job = Job.getInstance(getConf(),
            "UpdateStatistics-" + tableName + "-" + snapshotName);
    PhoenixMapReduceUtil.setInput(job, NullDBWritable.class,
            snapshotName, tableName, restoreDir);

    PhoenixConfigurationUtil.setMRJobType(job.getConfiguration(), MRJobType.UPDATE_STATS);

    // DO NOT allow mapper splits using statistics since it may result into many smaller chunks
    PhoenixConfigurationUtil.setSplitByStats(job.getConfiguration(), false);

    job.setJarByClass(UpdateStatisticsTool.class);
    job.setMapperClass(TableSnapshotMapper.class);
    job.setMapOutputKeyClass(NullWritable.class);
    job.setMapOutputValueClass(NullWritable.class);
    job.setOutputFormatClass(NullOutputFormat.class);
    job.setNumReduceTasks(0);
    job.setPriority(this.jobPriority);

    TableMapReduceUtil.addDependencyJars(job);
    TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), PhoenixConnection.class, Chronology.class,
            CharStream.class, TransactionSystemClient.class, TransactionNotInProgressException.class,
            ZKClient.class, DiscoveryServiceClient.class, ZKDiscoveryService.class,
            Cancellable.class, TTransportException.class, SpanReceiver.class, TransactionProcessor.class, Gauge.class, MetricRegistriesImpl.class);
    LOGGER.info("UpdateStatisticsTool running for: " + tableName
            + " on snapshot: " + snapshotName + " with restore dir: " + restoreDir);
}
Example #12
Source File: DBConfiguration.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public Class<?> getInputClass() {
  return conf.getClass(DBConfiguration.INPUT_CLASS_PROPERTY,
                       NullDBWritable.class);
}
Example #13
Source File: DBConfiguration.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public Class<?> getInputClass() {
  return conf.getClass(DBConfiguration.INPUT_CLASS_PROPERTY,
                       NullDBWritable.class);
}
Example #14
Source File: PhoenixConfigurationUtil.java    From phoenix with Apache License 2.0 4 votes vote down vote up 
public static Class<?> getInputClass(final Configuration configuration) {
    return configuration.getClass(INPUT_CLASS, NullDBWritable.class);
}
Example #15
Source File: PhoenixConfigurationUtil.java    From phoenix with Apache License 2.0 4 votes vote down vote up 
public static Class<?> getInputClass(final Configuration configuration) {
    return configuration.getClass(INPUT_CLASS, NullDBWritable.class);
}
Example #16
Source File: UpdateStatisticsTool.java    From phoenix with Apache License 2.0 4 votes vote down vote up 
@Override
protected void map(NullWritable key, NullDBWritable value,
                   Context context) {
}