Java Code Examples for org.apache.sqoop.model.MJob#getDriverConfig()

@SuppressWarnings("unchecked")
private JSONObject extractJob(boolean skipSensitive, MJob job) {
  JSONObject object = new JSONObject();
  object.put(ID, job.getPersistenceId());
  object.put(NAME, job.getName());
  object.put(ENABLED, job.getEnabled());
  object.put(CREATION_USER, job.getCreationUser());
  object.put(CREATION_DATE, job.getCreationDate().getTime());
  object.put(UPDATE_USER, job.getLastUpdateUser());
  object.put(UPDATE_DATE, job.getLastUpdateDate().getTime());
  // job link associated connectors
  // TODO(SQOOP-1634): fix not to require the connectorIds in the post data
  object.put(FROM_CONNECTOR_ID, job.getConnectorId(Direction.FROM));
  object.put(TO_CONNECTOR_ID, job.getConnectorId(Direction.TO));
  // job associated links
  object.put(FROM_LINK_ID, job.getLinkId(Direction.FROM));
  object.put(TO_LINK_ID, job.getLinkId(Direction.TO));
  // job configs
  MFromConfig fromConfigList = job.getFromJobConfig();
  object.put(FROM_CONFIG_VALUES,
      extractConfigList(fromConfigList.getConfigs(), fromConfigList.getType(), skipSensitive));
  MToConfig toConfigList = job.getToJobConfig();
  object.put(TO_CONFIG_VALUES,
      extractConfigList(toConfigList.getConfigs(), toConfigList.getType(), skipSensitive));
  MDriverConfig driverConfigList = job.getDriverConfig();
  object.put(
      DRIVER_CONFIG_VALUES,
      extractConfigList(driverConfigList.getConfigs(), driverConfigList.getType(),
          skipSensitive));

  return object;
}
 

@Test
public void testCities() throws Exception {
  // Job creation
  MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), kiteLink.getPersistenceId());

  // Set rdbms "FROM" config
  MConfigList fromConfig = job.getJobConfig(Direction.FROM);
  fillRdbmsFromConfig(job, "id");
  // TODO: Kite have troubles with some data types, so we're limiting the columns to int only
  fromConfig.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id"));

  // Fill the Kite "TO" config
  MConfigList toConfig = job.getJobConfig(Direction.TO);
  toConfig.getStringInput("toJobConfig.uri").setValue("dataset:hive:testtable");
  toConfig.getEnumInput("toJobConfig.fileFormat").setValue(FileFormat.AVRO);

  // driver config
  MDriverConfig driverConfig = job.getDriverConfig();
  driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(1);

  saveJob(job);

  executeJob(job);

  // Assert correct output
  ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 1}, "1");
  ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 2}, "2");
  ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 3}, "3");
  ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 4}, "4");

  hiveProvider.dropTable(new TableName("testtable"));
}
 

@Test
public void testBasic() throws Exception {
  createHdfsFromFile("input-0001",input);

  // Create Kafka link
  MLink kafkaLink = getClient().createLink("kafka-connector");
  fillKafkaLinkConfig(kafkaLink);
  saveLink(kafkaLink);

  // HDFS link
  MLink hdfsLink = getClient().createLink("hdfs-connector");
  fillHdfsLink(hdfsLink);
  saveLink(hdfsLink);

  // Job creation
  MJob job = getClient().createJob(hdfsLink.getPersistenceId(), kafkaLink.getPersistenceId());

  // Job connector configs
  fillHdfsFromConfig(job);
  fillKafkaToConfig(job);

  // driver config
  MDriverConfig driverConfig = job.getDriverConfig();
  driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3);
  saveJob(job);

  executeJob(job);

  // this will assert the content of the array matches the content of the topic
  validateContent(input);
}
 

@Test
public void testBasic() throws Exception {
  createAndLoadTableCities();

  // Kafka link
  MLink kafkaLink = getClient().createLink("kafka-connector");
  fillKafkaLinkConfig(kafkaLink);
  saveLink(kafkaLink);

  // RDBMS link
  MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
  fillRdbmsLinkConfig(rdbmsLink);
  saveLink(rdbmsLink);

  // Job creation
  MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), kafkaLink.getPersistenceId());

  // set rdbms "FROM" job config
  fillRdbmsFromConfig(job, "id");

  // set Kafka  "TO" job config
  fillKafkaToConfig(job);

  // driver config
  MDriverConfig driverConfig = job.getDriverConfig();
  driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3);
  saveJob(job);

  executeJob(job);

  // this will assert the content of the array matches the content of the topic
  validateContent(input);
}
 

public static void main(String[] args) throws Exception {

    final SqoopSparkJob sparkJob = new SqoopSparkJob();
    CommandLine cArgs = SqoopSparkJob.parseArgs(createOptions(), args);
    SparkConf conf = sparkJob.init(cArgs);
    JavaSparkContext context = new JavaSparkContext(conf);

    MConnector fromConnector = RepositoryManager.getInstance().getRepository()
        .findConnector("generic-jdbc-connector");

    MLinkConfig fromLinkConfig = fromConnector.getLinkConfig();
    MLink fromLink = new MLink(fromConnector.getPersistenceId(), fromLinkConfig);
    fromLink.setName("jdbcLink-" + System.currentTimeMillis());

    fromLink.getConnectorLinkConfig().getStringInput("linkConfig.jdbcDriver")
        .setValue("com.mysql.jdbc.Driver");

    fromLink.getConnectorLinkConfig().getStringInput("linkConfig.connectionString")
        .setValue(cArgs.getOptionValue("jdbcString"));
    fromLink.getConnectorLinkConfig().getStringInput("linkConfig.username")
        .setValue(cArgs.getOptionValue("u"));
    fromLink.getConnectorLinkConfig().getStringInput("linkConfig.password")
        .setValue(cArgs.getOptionValue("p"));

    RepositoryManager.getInstance().getRepository().createLink(fromLink);

    MConnector toConnector = RepositoryManager.getInstance().getRepository()
        .findConnector("kafka-connector");

    MLinkConfig toLinkConfig = toConnector.getLinkConfig();

    MLink toLink = new MLink(toConnector.getPersistenceId(), toLinkConfig);
    toLink.setName("kafkaLink-" + System.currentTimeMillis());

    toLink.getConnectorLinkConfig().getStringInput("linkConfig.brokerList")
        .setValue(cArgs.getOptionValue("broker"));
    toLink.getConnectorLinkConfig().getStringInput("linkConfig.zookeeperConnect")
        .setValue(cArgs.getOptionValue("zk"));

    RepositoryManager.getInstance().getRepository().createLink(toLink);

    MFromConfig fromJobConfig = fromConnector.getFromConfig();
    MToConfig toJobConfig = toConnector.getToConfig();

    MJob sqoopJob = new MJob(fromConnector.getPersistenceId(), toConnector.getPersistenceId(),
        fromLink.getPersistenceId(), toLink.getPersistenceId(), fromJobConfig, toJobConfig, Driver
            .getInstance().getDriver().getDriverConfig());
    // jdbc configs
    MFromConfig fromConfig = sqoopJob.getFromJobConfig();
    fromConfig.getStringInput("fromJobConfig.tableName").setValue(cArgs.getOptionValue("table"));
    fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(cArgs.getOptionValue("partitionCol"));
    // kafka configs
    MToConfig toConfig = sqoopJob.getToJobConfig();
    toConfig.getStringInput("toJobConfig.topic").setValue("test-spark-topic");

    MDriverConfig driverConfig = sqoopJob.getDriverConfig();
    if (cArgs.getOptionValue("numE") != null) {
      driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(
          Integer.valueOf(cArgs.getOptionValue("numE")));
    }
    if (cArgs.getOptionValue("numL") != null) {

      driverConfig.getIntegerInput("throttlingConfig.numLoaders").setValue(
          Integer.valueOf(cArgs.getOptionValue("numL")));
    }    RepositoryManager.getInstance().getRepository().createJob(sqoopJob);
    sparkJob.setJob(sqoopJob);
    sparkJob.execute(conf, context);
  }
 

public static void main(String[] args) throws Exception {

    final SqoopSparkJob sparkJob = new SqoopSparkJob();
    CommandLine cArgs = SqoopSparkJob.parseArgs(createOptions(), args);
    SparkConf conf = sparkJob.init(cArgs);
    JavaSparkContext context = new JavaSparkContext(conf);

    MConnector fromConnector = RepositoryManager.getInstance().getRepository()
        .findConnector("generic-jdbc-connector");
    MConnector toConnector = RepositoryManager.getInstance().getRepository()
        .findConnector("hdfs-connector");

    MLinkConfig fromLinkConfig = fromConnector.getLinkConfig();
    MLinkConfig toLinkConfig = toConnector.getLinkConfig();

    MLink fromLink = new MLink(fromConnector.getPersistenceId(), fromLinkConfig);
    fromLink.setName("jdbcLink-" + System.currentTimeMillis());
    fromLink.getConnectorLinkConfig().getStringInput("linkConfig.jdbcDriver")
        .setValue("com.mysql.jdbc.Driver");

    fromLink.getConnectorLinkConfig().getStringInput("linkConfig.connectionString")
        .setValue(cArgs.getOptionValue("jdbcString"));
    fromLink.getConnectorLinkConfig().getStringInput("linkConfig.username")
        .setValue(cArgs.getOptionValue("u"));
    fromLink.getConnectorLinkConfig().getStringInput("linkConfig.password")
        .setValue(cArgs.getOptionValue("p"));
    RepositoryManager.getInstance().getRepository().createLink(fromLink);

    MLink toLink = new MLink(toConnector.getPersistenceId(), toLinkConfig);
    toLink.setName("hdfsLink-" + System.currentTimeMillis());
    toLink.getConnectorLinkConfig().getStringInput("linkConfig.confDir")
        .setValue(cArgs.getOptionValue("outputDir"));
    RepositoryManager.getInstance().getRepository().createLink(toLink);

    MFromConfig fromJobConfig = fromConnector.getFromConfig();
    MToConfig toJobConfig = toConnector.getToConfig();

    MJob sqoopJob = new MJob(fromConnector.getPersistenceId(), toConnector.getPersistenceId(),
        fromLink.getPersistenceId(), toLink.getPersistenceId(), fromJobConfig, toJobConfig, Driver
            .getInstance().getDriver().getDriverConfig());

    MConfigList fromConfig = sqoopJob.getJobConfig(Direction.FROM);
    fromConfig.getStringInput("fromJobConfig.tableName").setValue(cArgs.getOptionValue("table"));
    fromConfig.getStringInput("fromJobConfig.partitionColumn").setValue(
        cArgs.getOptionValue("paritionCol"));

    MToConfig toConfig = sqoopJob.getToJobConfig();
    toConfig.getStringInput("toJobConfig.outputDirectory").setValue(
        cArgs.getOptionValue("outputDir") + System.currentTimeMillis());
    MDriverConfig driverConfig = sqoopJob.getDriverConfig();
    if (cArgs.getOptionValue("numE") != null) {
      driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(
          Integer.valueOf(cArgs.getOptionValue("numE")));
    }
    if (cArgs.getOptionValue("numL") != null) {

      driverConfig.getIntegerInput("throttlingConfig.numLoaders").setValue(
          Integer.valueOf(cArgs.getOptionValue("numL")));
    }
    RepositoryManager.getInstance().getRepository().createJob(sqoopJob);
    sparkJob.setJob(sqoopJob);
    sparkJob.execute(conf, context);
  }
 

@Test
public void testStagedTransfer() throws Exception {
  final TableName stageTableName = new TableName("STAGE_" + getTableName());
  createTableCities();
  createHdfsFromFile("input-0001",
      "1,'USA','2004-10-23','San Francisco'",
      "2,'USA','2004-10-24','Sunnyvale'",
      "3,'Czech Republic','2004-10-25','Brno'",
      "4,'USA','2004-10-26','Palo Alto'"
    );
  new Cities(provider, stageTableName).createTables();

  // RDBMS link
  MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
  fillRdbmsLinkConfig(rdbmsLink);
  saveLink(rdbmsLink);

  // HDFS link
  MLink hdfsLink = getClient().createLink("hdfs-connector");
  fillHdfsLink(hdfsLink);
  saveLink(hdfsLink);

  // Job creation
  MJob job = getClient().createJob(hdfsLink.getPersistenceId(),
      rdbmsLink.getPersistenceId());

  // fill HDFS "FROM" config
  fillHdfsFromConfig(job);

  // fill rdbms "TO" config here
  fillRdbmsToConfig(job);
  MConfigList configs = job.getJobConfig(Direction.TO);
  configs.getStringInput("toJobConfig.stageTableName").setValue(provider.escapeTableName(stageTableName.getTableName()));

  // driver config
  MConfigList driverConfig = job.getDriverConfig();
  driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3);

  saveJob(job);

  executeJob(job);

  assertEquals(0L, provider.rowCount(stageTableName));
  assertEquals(4L, provider.rowCount(getTableName()));
  assertRowInCities(1, "USA", "2004-10-23", "San Francisco");
  assertRowInCities(2, "USA", "2004-10-24", "Sunnyvale");
  assertRowInCities(3, "Czech Republic", "2004-10-25", "Brno");
  assertRowInCities(4, "USA", "2004-10-26", "Palo Alto");

  // Clean up testing table
  provider.dropTable(stageTableName);
  dropTable();
}
 

@Test
public void testFrom() throws Exception {
  createTable("id",
    "id", "INT",
    "value", type.name
  );

  int i = 1;
  for(ExampleValue value: type.values) {
    insertRow(false, Integer.toString(i++), value.insertStatement);
  }

  // RDBMS link
  MLink rdbmsConnection = getClient().createLink("generic-jdbc-connector");
  fillRdbmsLinkConfig(rdbmsConnection);
  saveLink(rdbmsConnection);

  // HDFS link
  MLink hdfsConnection = getClient().createLink("hdfs-connector");
  fillHdfsLink(hdfsConnection);
  saveLink(hdfsConnection);

  // Job creation
  MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId());

  // Fill rdbms "FROM" config
  fillRdbmsFromConfig(job, "id");
  MConfigList fromConfig = job.getJobConfig(Direction.FROM);
  fromConfig.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("value"));

  // Fill the hdfs "TO" config
  fillHdfsToConfig(job, ToFormat.TEXT_FILE);

  // driver config
  MDriverConfig driverConfig = job.getDriverConfig();
  driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(1);

  saveJob(job);
  executeJob(job);

  // Assert correct output
  assertHdfsTo(type.escapedStringValues());

  // Clean up testing table
  dropTable();
}
 

@Test
public void testTo() throws Exception {
  createTable(null,
    "value", type.name
  );

  createHdfsFromFile("input-0001", type.escapedStringValues());

  // RDBMS link
  MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
  fillRdbmsLinkConfig(rdbmsLink);
  saveLink(rdbmsLink);

  // HDFS link
  MLink hdfsLink = getClient().createLink("hdfs-connector");
  fillHdfsLink(hdfsLink);
  saveLink(hdfsLink);

  // Job creation
  MJob job = getClient().createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId());
  fillHdfsFromConfig(job);

  // Set the rdbms "TO" config here
  fillRdbmsToConfig(job);

  // Driver config
  MDriverConfig driverConfig = job.getDriverConfig();
  driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(1);

  saveJob(job);
  executeJob(job);
  dumpTable();

  assertEquals(type.values.size(), rowCount());
  for(ExampleValue value : type.values) {
    assertRow(
      new Object[] {"value", value.insertStatement},
      false,
      value.objectValue);
  }

  // Clean up testing table
  dropTable();
}
 

@Test
public void testSplitter() throws Exception {
  createAndLoadTableUbuntuReleases();

  // RDBMS link
  MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
  fillRdbmsLinkConfig(rdbmsLink);
  saveLink(rdbmsLink);

  // HDFS link
  MLink hdfsLink = getClient().createLink("hdfs-connector");
  fillHdfsLink(hdfsLink);
  saveLink(hdfsLink);

  // Job creation
  MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), hdfsLink.getPersistenceId());

  // set the rdbms "FROM" config
  fillRdbmsFromConfig(job, partitionColumn);

  // fill hdfs "TO" config
  fillHdfsToConfig(job, ToFormat.TEXT_FILE);

  // set driver config
  MDriverConfig driverConfig = job.getDriverConfig();
  driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(extractors);

  saveJob(job);

  executeJob(job);

  // Assert correct output
  assertHdfsToFiles((extractors > maxOutputFiles) ? maxOutputFiles : extractors);
  assertHdfsTo(
      "1,'Warty Warthog',4.10,'2004-10-20',false",
      "2,'Hoary Hedgehog',5.04,'2005-04-08',false",
      "3,'Breezy Badger',5.10,'2005-10-13',false",
      "4,'Dapper Drake',6.06,'2006-06-01',true",
      "5,'Edgy Eft',6.10,'2006-10-26',false",
      "6,'Feisty Fawn',7.04,'2007-04-19',false",
      "7,'Gutsy Gibbon',7.10,'2007-10-18',false",
      "8,'Hardy Heron',8.04,'2008-04-24',true",
      "9,'Intrepid Ibex',8.10,'2008-10-18',false",
      "10,'Jaunty Jackalope',9.04,'2009-04-23',false",
      "11,'Karmic Koala',9.10,'2009-10-29',false",
      "12,'Lucid Lynx',10.04,'2010-04-29',true",
      "13,'Maverick Meerkat',10.10,'2010-10-10',false",
      "14,'Natty Narwhal',11.04,'2011-04-28',false",
      "15,'Oneiric Ocelot',11.10,'2011-10-10',false",
      "16,'Precise Pangolin',12.04,'2012-04-26',true",
      "17,'Quantal Quetzal',12.10,'2012-10-18',false",
      "18,'Raring Ringtail',13.04,'2013-04-25',false",
      "19,'Saucy Salamander',13.10,'2013-10-17',false"
    );

  // Clean up testing table
  dropTable();
}
 

@Test
public void testCities() throws Exception {
  createAndLoadTableCities();

  // RDBMS link
  MLink rdbmsConnection = getClient().createLink("generic-jdbc-connector");
  fillRdbmsLinkConfig(rdbmsConnection);
  saveLink(rdbmsConnection);

  // HDFS link
  MLink hdfsConnection = getClient().createLink("hdfs-connector");
  fillHdfsLink(hdfsConnection);
  saveLink(hdfsConnection);

  // Job creation
  MJob job = getClient().createJob(rdbmsConnection.getPersistenceId(), hdfsConnection.getPersistenceId());

  // Set rdbms "FROM" config
  fillRdbmsFromConfig(job, "id");

  // fill the hdfs "TO" config
  fillHdfsToConfig(job, ToFormat.TEXT_FILE);
  // driver config
  MDriverConfig driverConfig = job.getDriverConfig();
  driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3);

  saveJob(job);

  executeJob(job);

  // Assert correct output
  assertHdfsTo(
    "1,'USA','2004-10-23','San Francisco'",
    "2,'USA','2004-10-24','Sunnyvale'",
    "3,'Czech Republic','2004-10-25','Brno'",
    "4,'USA','2004-10-26','Palo Alto'"
  );

  // Clean up testing table
  dropTable();
}
 

@Test
public void testBasic() throws Exception {
  createHdfsFromFile("input-0001",
      "1,'USA','2004-10-23','San Francisco'",
      "2,'USA','2004-10-24','Sunnyvale'",
      "3,'Czech Republic','2004-10-25','Brno'",
      "4,'USA','2004-10-26','Palo Alto'"
  );

  // RDBMS link
  MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
  fillRdbmsLinkConfig(rdbmsLink);
  saveLink(rdbmsLink);

  // HDFS link
  MLink hdfsLink = getClient().createLink("hdfs-connector");
  fillHdfsLink(hdfsLink);
  saveLink(hdfsLink);

  // Job creation
  MJob job = getClient().createJob(hdfsLink.getPersistenceId(), rdbmsLink.getPersistenceId());

  // set hdfs "FROM" config for the job, since the connector test case base class only has utilities for hdfs!
  fillHdfsFromConfig(job);

  // set the rdbms "TO" config here
  fillRdbmsToConfig(job);

  // driver config
  MDriverConfig driverConfig = job.getDriverConfig();
  driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3);
  saveJob(job);

  executeJob(job);

  assertEquals(4L, provider.rowCount(getTableName()));
  assertRowInCities(1, "USA", "2004-10-23", "San Francisco");
  assertRowInCities(2, "USA", "2004-10-24", "Sunnyvale");
  assertRowInCities(3, "Czech Republic", "2004-10-25", "Brno");
  assertRowInCities(4, "USA", "2004-10-26", "Palo Alto");
}
 

@Test
public void testCities() throws Exception {
  // RDBMS link
  MLink rdbmsLink = getClient().createLink("generic-jdbc-connector");
  fillRdbmsLinkConfig(rdbmsLink);
  saveLink(rdbmsLink);

  // Kite link
  MLink kiteLink = getClient().createLink("kite-connector");
  kiteLink.getConnectorLinkConfig().getStringInput("linkConfig.authority").setValue(hdfsClient.getUri().getAuthority());
  saveLink(kiteLink);

  // Job creation
  MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), kiteLink.getPersistenceId());

  // Set rdbms "FROM" config
  fillRdbmsFromConfig(job, "id");
  // TODO: Kite have troubles with some data types, so we're limiting the columns to int only
  MConfigList fromConfig = job.getJobConfig(Direction.FROM);
  fromConfig.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id"));

  // Fill the Kite "TO" config
  MConfigList toConfig = job.getJobConfig(Direction.TO);
  toConfig.getStringInput("toJobConfig.uri").setValue("dataset:hdfs:" + getHadoopTestDirectory());
  toConfig.getEnumInput("toJobConfig.fileFormat").setValue(FileFormat.CSV);

  // driver config
  MDriverConfig driverConfig = job.getDriverConfig();
  driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(1);

  saveJob(job);

  executeJob(job);

  // Assert correct output
  assertHdfsTo(
    "\"1\"",
    "\"2\"",
    "\"3\"",
    "\"4\""
  );
}