Java Code Examples for org.pentaho.di.core.KettleEnvironment#init()

@Test
public void testGetXmlWithAllTypes() throws Exception {
  KettleEnvironment.init();
  Mapping mapping = getMappingWithAllTypes();
  Map<String, HBaseValueMeta> columnsMap = mapping.getMappedColumns();
  String xmlMapping = mapping.getXML();
  DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
  Document document = dbFactory.newDocumentBuilder().parse( new ByteArrayInputStream( xmlMapping.getBytes() ) );
  Element rootElement = document.getDocumentElement();
  NodeList nodeList = rootElement.getElementsByTagName( "mapped_column" );
  for ( int i = 0; i < nodeList.getLength(); i++ ) {
    NodeList childNotes = nodeList.item( i ).getChildNodes();
    String rowKeyName = childNotes.item( 1 ).getTextContent();
    String valueType = childNotes.item( 7 ).getTextContent();
    HBaseValueMeta hBaseValueMeta = columnsMap.get( rowKeyName );
    assertEquals( hBaseValueMeta.getHBaseTypeDesc(), valueType );
  }
}
 

/**
 * Initialize the Kettle environment with settings from the provided configuration
 *
 * @param conf Configuration to configure Kettle environment with
 */
private static void initKettleEnvironment( Configuration conf ) throws KettleException {
  if ( !KettleEnvironment.isInitialized() ) {
    String kettleHome = getKettleHomeProperty( conf );
    String pluginDir = getPluginDirProperty( conf );
    String metaStoreDir = getMetastoreDirProperty( conf );
    System.setProperty( "KETTLE_HOME", kettleHome );
    System.setProperty( Const.PLUGIN_BASE_FOLDERS_PROP, pluginDir );
    System.setProperty( Const.PENTAHO_METASTORE_FOLDER, metaStoreDir );

    KettleEnvironment.init();

    log.logBasic( BaseMessages.getString( MRUtil.class, "KettleHome.Info", kettleHome ) );
    log.logBasic( BaseMessages.getString( MRUtil.class, "PluginDirectory.Info", pluginDir ) );
    log.logBasic( BaseMessages.getString( MRUtil.class, "MetasStoreDirectory.Info", metaStoreDir ) );

  }
}
 

@BeforeClass
public static void createServer() throws Exception {
  KettleEnvironment.init();

  server = FtpsServer.createDefaultServer();
  server.start();

  ramDir = "ram://" + JobEntryFTPSGetIT.class.getSimpleName();
  KettleVFS.getFileObject( ramDir ).createFolder();
}
 

public static DatabaseMeta convertToLegacy( String name, SqlDataSource datasource ) {
  // make sure that the Kettle environment is initialized before DatabaseMeta creation
  try {
    KettleEnvironment.init( false );
  } catch ( KettleException e ) {
    logger.error( "Error initializing the Kettle Environment", e );
    throw new RuntimeException( "Error initializing the Kettle Environment", e );
  }

  DatabaseMeta databaseMeta = new DatabaseMeta();

  databaseMeta.setName( name );

  databaseMeta.setHostname( datasource.getHostname() );
  if ( datasource.getDialectType() == null ) {
    // default to mysql if dialect is null
    databaseMeta.setDatabaseType( "GENERIC" ); //$NON-NLS-1$
  } else {
    databaseMeta.setDatabaseType( datasource.getDialectType() );
  }
  databaseMeta.setAccessType( DatabaseMeta.getAccessType( datasource.getType().toString() ) );
  databaseMeta.setDBName( datasource.getDatabaseName() );
  databaseMeta.setDBPort( datasource.getPort() );
  databaseMeta.setUsername( datasource.getUsername() );
  databaseMeta.setPassword( datasource.getPassword() );
  databaseMeta.setServername( datasource.getServername() );

  // And now load the attributes...
  for ( String key : datasource.getAttributes().keySet() ) {
    databaseMeta.getAttributes().put( key, datasource.getAttributes().get( key ) );
  }
  return databaseMeta;
}
 

@Before
public void setUpLoadSave() throws Exception {
  KettleEnvironment.init();
  PluginRegistry.init( false );
  List<String> attributes =
      Arrays.asList( "conditionReceivedDate", "valueimaplist", "serverName", "userName", "password", "useSSL", "port",
          "firstMails", "retrievemails", "delete", "protocol", "firstIMAPMails", "IMAPFolder", "senderSearchTerm",
          "notTermSenderSearch", "recipientSearch", "subjectSearch", "receivedDate1", "receivedDate2",
          "notTermSubjectSearch", "notTermRecipientSearch", "notTermReceivedDateSearch", "includeSubFolders", "useProxy",
          "proxyUsername", "folderField", "dynamicFolder", "rowLimit", "useBatch", "start", "end", "batchSize",
          "stopOnError", "inputFields" );

  Map<String, String> getterMap = new HashMap<String, String>();
  Map<String, String> setterMap = new HashMap<String, String>();

  Map<String, FieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>();
  attrValidatorMap.put( "inputFields",
      new ArrayLoadSaveValidator<MailInputField>( new MailInputFieldLoadSaveValidator(), 5 ) );
  attrValidatorMap.put( "batchSize", new IntLoadSaveValidator( 1000 ) );
  attrValidatorMap.put( "conditionReceivedDate", new IntLoadSaveValidator( MailConnectionMeta.conditionDateCode.length ) );
  attrValidatorMap.put( "valueimaplist", new IntLoadSaveValidator( MailConnectionMeta.valueIMAPListCode.length ) );
  attrValidatorMap.put( "port", new StringIntLoadSaveValidator( 65534 ) );

  Map<String, FieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>();

  loadSaveTester =
      new LoadSaveTester( testMetaClass, attributes, new ArrayList<String>(), new ArrayList<String>(),
          getterMap, setterMap, attrValidatorMap, typeValidatorMap, this );
}
 

@Before
public void setUpLoadSave() throws Exception {
  KettleEnvironment.init();
  PluginRegistry.init( false );
  List<String> attributes =
      Arrays.asList( "currency", "decimal", "group", "fieldName", "fieldType", "fieldFormat", "fieldLength",
          "fieldPrecision", "setEmptyString", "dataLines", "fieldNullIf" );

  Map<String, String> getterMap = new HashMap<>();
  Map<String, String> setterMap = new HashMap<String, String>() {
    {
      put( "setEmptyString", "setEmptyString" );
    }
  };
  FieldLoadSaveValidator<String[]> stringArrayLoadSaveValidator =
      new ArrayLoadSaveValidator<String>( new StringLoadSaveValidator(), 3 );

  Map<String, FieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>();
  attrValidatorMap.put( "currency", stringArrayLoadSaveValidator );
  attrValidatorMap.put( "decimal", stringArrayLoadSaveValidator );
  attrValidatorMap.put( "group", stringArrayLoadSaveValidator );
  attrValidatorMap.put( "fieldName", stringArrayLoadSaveValidator );
  attrValidatorMap.put( "fieldType", stringArrayLoadSaveValidator );
  attrValidatorMap.put( "fieldFormat", stringArrayLoadSaveValidator );
  attrValidatorMap.put( "fieldLength",
    new PrimitiveIntArrayLoadSaveValidator( new IntLoadSaveValidator( 75 ), 3 ) );
  attrValidatorMap.put( "fieldPrecision",
      new PrimitiveIntArrayLoadSaveValidator( new IntLoadSaveValidator( 9 ), 3 ) );
  attrValidatorMap.put( "setEmptyString",
       new PrimitiveBooleanArrayLoadSaveValidator( new BooleanLoadSaveValidator(), 3 ) );
  attrValidatorMap.put( "dataLines", new DataGridLinesLoadSaveValidator() );
  attrValidatorMap.put( "fieldNullIf", stringArrayLoadSaveValidator );

  Map<String, FieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>();

  loadSaveTester =
      new LoadSaveTester( testMetaClass, attributes, new ArrayList<String>(), new ArrayList<String>(),
          getterMap, setterMap, attrValidatorMap, typeValidatorMap, this );
}
 

@Test
public void testGetDataFromFolderRecursively() throws KettleException {
  KettleEnvironment.init();
  String path = getClass().getResource( "text-file-input-get-data-from-folder-step.ktr" ).getPath();
  Variables variables = new Variables();
  variables.setVariable( "testfolder", getClass().getResource( "" ).getPath() );
  TransMeta transMeta = new TransMeta( path, variables );
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );
  trans.startThreads();
  trans.waitUntilFinished();
  assertEquals( 14, trans.getSteps().get( 0 ).step.getLinesWritten() );
  assertEquals( 21, trans.getSteps().get( 0 ).step.getLinesInput() );
}
 

@BeforeClass
public static void before() throws KettleException {
  KettleEnvironment.init();
}
 

@Override
protected void setUp() throws Exception {
  KettleEnvironment.init();
}
 

@BeforeClass
public static void initKettle() throws Exception {
  KettleEnvironment.init();
}
 

private static void setKettleEnvironment() throws Exception {
  KettleClientEnvironment.getInstance().setClient( KettleClientEnvironment.ClientType.CARTE );
  KettleEnvironment.init();
}
 

@BeforeClass
public static void setUpBeforeClass() throws KettleException {
  KettleEnvironment.init( false );
}
 

@BeforeClass
public static void init() throws KettleException {
  KettleEnvironment.init( false );
}
 

@BeforeClass
public static void initKettle() throws Exception {
  KettleEnvironment.init();
}
 

/**
 * Test case for normal table output where the table is included in the instream, but the tablename is not stored in
 * the table.
 */
public void testTableOutputJIRA897() throws Exception {
  KettleEnvironment.init();

  //
  // Create a new transformation...
  //
  TransMeta transMeta = new TransMeta();
  transMeta.setName( "table output JIRA897 test" );

  // Add the database connections
  for ( int i = 0; i < databasesXML.length; i++ ) {
    DatabaseMeta databaseMeta = new DatabaseMeta( databasesXML[i] );
    transMeta.addDatabase( databaseMeta );
  }

  DatabaseMeta dbInfo = transMeta.findDatabase( "db" );

  // Execute our setup SQLs in the database.
  Database database = new Database( transMeta, dbInfo );
  database.connect();
  createTable( database, target_table1, createSourceRowMetaInterface1() );
  createTable( database, target_table2, createSourceRowMetaInterface1() );

  PluginRegistry registry = PluginRegistry.getInstance();

  //
  // create an injector step...
  //
  String injectorStepname = "injector step";
  InjectorMeta im = new InjectorMeta();

  // Set the information of the injector.
  String injectorPid = registry.getPluginId( StepPluginType.class, im );
  StepMeta injectorStep = new StepMeta( injectorPid, injectorStepname, im );
  transMeta.addStep( injectorStep );

  //
  // create the source step...
  //
  String outputname = "output to [" + target_table1 + "] and [" + target_table2 + "]";
  TableOutputMeta tom = new TableOutputMeta();
  tom.setDatabaseMeta( transMeta.findDatabase( "db" ) );
  tom.setTableNameInField( true );
  tom.setTableNameField( "TABLE" );
  tom.setTableNameInTable( false );

  String fromid = registry.getPluginId( StepPluginType.class, tom );
  StepMeta fromstep = new StepMeta( fromid, outputname, tom );
  fromstep.setDescription( "write data to tables on database [" + dbInfo + "]" );
  transMeta.addStep( fromstep );

  TransHopMeta hi = new TransHopMeta( injectorStep, fromstep );
  transMeta.addTransHop( hi );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( outputname, 0 );
  RowStepCollector rc = new RowStepCollector();
  si.addRowListener( rc );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();

  // add rows
  List<RowMetaAndData> inputList = createJIRA897DataRows();
  for ( RowMetaAndData rm : inputList ) {
    rp.putRow( rm.getRowMeta(), rm.getData() );
  }
  rp.finished();

  trans.waitUntilFinished();

  List<RowMetaAndData> resultRows = rc.getRowsWritten();

  // The name of the table should still be in here.
  List<RowMetaAndData> goldRows = createJIRA897DataRows();
  checkRows( goldRows, resultRows );
  checkResultsJIRA897( database );
}
 

@BeforeClass
public static void setUpClass() throws Exception {
  KettleEnvironment.init();
}
 

@BeforeClass
public static void initKettle() throws Exception {
  KettleEnvironment.init();
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  KettleEnvironment.init( false );
}
 

@BeforeClass
public static void initKettle() throws Exception {
  KettleEnvironment.init();
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  KettleEnvironment.init();
}