Java Code Examples for org.pentaho.di.trans.TransMeta#setMetaStore()

The following examples show how to use org.pentaho.di.trans.TransMeta#setMetaStore() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: RunBeamTransExecutionPoint.java From kettle-beam with Apache License 2.0

6 votes

private TransMeta copyCleanTransMeta( TransMeta transMeta, Repository repository, IMetaStore metaStore, VariableSpace space ) throws KettleException {

    try {
      String transMetaXml = transMeta.getXML();
      TransMeta copy = new TransMeta();
      copy.setMetaStore( metaStore );
      copy.loadXML( XMLHandler.loadXMLString( transMetaXml, TransMeta.XML_TAG ), null, metaStore, repository, true, space, null );

      for ( StepMeta stepMeta : copy.getSteps() ) {
        stepMeta.setCopies( 1 );

        DummyTransMeta dummyTransMeta = new DummyTransMeta();

        // Replace all stepMeta with a Dummy with the same name
        //
        stepMeta.setStepID( "Dummy" );
        stepMeta.setStepMetaInterface( dummyTransMeta );
      }

      return copy;

    } catch ( Exception e ) {
      throw new KettleException( "Error copying/cleaning transformation metadata", e );
    }
  }

Example 2

Source File: KettleFileRepository.java From pentaho-kettle with Apache License 2.0

6 votes

@Override
public TransMeta loadTransformation( String transname, RepositoryDirectoryInterface repdir,
  ProgressMonitorListener monitor, boolean setInternalVariables, String versionName ) throws KettleException {

  // This is a standard load of a transformation serialized in XML...
  //
  String filename = calcDirectoryName( repdir ) + transname + ".ktr";
  TransMeta transMeta = new TransMeta( filename, this, setInternalVariables );
  transMeta.setRepository( this );
  transMeta.setMetaStore( getMetaStore() );
  transMeta.setFilename( null );
  transMeta.setName( transname );
  transMeta.setObjectId( new StringObjectId( calcObjectId( repdir, transname, EXT_TRANSFORMATION ) ) );

  readDatabases( transMeta, true );
  transMeta.clearChanged();

  return transMeta;
}

Example 3

Source File: PurRepository.java From pentaho-kettle with Apache License 2.0

6 votes

private TransMeta buildTransMeta( final RepositoryFile file, final RepositoryDirectoryInterface parentDir,
                                  final NodeRepositoryFileData data, final ObjectRevision revision )
  throws KettleException {
  TransMeta transMeta = new TransMeta();
  transMeta.setName( file.getTitle() );
  transMeta.setFilename( file.getName() );
  transMeta.setDescription( file.getDescription() );
  transMeta.setObjectId( new StringObjectId( file.getId().toString() ) );
  transMeta.setObjectRevision( revision );
  transMeta.setRepository( this );
  transMeta.setRepositoryDirectory( parentDir );
  transMeta.setMetaStore( getMetaStore() );
  readTransSharedObjects( transMeta ); // This should read from the local cache
  transDelegate.dataNodeToElement( data.getNode(), transMeta );
  transMeta.clearChanged();
  return transMeta;
}

Example 4

Source File: MainBeam.java From kettle-beam with Apache License 2.0

4 votes

public static final int mainMethod( final String[] args, final String environment ) {

    try {
      System.out.println( "Starting clustered transformation execution on environment: '"+environment+"'" );

      System.out.println( "Transformation ktr / args[0] : " + args[ 0 ] );
      System.out.println( "MetaStore JSON     / args[1] : " + args[ 1 ] );
      System.out.println( "Beam Job Config    / args[2] : " + args[ 2 ] );

      // Read the transformation XML and MetaStore from Hadoop FS
      //
      Configuration hadoopConfiguration = new Configuration();
      String transMetaXml = readFileIntoString( args[ 0 ], hadoopConfiguration, "UTF-8" );
      String metaStoreJson = readFileIntoString( args[ 1 ], hadoopConfiguration, "UTF-8" );

      // Third argument: the beam job config
      //
      String jobConfigName = args[ 2 ];

      // Inflate the metaStore...
      //
      IMetaStore metaStore = new SerializableMetaStore( metaStoreJson );

      System.out.println( ">>>>>> Loading Kettle Beam Job Config '" + jobConfigName + "'" );
      MetaStoreFactory<BeamJobConfig> configFactory = new MetaStoreFactory<>( BeamJobConfig.class, metaStore, PentahoDefaults.NAMESPACE );
      BeamJobConfig jobConfig = configFactory.loadElement( jobConfigName );

      List<String> stepPluginsList = new ArrayList<>( Arrays.asList( Const.NVL(jobConfig.getStepPluginClasses(), "").split( "," ) ) );
      List<String> xpPluginsList = new ArrayList<>( Arrays.asList( Const.NVL(jobConfig.getXpPluginClasses(), "").split( "," ) ) );

      System.out.println( ">>>>>> Initializing Kettle runtime (" + stepPluginsList.size() + " step classes, " + xpPluginsList.size() + " XP classes)" );

      BeamKettle.init( stepPluginsList, xpPluginsList );

      System.out.println( ">>>>>> Loading transformation metadata" );
      TransMeta transMeta = new TransMeta( XMLHandler.loadXMLString( transMetaXml, TransMeta.XML_TAG ), null );
      transMeta.setMetaStore( metaStore );


      String hadoopConfDir = System.getenv( "HADOOP_CONF_DIR" );
      System.out.println( ">>>>>> HADOOP_CONF_DIR='" + hadoopConfDir + "'" );

      System.out.println( ">>>>>> Building Apache Beam Kettle Pipeline..." );
      PluginRegistry registry = PluginRegistry.getInstance();
      PluginInterface beamInputPlugin = registry.getPlugin( StepPluginType.class, BeamConst.STRING_BEAM_INPUT_PLUGIN_ID );
      if ( beamInputPlugin != null ) {
        System.out.println( ">>>>>> Found Beam Input step plugin class loader" );
      } else {
        throw new KettleException( "Unable to find Beam Input step plugin, bailing out!" );
      }
      ClassLoader pluginClassLoader = PluginRegistry.getInstance().getClassLoader( beamInputPlugin );
      if ( pluginClassLoader != null ) {
        System.out.println( ">>>>>> Found Beam Input step plugin class loader" );
      } else {
        System.out.println( ">>>>>> NOT found Beam Input step plugin class loader, using system classloader" );
        pluginClassLoader = ClassLoader.getSystemClassLoader();
      }
      KettleBeamPipelineExecutor executor = new KettleBeamPipelineExecutor( LogChannel.GENERAL, transMeta, jobConfig, metaStore, pluginClassLoader, stepPluginsList, xpPluginsList );

      System.out.println( ">>>>>> Pipeline executing starting..." );
      executor.setLoggingMetrics( true );
      executor.execute( true );
      System.out.println( ">>>>>> Execution finished..." );
      return 0;
    } catch ( Exception e ) {
      System.err.println( "Error running Beam pipeline on '"+environment+"': " + e.getMessage() );
      e.printStackTrace();
      return 1;
    }

  }

Example 5