Java Code Examples for org.pentaho.di.trans.TransMeta#findPartitionSchema()

The following examples show how to use org.pentaho.di.trans.TransMeta#findPartitionSchema() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KettleDatabaseRepositoryTransDelegate.java    From pentaho-kettle with Apache License 2.0 6 votes vote down vote up 
/**
 * Read the partitions in the repository and add them to this transformation if they are not yet present.
 *
 * @param transMeta
 *          The transformation to load into.
 * @param overWriteShared
 *          if an object with the same name exists, overwrite
 * @throws KettleException
 */
public void readPartitionSchemas( TransMeta transMeta, boolean overWriteShared ) throws KettleException {
  try {
    ObjectId[] dbids = repository.getPartitionSchemaIDs( false );
    for ( int i = 0; i < dbids.length; i++ ) {
      PartitionSchema partitionSchema = repository.loadPartitionSchema( dbids[i], null ); // Load last version
      PartitionSchema check = transMeta.findPartitionSchema( partitionSchema.getName() ); // Check if there already is
                                                                                          // one in the transformation
      if ( check == null || overWriteShared ) {
        if ( !Utils.isEmpty( partitionSchema.getName() ) ) {
          transMeta.addOrReplacePartitionSchema( partitionSchema );
          if ( !overWriteShared ) {
            partitionSchema.setChanged( false );
          }
        }
      }
    }
  } catch ( KettleException dbe ) {
    throw new KettleException( BaseMessages.getString(
      PKG, "TransMeta.Log.UnableToReadPartitionSchemaFromRepository" ), dbe );
  }
}
Example 2
Source File: TransSplitter.java    From pentaho-kettle with Apache License 2.0 6 votes vote down vote up 
/**
 * Create a copy of a step from the original transformation for use in the a slave transformation. If the step is
 * partitioned, the partitioning will be changed to "schemaName (slave)"
 *
 * @param stepMeta
 *          The step to copy / clone.
 * @return a copy of the specified step for use in a slave transformation.
 */
private StepMeta addSlaveCopy( TransMeta transMeta, StepMeta stepMeta, SlaveServer slaveServer ) {
  StepMeta copy = (StepMeta) stepMeta.clone();
  if ( copy.isPartitioned() ) {
    StepPartitioningMeta stepPartitioningMeta = copy.getStepPartitioningMeta();
    PartitionSchema partitionSchema = stepPartitioningMeta.getPartitionSchema();
    String slavePartitionSchemaName = createSlavePartitionSchemaName( partitionSchema.getName() );
    PartitionSchema slaveSchema = transMeta.findPartitionSchema( slavePartitionSchemaName );
    if ( slaveSchema != null ) {
      stepPartitioningMeta.setPartitionSchema( slaveSchema );
    }
    // Always just start a single copy on the slave server...
    // Otherwise the confusion w.r.t. to partitioning & re-partitioning would be complete.
    //
    copy.setCopies( 1 );
  }

  transMeta.addStep( copy );
  return copy;
}
Example 3
Source File: TransSplitter.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up 
private void verifySlavePartitioningConfiguration( TransMeta slave, StepMeta stepMeta,
  ClusterSchema clusterSchema, SlaveServer slaveServer ) {
  Map<StepMeta, String> stepPartitionFlag = slaveStepPartitionFlag.get( slave );
  if ( stepPartitionFlag == null ) {
    stepPartitionFlag = new Hashtable<StepMeta, String>();
    slaveStepPartitionFlag.put( slave, stepPartitionFlag );
  }
  if ( stepPartitionFlag.get( stepMeta ) != null ) {
    return; // already done;
  }

  StepPartitioningMeta partitioningMeta = stepMeta.getStepPartitioningMeta();
  if ( partitioningMeta != null
    && partitioningMeta.getMethodType() != StepPartitioningMeta.PARTITIONING_METHOD_NONE
    && partitioningMeta.getPartitionSchema() != null ) {
    // Find the schemaPartitions map to use
    Map<PartitionSchema, List<String>> schemaPartitionsMap = slaveServerPartitionsMap.get( slaveServer );
    if ( schemaPartitionsMap != null ) {
      PartitionSchema partitionSchema = partitioningMeta.getPartitionSchema();
      List<String> partitionsList = schemaPartitionsMap.get( partitionSchema );
      if ( partitionsList != null ) {
        // We found a list of partitions, now let's create a new partition schema with this data.
        String targetSchemaName = createSlavePartitionSchemaName( partitionSchema.getName() );
        PartitionSchema targetSchema = slave.findPartitionSchema( targetSchemaName );
        if ( targetSchema == null ) {
          targetSchema = new PartitionSchema( targetSchemaName, partitionsList );
          slave.getPartitionSchemas().add( targetSchema ); // add it to the slave if it doesn't exist.
        }
      }
    }
  }

  stepPartitionFlag.put( stepMeta, "Y" ); // is done.
}
Example 4
Source File: TransDelegate.java    From pentaho-kettle with Apache License 2.0 5 votes vote down vote up 
/**
 * Add the partitions in the repository to this transformation if they are not yet present.
 * 
 * @param TransMeta
 *          The transformation to load into.
 * @param overWriteShared
 *          if an object with the same name exists, overwrite
 */
protected void readPartitionSchemas( TransMeta transMeta, boolean overWriteShared,
    List<PartitionSchema> partitionSchemas ) {
  for ( PartitionSchema partitionSchema : partitionSchemas ) {
    if ( overWriteShared || transMeta.findPartitionSchema( partitionSchema.getName() ) == null ) {
      if ( !Utils.isEmpty( partitionSchema.getName() ) ) {
        transMeta.addOrReplacePartitionSchema( partitionSchema );
        if ( !overWriteShared ) {
          partitionSchema.setChanged( false );
        }
      }
    }
  }
}