org.pentaho.di.core.exception.KettleFileException Java Examples

/**
 * Internal image loading from Kettle's VFS.
 */
private static SwtUniversalImage loadFromSimpleVFS( Display display, String location ) {
  try {
    InputStream s = KettleVFS.getInputStream( location );
    if ( s == null ) {
      return null;
    }
    try {
      return loadImage( display, s, location );
    } finally {
      IOUtils.closeQuietly( s );
    }
  } catch ( KettleFileException e ) {
    // do nothing. try to load next
  }
  return null;
}
 

@Override public VFSFile getFile( VFSFile file ) {
  try {
    FileObject fileObject = KettleVFS
      .getFileObject( file.getPath(), new Variables(), VFSHelper.getOpts( file.getPath(), file.getConnection() ) );
    if ( !fileObject.exists() ) {
      return null;
    }
    String parent = null;
    if ( fileObject.getParent() != null && fileObject.getParent().getName() != null ) {
      parent = fileObject.getParent().getName().getURI();
    } else {
      parent = fileObject.getURL().getProtocol() + "://";
    }
    if ( fileObject.getType().equals( FileType.FOLDER ) ) {
      return VFSDirectory.create( parent, fileObject, null, file.getDomain() );
    } else {
      return VFSFile.create( parent, fileObject, null, file.getDomain() );
    }
  } catch ( KettleFileException | FileSystemException e ) {
    // File does not exist
  }
  return null;
}
 

@Test
public void testFindJarFiles_SeveralJarsInDifferentDirs() throws IOException, KettleFileException {
  // Files in plugins/job.jar folder
  Files.createDirectories( PATH_TO_DIR_WITH_JAR_IN_NAME );
  Files.createFile( PATH_TO_JAR_FILE1 );
  Files.createFile( PATH_TO_NOT_JAR_FILE );
  // Files in plugins/test_dir folder
  Files.createDirectories( PATH_TO_TEST_DIR_NAME );
  Files.createFile( PATH_TO_JAR_FILE2 );
  Files.createFile( PATH_TO_NOT_JAR_FILE_IN_TEST_DIR );
  // Files in plugins folder
  Files.createFile( Paths.get( BASE_TEMP_DIR, PLUGINS_DIR_NAME, JAR_FILE2_NAME ) );
  Files.createFile( Paths.get( BASE_TEMP_DIR, PLUGINS_DIR_NAME, NOT_JAR_FILE_NAME ) );

  FileObject[] findJarFiles = plFolder.findJarFiles();
  assertNotNull( findJarFiles );
  assertEquals( 3, findJarFiles.length );
}
 

private void closeOutput() throws KettleFileException {
  try {
    if ( data.dosToTempFile != null ) {
      data.dosToTempFile.close();
      data.dosToTempFile = null;
    }
    if ( data.fosToTempFile != null ) {
      data.fosToTempFile.close();
      data.fosToTempFile = null;
    }
    data.firstRead = true;
  } catch ( IOException e ) {
    throw new KettleFileException(
        BaseMessages.getString( PKG, "GroupBy.Exception.UnableToCloseInputStream", data.tempFile.getPath() ), e );
  }
}
 

@Test
public void stageForCache_missing_source() throws Exception {
  DistributedCacheUtilImpl ch = new DistributedCacheUtilImpl();

  Configuration conf = new Configuration();
  FileSystem fs = DistributedCacheTestUtil.getLocalFileSystem( conf );

  Path dest = new Path( "bin/test/bogus-destination" );
  FileObject bogusSource = KettleVFS.getFileObject( "bogus" );
  try {
    ch.stageForCache( bogusSource, fs, dest, true );
    fail( "expected exception when source does not exist" );
  } catch ( KettleFileException ex ) {
    assertEquals( BaseMessages
        .getString( DistributedCacheUtilImpl.class, "DistributedCacheUtil.SourceDoesNotExist", bogusSource ),
      ex.getMessage().trim() );
  }
}
 

@Test
public void stageForCache_destination_no_overwrite() throws Exception {
  DistributedCacheUtilImpl ch = new DistributedCacheUtilImpl();

  Configuration conf = new Configuration();
  FileSystem fs = DistributedCacheTestUtil.getLocalFileSystem( conf );

  FileObject source = DistributedCacheTestUtil.createTestFolderWithContent();
  try {
    Path root = new Path( "bin/test/stageForCache_destination_exists" );
    Path dest = new Path( root, "dest" );

    fs.mkdirs( dest );
    assertTrue( fs.exists( dest ) );
    assertTrue( fs.getFileStatus( dest ).isDir() );
    try {
      ch.stageForCache( source, fs, dest, false );
    } catch ( KettleFileException ex ) {
      assertTrue( ex.getMessage(), ex.getMessage().contains( "Destination exists" ) );
    } finally {
      fs.delete( root, true );
    }
  } finally {
    source.delete( new AllFileSelector() );
  }
}
 

/**
 * @param destDir
 * @param newPath
 * @return
 * @throws FileException
 */
@Override public String getNewName( VFSFile destDir, String newPath ) throws FileException {
  String extension = Utils.getExtension( newPath );
  String parent = Utils.getParent( newPath );
  String name = Utils.getName( newPath ).replace( "." + extension, "" );
  int i = 1;
  String testName = sanitizeName( destDir, newPath );
  try {
    while ( KettleVFS
      .getFileObject( testName, new Variables(), VFSHelper.getOpts( testName, destDir.getConnection() ) )
      .exists() ) {
      if ( Utils.isValidExtension( extension ) ) {
        testName = sanitizeName( destDir, parent + name + " " + i + "." + extension );
      } else {
        testName = sanitizeName( destDir, newPath + " " + i );
      }
      i++;
    }
  } catch ( KettleFileException | FileSystemException e ) {
    return testName;
  }
  return testName;
}
 

/**
 * Write ONLY the specified metadata to the outputStream
 *
 * @throws KettleFileException in case things go awry
 */
@Override
public void writeMeta( DataOutputStream outputStream ) throws KettleFileException {
  lock.readLock().lock();
  try {
    // First handle the number of fields in a row
    try {
      outputStream.writeInt( size() );
    } catch ( IOException e ) {
      throw new KettleFileException( "Unable to write nr of metadata values", e );
    }

    // Write all values in the row
    for ( int i = 0; i < size(); i++ ) {
      getValueMeta( i ).writeMeta( outputStream );
    }
  } finally {
    lock.readLock().unlock();
  }

}
 

/**
 * Loads a properties file from the plugin directory for the plugin interface provided
 *
 * @param plugin
 * @return
 * @throws KettleFileException
 * @throws IOException
 */
private static Properties loadProperties( PluginInterface plugin, String relativeName ) throws KettleFileException,
  IOException {
  if ( plugin == null ) {
    throw new NullPointerException();
  }
  FileObject propFile =
    KettleVFS.getFileObject( plugin.getPluginDirectory().getPath() + Const.FILE_SEPARATOR + relativeName );
  if ( !propFile.exists() ) {
    throw new FileNotFoundException( propFile.toString() );
  }
  try {
    Properties pluginProperties = new Properties();
    pluginProperties.load( new FileInputStream( propFile.getName().getPath() ) );
    return pluginProperties;
  } catch ( Exception e ) {
    // Do not catch ConfigurationException. Different shims will use different
    // packages for this exception.
    throw new IOException( e );
  }
}
 

private List<FileObject> getFiles( String localfolder ) throws KettleFileException {
  try {
    List<FileObject> myFileList = new ArrayList<FileObject>();

    // Get all the files in the local directory...

    FileObject localFiles = KettleVFS.getFileObject( localfolder, this );
    FileObject[] children = localFiles.getChildren();
    if ( children != null ) {
      for ( int i = 0; i < children.length; i++ ) {
        // Get filename of file or directory
        if ( children[i].getType().equals( FileType.FILE ) ) {
          myFileList.add( children[i] );

        }
      } // end for
    }

    return myFileList;
  } catch ( IOException e ) {
    throw new KettleFileException( e );
  }

}
 

private void prepareFile() throws KettleFileException {
  try {
    String filename = environmentSubstitute( meta.getFilename() );
    if ( meta.isAddToResultFiles() ) {
      // Add this to the result file names...
      ResultFile resultFile =
        new ResultFile(
          ResultFile.FILE_TYPE_GENERAL, KettleVFS.getFileObject( filename, getTransMeta() ), getTransMeta()
            .getName(), getStepname() );
      resultFile.setComment( "This file was created with a cube file output step" );
      addResultFile( resultFile );
    }

    data.fos = KettleVFS.getOutputStream( filename, getTransMeta(), false );
    data.zip = new GZIPOutputStream( data.fos );
    data.dos = new DataOutputStream( data.zip );
  } catch ( Exception e ) {
    throw new KettleFileException( e );
  }
}
 

/**
 * Create a file appender
 * @param filename The (VFS) filename (URL) to write to.
 * @param exact is this an exact filename of a filename to be stored in "java.io.tmp"
 * @param append
 * @return A new file appender
 * @throws KettleFileException In case there is a problem opening the file.
 */
public static final Log4jFileAppender createFileAppender( String filename, boolean exact, boolean append ) throws KettleFileException {
  try {
    FileObject file;
    if ( !exact ) {
      file = KettleVFS.createTempFile( filename, ".log", System.getProperty( "java.io.tmpdir" ) );
    } else {
      file = KettleVFS.getFileObject( filename );
    }

    Log4jFileAppender appender = new Log4jFileAppender( file, append );
    appender.setLayout( new Log4jKettleLayout( true ) );
    appender.setName( LogWriter.createFileAppenderName( filename, exact ) );

    return appender;
  } catch ( IOException e ) {
    throw new KettleFileException( "Unable to add Kettle file appender to Log4J", e );
  }
}
 

void addToBuffer( Object[] row ) throws KettleFileException {
  data.bufferList.add( row );
  if ( data.bufferList.size() > 5000 && data.rowsOnFile == 0 ) {
    String pathToTmp = environmentSubstitute( getMeta().getDirectory() );
    try {
      File ioFile = new File( pathToTmp );
      if ( !ioFile.exists() ) {
        // try to resolve as Apache VFS file
        pathToTmp = retrieveVfsPath( pathToTmp );
      }
      data.tempFile = File.createTempFile( getMeta().getPrefix(), ".tmp", new File( pathToTmp ) );
      data.fosToTempFile = new FileOutputStream( data.tempFile );
      data.dosToTempFile = new DataOutputStream( data.fosToTempFile );
      data.firstRead = true;
    } catch ( IOException e ) {
      throw new KettleFileException( BaseMessages.getString( PKG, "GroupBy.Exception.UnableToCreateTemporaryFile" ),
          e );
    }
    // OK, save the oldest rows to disk!
    Object[] oldest = data.bufferList.get( 0 );
    data.inputRowMeta.writeData( data.dosToTempFile, oldest );
    data.bufferList.remove( 0 );
    data.rowsOnFile++;
  }
}
 

/**
 * @param files
 * @return
 */
public List<VFSFile> delete( List<VFSFile> files ) {
  List<VFSFile> deletedFiles = new ArrayList<>();
  for ( VFSFile file : files ) {
    try {
      FileObject fileObject = KettleVFS
        .getFileObject( file.getPath(), new Variables(), VFSHelper.getOpts( file.getPath(), file.getConnection() ) );
      if ( fileObject.delete() ) {
        deletedFiles.add( file );
      }
    } catch ( KettleFileException | FileSystemException kfe ) {
      // Ignore don't add
    }
  }
  return deletedFiles;
}
 

public void getFields( RowMetaInterface r, String name, RowMetaInterface[] info, StepMeta nextStep,
  VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException {

  // Add the fields from a ResultFile
  try {
    ResultFile resultFile =
      new ResultFile(
        ResultFile.FILE_TYPE_GENERAL, KettleVFS.getFileObject( "foo.bar", space ), "parentOrigin", "origin" );
    RowMetaAndData add = resultFile.getRow();

    // Set the origin on the fields...
    for ( int i = 0; i < add.size(); i++ ) {
      add.getValueMeta( i ).setOrigin( name );
    }
    r.addRowMeta( add.getRowMeta() );
  } catch ( KettleFileException e ) {
    throw new KettleStepException( e );
  }
}
 

public boolean init( StepMetaInterface smi, StepDataInterface sdi ) {
  meta = (CubeOutputMeta) smi;
  data = (CubeOutputData) sdi;

  if ( super.init( smi, sdi ) ) {
    if ( !meta.isDoNotOpenNewFileInit() ) {
      try {
        prepareFile();
        data.oneFileOpened = true;
        return true;
      } catch ( KettleFileException ioe ) {
        logError( BaseMessages.getString( PKG, "CubeOutput.Log.ErrorOpeningCubeOutputFile" ) + ioe.toString() );
      }
    } else {
      return true;
    }

  }
  return false;
}
 

/**
 * @param file
 * @param newPath
 * @param overwrite
 * @return
 */
private VFSFile doMove( VFSFile file, String newPath, boolean overwrite ) {
  try {
    FileObject fileObject = KettleVFS
      .getFileObject( file.getPath(), new Variables(), VFSHelper.getOpts( file.getPath(), file.getConnection() ) );
    FileObject renameObject = KettleVFS
      .getFileObject( newPath, new Variables(), VFSHelper.getOpts( file.getPath(), file.getConnection() ) );
    if ( overwrite && renameObject.exists() ) {
      renameObject.delete();
    }
    fileObject.moveTo( renameObject );
    if ( file instanceof VFSDirectory ) {
      return VFSDirectory.create( renameObject.getParent().getPublicURIString(), renameObject, file.getConnection(),
        file.getDomain() );
    } else {
      return VFSFile.create( renameObject.getParent().getPublicURIString(), renameObject, file.getConnection(),
        file.getDomain() );
    }
  } catch ( KettleFileException | FileSystemException e ) {
    return null;
  }
}
 

/**
 *
 * Returns in the first position a line; ;
 * on the second position how many lines from file were read to get a full line
 *
 */
public static final TextFileLine getLine( LogChannelInterface log, InputStreamReader reader, EncodingType encodingType,
                                    int fileFormatType, StringBuilder line, String regex, long lineNumberInFile )
  throws KettleFileException {

  String sline = getLine( log, reader, encodingType, fileFormatType, line );

  boolean lenientEnclosureHandling = ValueMetaBase.convertStringToBoolean(
    Const.NVL( EnvUtil.getSystemProperty( Const.KETTLE_COMPATIBILITY_TEXT_FILE_INPUT_USE_LENIENT_ENCLOSURE_HANDLING ), "N" ) );

  if ( !lenientEnclosureHandling ) {

    while ( sline != null ) {
    /*
    Check that the number of enclosures in a line is even.
    If not even it means that there was an enclosed line break.
    We need to read the next line(s) to get the remaining data in this row.
    */
      if ( checkPattern( sline, regex ) % 2 == 0 ) {
        return new TextFileLine( sline, lineNumberInFile, null );
      }

      String nextLine = getLine( log, reader, encodingType, fileFormatType, line );

      if ( nextLine == null ) {
        break;
      }

      sline = sline + nextLine;
      lineNumberInFile++;
    }

  }
  return new TextFileLine( sline, lineNumberInFile, null );
}
 

public static OutputStream getOutputStream( String vfsFilename, VariableSpace space, boolean append )
  throws KettleFileException {
  try {
    FileObject fileObject = getFileObject( vfsFilename, space );
    return getOutputStream( fileObject, append );
  } catch ( IOException e ) {
    throw new KettleFileException( e );
  }
}
 

@Test
public void testLocalFile() throws KettleFileException, MalformedURLException {
  Object keyValue = PentahoReportingOutput.getKeyValue(
    PentahoReportingOutput.getFileObject( testResourceUrl.getPath(), null ) );

  assertTrue( keyValue instanceof URL );

}
 

private Object[] getRowOfData( RowMetaInterface rowMeta ) throws KettleFileException {
  Object[] rowData = null;

  while ( !baseStep.isStopped() && rowData == null ) {
    try {
      rowData = rowMeta.readData( inputStream );
    } catch ( SocketTimeoutException e ) {
      rowData = null; // try again.
    }
  }

  return rowData;
}
 

protected FileObject getInitialFile( String filePath ) throws KettleFileException {
  FileObject initialFile = null;
  if ( filePath != null && !filePath.isEmpty() ) {
    String fileName = space.environmentSubstitute( filePath );
    if ( fileName != null && !fileName.isEmpty() ) {
      initialFile = KettleVFS.getFileObject( fileName );
    }
  }
  if ( initialFile == null ) {
    initialFile = KettleVFS.getFileObject( Spoon.getInstance().getLastFileOpened() );
  }
  return initialFile;
}
 

@Test
public void testGetLineOSX() throws KettleFileException, UnsupportedEncodingException {
  String input = "col1\tcol2\tcol3\rdata1\tdata2\tdata3\r";
  String expected = "col1\tcol2\tcol3";
  String output =
      TextFileInputUtils.getLine( null, getInputStreamReader( input ), TextFileInputMeta.FILE_FORMAT_UNIX,
          new StringBuilder( 1000 ) );
  assertEquals( expected, output );
}
 

public String getOutputFileName( Object[] row ) throws KettleException {
  String filename = null;
  if ( row == null ) {
    if ( data.writer != null ) {
      filename = data.getFileStreamsCollection().getLastFileName( );
    } else {
      filename = meta.getFileName();
      if ( filename == null ) {
        throw new KettleFileException( BaseMessages.getString( PKG, "TextFileOutput.Exception.FileNameNotSet" ) );
      }
      filename = buildFilename( environmentSubstitute( filename ), true );
    }
  } else {
    data.fileNameFieldIndex = getInputRowMeta().indexOfValue( meta.getFileNameField() );
    if ( data.fileNameFieldIndex < 0 ) {
      throw new KettleStepException( BaseMessages.getString( PKG, "TextFileOutput.Exception.FileNameFieldNotFound", meta.getFileNameField() ) );
    }
    data.fileNameMeta = getInputRowMeta().getValueMeta( data.fileNameFieldIndex );
    data.fileName = data.fileNameMeta.getString( row[data.fileNameFieldIndex] );

    if ( data.fileName == null ) {
      throw new KettleFileException( BaseMessages.getString( PKG, "TextFileOutput.Exception.FileNameNotSet" ) );
    }

    filename = buildFilename( environmentSubstitute( data.fileName ), true );
  }
  return filename;
}
 

/**
 * Write a row of Values to a DataOutputStream, without saving the meta-data.
 *
 * @param dos
 *          The DataOutputStream to write to
 * @return true if the row was written successfuly, false if something went wrong.
 */
public boolean writeData( DataOutputStream dos ) throws KettleFileException {
  // get all values in the row
  for ( int i = 0; i < size(); i++ ) {
    Value v = getValue( i );
    v.writeData( dos );
  }

  return true;
}
 

@Test
public void savesCacheToDisk() throws KettleFileException, IOException {
  Path tempFile = Files.createTempFile( "dbcache", "test" );
  DBCache.fileNameSupplier = tempFile::toString;
  DBCache dbCache = DBCache.getInstance();
  RowMeta fields = new RowMeta();
  fields.addValueMeta( new ValueMetaInteger( "int1" ) );
  String select = "select name from warehouse";
  dbCache.put( new DBCacheEntry( "warehouse", select ), fields );
  dbCache.saveCache();
  assertTrue( FileUtils.readFileToString( tempFile.toFile() ).contains( select ) );
}
 

private void closeInput() throws KettleFileException {
  try {
    if ( data.fisToTmpFile != null ) {
      data.fisToTmpFile.close();
      data.fisToTmpFile = null;
    }
    if ( data.disToTmpFile != null ) {
      data.disToTmpFile.close();
      data.disToTmpFile = null;
    }
  } catch ( IOException e ) {
    throw new KettleFileException(
        BaseMessages.getString( PKG, "GroupBy.Exception.UnableToCloseInputStream", data.tempFile.getPath() ), e );
  }
}
 

public PentahoMapReduceJobBuilderImpl( NamedCluster namedCluster,
                                       HadoopShim hadoopShim,
                                       LogChannelInterface log,
                                       VariableSpace variableSpace, PluginInterface pluginInterface,
                                       Properties pmrProperties,
                                       List<TransformationVisitorService> visitorServices )
  throws KettleFileException {
  this( namedCluster, hadoopShim, log, variableSpace, pluginInterface,
    KettleVFS.getFileObject( pluginInterface.getPluginDirectory().getPath() ), pmrProperties,
    new TransFactory(), new PMRArchiveGetter( pluginInterface, pmrProperties ), visitorServices );
}
 

private boolean tryToReadLine( boolean applyFilter ) throws KettleFileException {
  String line;
  line = getLine( log, data.isr, data.encodingType, data.fileFormatType, data.lineStringBuilder );
  if ( line != null ) {
    // when there is no header, check the filter for the first line
    if ( applyFilter ) {
      // Filter row?
      boolean isFilterLastLine = false;
      boolean filterOK = checkFilterRow( line, isFilterLastLine );
      if ( filterOK ) {
        data.lineBuffer.add( new TextFileLine( line, lineNumberInFile, data.file ) ); // Store it in the
        // line buffer...
      } else {
        return false;
      }
    } else { // don't checkFilterRow

      if ( !meta.noEmptyLines() || line.length() != 0 ) {
        data.lineBuffer.add( new TextFileLine( line, lineNumberInFile, data.file ) ); // Store it in the line
                                                                                      // buffer...
      }
    }
  } else {
    data.doneReading = true;
  }
  return true;
}
 

@Test
public void testFindJarFiles_DirWithKettleIgnoreFileIgnored() throws IOException, KettleFileException {
  Files.createDirectories( PATH_TO_TEST_DIR_NAME );
  Files.createFile( PATH_TO_JAR_FILE2 );
  Files.createFile( PATH_TO_KETTLE_IGNORE_FILE );

  FileObject[] findJarFiles = plFolder.findJarFiles();
  assertNotNull( findJarFiles );
  assertEquals( 0, findJarFiles.length );
}