Java Code Examples for org.pentaho.di.core.logging.LogChannelInterface#isDebug()

private void addFileToResultFilenames( String fileaddentry, LogChannelInterface log, Result result, Job parentJob ) {
  try {
    ResultFile resultFile =
      new ResultFile( ResultFile.FILE_TYPE_GENERAL, KettleVFS.getFileObject( fileaddentry, this ), parentJob
        .getJobname(), toString() );
    result.getResultFiles().put( resultFile.getFile().toString(), resultFile );
    if ( log.isDebug() ) {
      logDebug( BaseMessages.getString( PKG, "JobExportRepository.Log.FileAddedToResultFilesName", fileaddentry ) );
    }
  } catch ( Exception e ) {
    log.logError(
      BaseMessages.getString( PKG, "JobExportRepository.Error.AddingToFilenameResult" ), fileaddentry
        + "" + e.getMessage() );
  }
}
 

/**
 * Extract and return the correct name for the field that should be processed as NULL
 *
 * @param log
 *          the logging object
 * @param field
 *          the field that should be processed as NULL
 * @param isUseExtId
 *          the flag that indicates if the field is external id or not
 * @return return the correct name for the field that should be processed as NULL
 */
public static String getFieldToNullName( LogChannelInterface log, String field, boolean isUseExtId ) {
  String fieldToNullName = field;
  if ( isUseExtId ) {
    // verify if the field has correct syntax
    if ( !FIELD_NAME_WITH_EXTID_PATTERN.matcher( field ).matches() ) {
      if ( log.isDebug() ) {
        log.logDebug( BaseMessages.getString( PKG, "SalesforceUtils.Warn.IncorrectExternalKeySyntax", field,
            fieldToNullName ) );
      }
      return fieldToNullName;
    }

    String lookupField = field.substring( field.indexOf( EXTID_SEPARATOR ) + 1 );
    // working with custom objects and relationship
    // cut off _r and then add _c in the end of the name
    if ( lookupField.endsWith( CUSTOM_OBJECT_RELATIONSHIP_FIELD_SUFFIX ) ) {
      fieldToNullName =
          lookupField.substring( 0, lookupField.length() - CUSTOM_OBJECT_RELATIONSHIP_FIELD_SUFFIX.length() )
              + CUSTOM_OBJECT_SUFFIX;
      if ( log.isDebug() ) {
        log.logDebug( BaseMessages.getString( PKG, "SalesforceUtils.Debug.NullFieldName", fieldToNullName ) );
      }
      return fieldToNullName;
    }

    fieldToNullName = lookupField + "Id";
  }

  if ( log.isDebug() ) {
    log.logDebug( BaseMessages.getString( PKG, "SalesforceUtils.Debug.NullFieldName", fieldToNullName ) );
  }

  return fieldToNullName;
}
 

/**
 * Check if a named variable is set in python
 *
 * @param log          the log channel to use
 * @param varName      the name of the variable to check
 * @param inputStream  the input stream to read a response from
 * @param outputStream the output stream to talk to the server on
 * @return true if the named variable is set in python
 * @throws KettleException if a problem occurs
 */
@SuppressWarnings( "unchecked" ) protected static boolean checkIfPythonVariableIsSet( LogChannelInterface log,
    String varName, InputStream inputStream, OutputStream outputStream ) throws KettleException {

  boolean debug = log == null || log.isDebug();
  ObjectMapper mapper = new ObjectMapper();
  Map<String, Object> command = new HashMap<String, Object>();
  command.put( COMMAND_KEY, VARIABLE_IS_SET_KEY );
  command.put( VARIABLE_NAME_KEY, varName );
  command.put( DEBUG_KEY, debug );

  if ( inputStream != null && outputStream != null ) {
    try {
      ByteArrayOutputStream bos = new ByteArrayOutputStream();
      mapper.writeValue( bos, command );
      byte[] bytes = bos.toByteArray();

      if ( debug ) {
        outputCommandDebug( command, log );
      }

      // write the command
      writeDelimitedToOutputStream( bytes, outputStream );

      bytes = readDelimitedFromInputStream( inputStream );
      Map<String, Object> ack = mapper.readValue( bytes, Map.class );
      if ( !ack.get( RESPONSE_KEY ).toString().equals( OK_KEY ) ) {
        // fatal error
        throw new KettleException( ack.get( ERROR_MESSAGE_KEY ).toString() );
      }

      if ( !ack.get( VARIABLE_NAME_KEY ).toString().equals( varName ) ) {
        throw new KettleException( "Server sent back a response for a different " + "variable!" );
      }

      return (Boolean) ack.get( VARIABLE_EXISTS_KEY );
    } catch ( IOException ex ) {
      throw new KettleException( ex );
    }
  } else {
    outputCommandDebug( command, log );
  }

  return false;
}
 

/**
 * Receive the value of a variable in pickled or plain string form. If getting
 * a pickled variable, then in python 2 this is the pickled string; in python
 * 3 pickle.dumps returns a byte object, so the value is converted to base64
 * before leaving the server.
 *
 * @param varName      the name of the variable to get from the server
 * @param outputStream the output stream to write to
 * @param inputStream  the input stream to get server responses from
 * @param plainString  true if the plain string form of the variable is to be
 *                     returned; otherwise the variable value is pickled (and further
 *                     encoded to base64 in the case of python 3)
 * @param log          optional log
 * @return the variable value
 * @throws KettleException if a problem occurs
 */
@SuppressWarnings( "unchecked" ) protected static String receivePickledVariableValue( String varName,
    OutputStream outputStream, InputStream inputStream, boolean plainString, LogChannelInterface log )
    throws KettleException {

  boolean debug = log == null || log.isDebug();
  String objectValue = "";
  ObjectMapper mapper = new ObjectMapper();
  Map<String, Object> command = new HashMap<String, Object>();
  command.put( COMMAND_KEY, GET_VARIABLE_VALUE_KEY );
  command.put( VARIABLE_NAME_KEY, varName );
  command.put( VARIABLE_ENCODING_KEY, plainString ? STRING_ENCODING_KEY : PICKLED_ENCODING_KEY );
  command.put( DEBUG_KEY, debug );

  if ( inputStream != null && outputStream != null ) {
    try {
      ByteArrayOutputStream bos = new ByteArrayOutputStream();
      mapper.writeValue( bos, command );
      byte[] bytes = bos.toByteArray();

      if ( debug ) {
        outputCommandDebug( command, log );
      }

      // write the command
      writeDelimitedToOutputStream( bytes, outputStream );

      bytes = readDelimitedFromInputStream( inputStream );
      Map<String, Object> ack = mapper.readValue( bytes, Map.class );
      if ( !ack.get( RESPONSE_KEY ).toString().equals( OK_KEY ) ) {
        // fatal error
        throw new KettleException( ack.get( ERROR_MESSAGE_KEY ).toString() );
      }
      if ( !ack.get( VARIABLE_NAME_KEY ).toString().equals( varName ) ) {
        throw new KettleException( "Server sent back a value for a different " + "variable!" );
      }
      objectValue = ack.get( VARIABLE_VALUE_KEY ).toString();
    } catch ( IOException ex ) {
      throw new KettleException( ex );
    }
  } else if ( debug ) {
    outputCommandDebug( command, log );
  }

  return objectValue;
}
 

/**
 * Get an image from python. Assumes that the image is a
 * matplotlib.figure.Figure object. Retrieves this as png data and returns a
 * BufferedImage.
 *
 * @param varName      the name of the variable containing the image in python
 * @param outputStream the output stream to talk to the server on
 * @param inputStream  the input stream to receive server responses from
 * @param log          an optional log
 * @return a BufferedImage
 * @throws KettleException if a problem occurs
 */
@SuppressWarnings( "unchecked" ) protected static BufferedImage getPNGImageFromPython( String varName,
    OutputStream outputStream, InputStream inputStream, LogChannelInterface log ) throws KettleException {

  boolean debug = log == null || log.isDebug();
  ObjectMapper mapper = new ObjectMapper();
  Map<String, Object> command = new HashMap<String, Object>();
  command.put( COMMAND_KEY, GET_IMAGE_KEY );
  command.put( VARIABLE_NAME_KEY, varName );
  command.put( DEBUG_KEY, debug );

  if ( inputStream != null && outputStream != null ) {
    try {
      ByteArrayOutputStream bos = new ByteArrayOutputStream();
      mapper.writeValue( bos, command );
      byte[] bytes = bos.toByteArray();

      if ( debug ) {
        outputCommandDebug( command, log );
      }
      // write the command
      writeDelimitedToOutputStream( bytes, outputStream );

      bytes = readDelimitedFromInputStream( inputStream );

      Map<String, Object> ack = mapper.readValue( bytes, Map.class );
      if ( !ack.get( RESPONSE_KEY ).toString().equals( OK_KEY ) ) {
        // fatal error
        throw new KettleException( ack.get( ERROR_MESSAGE_KEY ).toString() );
      }

      if ( !ack.get( VARIABLE_NAME_KEY ).toString().equals( varName ) ) {
        throw new KettleException( "Server sent back a response for a different " + "variable!" );
      }

      String encoding = ack.get( IMAGE_ENCODING_KEY ).toString();
      String imageData = ack.get( IMAGE_DATA_KEY ).toString();
      byte[] imageBytes;
      if ( encoding.equals( BASE64_ENCODING_KEY ) ) {
        imageBytes = Base64.decodeBase64( imageData.getBytes() );
      } else {
        imageBytes = imageData.getBytes();
      }
      return ImageIO.read( new BufferedInputStream( new ByteArrayInputStream( imageBytes ) ) );
    } catch ( IOException ex ) {
      throw new KettleException( ex );
    }
  } else {
    outputCommandDebug( command, log );
  }
  return null;
}
 

/**
 * Execute a script on the server
 *
 * @param script       the script to execute
 * @param outputStream the output stream to write data to the server
 * @param inputStream  the input stream to read responses from
 * @param log          optional log to write to
 * @return a two element list that contains the sys out and sys error from the
 * script execution
 * @throws KettleException if a problem occurs
 */
@SuppressWarnings( "unchecked" ) protected static List<String> executeUserScript( String script,
    OutputStream outputStream, InputStream inputStream, LogChannelInterface log ) throws KettleException {
  if ( !script.endsWith( "\n" ) ) {
    script += "\n";
  }
  List<String> outAndErr = new ArrayList<String>();

  ObjectMapper mapper = new ObjectMapper();
  boolean debug = log == null || log.isDebug();
  Map<String, Object> command = new HashMap<String, Object>();
  command.put( "command", "execute_script" );
  command.put( "script", script );
  command.put( "debug", debug );
  if ( inputStream != null && outputStream != null ) {
    try {
      ByteArrayOutputStream bos = new ByteArrayOutputStream();
      mapper.writeValue( bos, command );
      byte[] bytes = bos.toByteArray();

      if ( debug ) {
        outputCommandDebug( command, log );
      }
      writeDelimitedToOutputStream( bytes, outputStream );

      // get the result of execution
      bytes = readDelimitedFromInputStream( inputStream );

      Map<String, Object> ack = mapper.readValue( bytes, Map.class );
      if ( !ack.get( RESPONSE_KEY ).toString().equals( OK_KEY ) ) {
        // fatal error
        throw new KettleException( ack.get( ERROR_MESSAGE_KEY ).toString() );
      }
      // get the script out and err
      outAndErr.add( ack.get( SCRIPT_OUT_KEY ).toString() );
      outAndErr.add( ack.get( SCRIPT_ERROR_KEY ).toString() );
      if ( debug ) {
        if ( log != null ) {
          log.logDebug(
              BaseMessages.getString( PKG, "ServerUtils.Message.ScriptOutput" ) + "\n" + outAndErr.get( 0 ) );
          log.logDebug(
              BaseMessages.getString( PKG, "ServerUtils.Message.ScriptError" ) + "\n" + outAndErr.get( 1 ) );
        } else {
          System.err.println( "Script output:\n" + outAndErr.get( 0 ) );
          System.err.println( "\nScript error:\n" + outAndErr.get( 1 ) );
        }
      }

      if ( outAndErr.get( 1 ).contains( "Warning:" ) ) {
        // clear warnings - we really just want to know if there
        // are major errors
        outAndErr.set( 1, "" );
      }
    } catch ( IOException ex ) {
      throw new KettleException( ex );
    }
  } else if ( debug ) {
    outputCommandDebug( command, log );
  }

  return outAndErr;
}
 

/**
 * Send rows to python to be converted to a pandas data frame
 *
 * @param log          the log channel to use
 * @param inputStream  the input stream to read a response from
 * @param outputStream the output stream to talk to the server on
 * @throws KettleException if a problem occurs
 */
protected static void sendRowsToPandasDataFrame( LogChannelInterface log, RowMetaInterface meta, List<Object[]> rows,
    String frameName, OutputStream outputStream, InputStream inputStream ) throws KettleException {
  ObjectMapper mapper = new ObjectMapper();

  boolean debug = log == null || log.isDebug();
  Map<String, Object> metaData = createMetadataMessage( frameName, meta );
  Map<String, Object> command = new HashMap<String, Object>();
  command.put( COMMAND_KEY, ACCEPT_ROWS_COMMAND );
  command.put( NUM_ROWS_KEY, rows.size() );
  command.put( ROW_META_KEY, metaData );
  command.put( DEBUG_KEY, debug );

  boolean needsBase64 = (boolean) metaData.get( BASE64_ENCODING_KEY );
  command.put( BASE64_ENCODING_KEY, needsBase64 );
  metaData.remove( BASE64_ENCODING_KEY );

  if ( inputStream != null && outputStream != null ) {
    try {
      List<Object> rowsInfo = rowsToCSVNew( meta, rows );

      // unfortunately we'll incur the base 64 transcoding overhead even if it
      // is only the header row that needs it.
      if ( !needsBase64 ) {
        command.put( BASE64_ENCODING_KEY, (boolean) rowsInfo.get( 1 ) );
      }

      ByteArrayOutputStream bos = new ByteArrayOutputStream();
      mapper.writeValue( bos, command );
      byte[] bytes = bos.toByteArray();

      if ( debug ) {
        outputCommandDebug( command, log );
      }

      // write the command
      writeDelimitedToOutputStream( bytes, outputStream );

      // now write the CSV data
      if ( rows.size() > 0 ) {
        if ( log != null && debug ) {
          log.logDebug( "Sending CSV data..." );
        }

        writeDelimitedToOutputStream( (byte[]) rowsInfo.get( 0 ), outputStream );

        /* // bos = new ByteArrayOutputStream();
        // BufferedWriter bw = new BufferedWriter( new OutputStreamWriter( bos ) );
        StringBuilder csv = rowsToCSV( meta, rows );
        Charset utf8 = Charset.forName( "UTF-8" );
        ByteBuffer
            bb =
            utf8.newEncoder().onUnmappableCharacter( CodingErrorAction.IGNORE )
                .onMalformedInput( CodingErrorAction.IGNORE ).encode( CharBuffer.wrap( csv.toString() ) );
        // byte[] ptext = csv.toString().getBytes( Charset.forName( "UTF-8" ) );
        System.out.println( csv.toString() );
        System.out.println( "-----------------" );
        // bw.write( csv.toString() );
        // bw.flush();
        // bw.close();
        // bytes = bos.toByteArray();
        // writeDelimitedToOutputStream( bytes, outputStream );
        writeDelimitedToOutputStream( bb.array(), outputStream ); */
      }

      String serverAck = receiveServerAck( inputStream );
      if ( serverAck != null ) {
        throw new KettleException(
            BaseMessages.getString( PKG, "ServerUtils.Error.TransferOfRowsFailed" ) + serverAck );
      }
    } catch ( IOException ex ) {
      throw new KettleException( ex );
    }
  } else if ( debug ) {
    outputCommandDebug( command, log );
  }
}
 

/**
 * Receive rows taken from a python pandas data frame
 *
 * @param log             the log channel to use
 * @param frameName       the name of the pandas frame to get
 * @param includeRowIndex true to include the frame row index as a field
 * @param inputStream     the input stream to read a response from
 * @param outputStream    the output stream to talk to the server on
 * @return the data frame converted to rows along with its associated row metadata
 * @throws KettleException if a problem occurs
 */
@SuppressWarnings( "unchecked" ) protected static RowMetaAndRows receiveRowsFromPandasDataFrame(
    LogChannelInterface log, String frameName, boolean includeRowIndex, OutputStream outputStream,
    InputStream inputStream ) throws KettleException {

  boolean debug = log == null || log.isDebug();
  ObjectMapper mapper = new ObjectMapper();
  Map<String, Object> command = new HashMap<String, Object>();
  command.put( COMMAND_KEY, GET_FRAME_COMMAND );
  command.put( FRAME_NAME_KEY, frameName );
  command.put( FRAME_INCLUDE_ROW_INDEX, includeRowIndex );
  command.put( DEBUG_KEY, debug );

  RowMetaAndRows result = null;
  if ( inputStream != null && outputStream != null ) {
    try {
      ByteArrayOutputStream bos = new ByteArrayOutputStream();
      mapper.writeValue( bos, command );
      byte[] bytes = bos.toByteArray();

      if ( debug ) {
        outputCommandDebug( command, log );
      }

      writeDelimitedToOutputStream( bytes, outputStream );
      String serverAck = receiveServerAck( inputStream );
      if ( serverAck != null ) {
        throw new KettleException( serverAck );
      }

      // read the header
      bytes = readDelimitedFromInputStream( inputStream );
      Map<String, Object> headerResponse = mapper.readValue( bytes, Map.class );
      if ( headerResponse == null ) {
        throw new KettleException( BaseMessages.getString( PKG, "ServerUtils.Error.HeaderMetadataMapIsNull" ) );
      }
      if ( headerResponse.get( RESPONSE_KEY ).toString().equals( ROW_META_KEY ) ) {
        if ( log != null ) {
          log.logDebug( BaseMessages
              .getString( PKG, "ServerUtils.Message.ReceivedMetadataResponse", headerResponse.get( NUM_ROWS_KEY ) ) );
        } else {
          System.err.println( BaseMessages
              .getString( PKG, "ServerUtils.Message.ReceivedMetadataResponse", headerResponse.get( NUM_ROWS_KEY ) ) );
        }
      } else {
        throw new KettleException( BaseMessages.getString( PKG, "ServerUtils.Error.UnknownResponseType" ) );
      }

      RowMetaInterface convertedMeta = jsonRowMetadataToRowMeta( frameName, headerResponse );
      int numRows = (Integer) headerResponse.get( NUM_ROWS_KEY );

      bytes = readDelimitedFromInputStream( inputStream );
      String csv = new String( bytes, Charset.forName( "UTF-8" ) );
      result = csvToRows( csv, convertedMeta, numRows );
    } catch ( IOException ex ) {
      throw new KettleException( ex );
    }
  } else {
    outputCommandDebug( command, log );
  }

  return result;
}
 

/**
 * Std out and err are redirected to StringIO objects in the server. This
 * method retrieves the values of those buffers.
 *
 * @param outputStream the output stream to talk to the server on
 * @param inputStream  the input stream to receive server responses from
 * @param log          optional log
 * @return the std out and err strings as a two element list
 * @throws KettleException if a problem occurs
 */
@SuppressWarnings( "unchecked" ) protected static List<String> receiveDebugBuffer( OutputStream outputStream,
    InputStream inputStream, LogChannelInterface log ) throws KettleException {
  List<String> stdOutStdErr = new ArrayList<String>();

  boolean debug = log == null || log.isDebug();
  ObjectMapper mapper = new ObjectMapper();
  Map<String, Object> command = new HashMap<String, Object>();
  command.put( "command", "get_debug_buffer" );

  if ( inputStream != null && outputStream != null ) {
    try {
      if ( debug ) {
        outputCommandDebug( command, log );
      }
      ByteArrayOutputStream bos = new ByteArrayOutputStream();
      mapper.writeValue( bos, command );
      byte[] bytes = bos.toByteArray();

      // write the command
      writeDelimitedToOutputStream( bytes, outputStream );

      bytes = readDelimitedFromInputStream( inputStream );
      Map<String, Object> ack = mapper.readValue( bytes, Map.class );
      if ( !ack.get( "response" ).toString().equals( "ok" ) ) {
        // fatal error
        throw new KettleException( ack.get( ERROR_MESSAGE_KEY ).toString() );
      }
      Object stOut = ack.get( "std_out" );
      stdOutStdErr.add( stOut != null ? stOut.toString() : "" );
      Object stdErr = ack.get( "std_err" );
      stdOutStdErr.add( stdErr != null ? stdErr.toString() : "" );
    } catch ( IOException ex ) {
      throw new KettleException( ex );
    }
  } else if ( debug ) {
    outputCommandDebug( command, log );
  }

  return stdOutStdErr;
}
 

/**
 * Get the type of a variable in python
 *
 * @param varName      the name of the variable to check
 * @param outputStream the output stream to talk to the server on
 * @param inputStream  the input stream to receive server responses from
 * @param log          an optional log
 * @return the type of the variable in python
 * @throws KettleException if a problem occurs
 */
@SuppressWarnings( "unchecked" ) protected static PythonVariableType getPythonVariableType( String varName,
    OutputStream outputStream, InputStream inputStream, LogChannelInterface log ) throws KettleException {

  boolean debug = log == null || log.isDebug();
  ObjectMapper mapper = new ObjectMapper();
  Map<String, Object> command = new HashMap<String, Object>();
  command.put( COMMAND_KEY, GET_VARIABLE_TYPE_KEY );
  command.put( VARIABLE_NAME_KEY, varName );
  command.put( DEBUG_KEY, debug );
  if ( inputStream != null && outputStream != null ) {
    try {
      if ( debug ) {
        outputCommandDebug( command, log );
      }
      ByteArrayOutputStream bos = new ByteArrayOutputStream();
      mapper.writeValue( bos, command );
      byte[] bytes = bos.toByteArray();

      // write the command
      writeDelimitedToOutputStream( bytes, outputStream );

      bytes = readDelimitedFromInputStream( inputStream );
      Map<String, Object> ack = mapper.readValue( bytes, Map.class );
      if ( !ack.get( RESPONSE_KEY ).toString().equals( OK_KEY ) ) {
        // fatal error
        throw new KettleException( ack.get( ERROR_MESSAGE_KEY ).toString() );
      }

      if ( !ack.get( VARIABLE_NAME_KEY ).toString().equals( varName ) ) {
        throw new KettleException( "Server sent back a response for a different " + "variable!" );
      }

      String varType = ack.get( VARIABLE_TYPE_RESPONSE_KEY ).toString();
      PythonSession.PythonVariableType pvt = PythonSession.PythonVariableType.Unknown;
      for ( PythonSession.PythonVariableType t : PythonSession.PythonVariableType.values() ) {
        if ( t.toString().toLowerCase().equals( varType ) ) {
          pvt = t;
          break;
        }
      }
      return pvt;
    } catch ( IOException ex ) {
      throw new KettleException( ex );
    }
  } else {
    outputCommandDebug( command, log );
  }

  return PythonSession.PythonVariableType.Unknown;
}