Java Code Examples for org.apache.flume.Context#putAll()

@Override
public void configure(Context context) throws ConfigurationException {
  super.configure(context);
  sinks = Arrays.asList(context.getString(
      BasicConfigurationConstants.CONFIG_SINKS).split("\\s+"));
  Map<String, String> params = context.getSubProperties(
      BasicConfigurationConstants.CONFIG_SINK_PROCESSOR_PREFIX);
  processorContext = new Context();
  processorContext.putAll(params);
  SinkProcessorType spType = getKnownSinkProcessor(processorContext.getString(
          BasicConfigurationConstants.CONFIG_TYPE));

  if (spType != null) {
    processorConf =
        (SinkProcessorConfiguration) ComponentConfigurationFactory.create(
            this.getComponentName() + "-processor",
            spType.toString(),
            ComponentType.SINK_PROCESSOR);
    if (processorConf != null) {
      processorConf.setSinks(new HashSet<String>(sinks));
      processorConf.configure(processorContext);
    }
  }
  setConfigured();
}
 

@Test(expected=ChannelException.class)
public void testTransactionPutCapacityOverload() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("capacity", "5");
  parms.put("transactionCapacity", "2");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  Transaction transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  // shouldn't be able to fit a third in the buffer
  channel.put(EventBuilder.withBody("test".getBytes()));
  Assert.fail();
}
 

@Test(expected=ChannelException.class)
public void testCapacityOverload() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("capacity", "5");
  parms.put("transactionCapacity", "3");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  Transaction transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  transaction.commit();
  transaction.close();

  transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  // this should kill  it
  transaction.commit();
  Assert.fail();
}
 

@Test
public void testCapacityBufferEmptyingAfterRollback() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("capacity", "3");
  parms.put("transactionCapacity", "3");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  Transaction tx = channel.getTransaction();
  tx.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  tx.rollback();
  tx.close();

  tx = channel.getTransaction();
  tx.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  tx.commit();
  tx.close();
}
 

@Test
public void testNullEmptyEvent() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("byteCapacity", "2000");
  parms.put("byteCapacityBufferPercentage", "20");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  Transaction tx = channel.getTransaction();
  tx.begin();
  //This line would cause a NPE without FLUME-1622.
  channel.put(EventBuilder.withBody(null));
  tx.commit();
  tx.close();

  tx = channel.getTransaction();
  tx.begin();
  channel.put(EventBuilder.withBody(new byte[0]));
  tx.commit();
  tx.close();


}
 

@Test
public void testNegativeCapacities() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("capacity", "-3");
  parms.put("transactionCapacity", "-1");
  context.putAll(parms);
  Configurables.configure(channel, context);

  Assert.assertTrue(field("queue")
          .ofType(LinkedBlockingDeque.class)
          .in(channel).get()
          .remainingCapacity() > 0);

  Assert.assertTrue(field("transCapacity")
          .ofType(Integer.class)
          .in(channel).get() > 0);
}
 

/**
 * Initializes the serializer for flume events.
 * @param eventSerializerType
 */
private void initializeSerializer(final Context context,final String eventSerializerType) {
    
   EventSerializers eventSerializer = null;
   try {
           eventSerializer =  EventSerializers.valueOf(eventSerializerType.toUpperCase());
    } catch(IllegalArgumentException iae) {
           logger.error("An invalid eventSerializer {} was passed. Please specify one of {} ",eventSerializerType,
                   Joiner.on(",").skipNulls().join(EventSerializers.values()));
           Throwables.propagate(iae);
    }
   
   final Context serializerContext = new Context();
   serializerContext.putAll(context.getSubProperties(FlumeConstants.CONFIG_SERIALIZER_PREFIX));
   copyPropertiesToSerializerContext(context,serializerContext);
         
   try {
     @SuppressWarnings("unchecked")
     Class<? extends EventSerializer> clazz = (Class<? extends EventSerializer>) Class.forName(eventSerializer.getClassName());
     serializer = clazz.newInstance();
     serializer.configure(serializerContext);
     
   } catch (Exception e) {
     logger.error("Could not instantiate event serializer." , e);
     Throwables.propagate(e);
   }
}
 

/**
 * Initializes the serializer for flume events.
 * @param eventSerializerType
 */
private void initializeSerializer(final Context context,final String eventSerializerType) {
    
   EventSerializers eventSerializer = null;
   try {
           eventSerializer =  EventSerializers.valueOf(eventSerializerType.toUpperCase());
    } catch(IllegalArgumentException iae) {
           logger.error("An invalid eventSerializer {} was passed. Please specify one of {} ",eventSerializerType,
                   Joiner.on(",").skipNulls().join(EventSerializers.values()));
           Throwables.propagate(iae);
    }
   
   final Context serializerContext = new Context();
   serializerContext.putAll(context.getSubProperties(FlumeConstants.CONFIG_SERIALIZER_PREFIX));
   copyPropertiesToSerializerContext(context,serializerContext);
         
   try {
     @SuppressWarnings("unchecked")
     Class<? extends EventSerializer> clazz = (Class<? extends EventSerializer>) Class.forName(eventSerializer.getClassName());
     serializer = clazz.newInstance();
     serializer.configure(serializerContext);
     
   } catch (Exception e) {
     logger.error("Could not instantiate event serializer." , e);
     Throwables.propagate(e);
   }
}
 

public static Context createFileChannelContext(String checkpointDir,
    String dataDir, String backupDir, Map<String, String> overrides) {
  Context context = new Context();
  context.put(FileChannelConfiguration.CHECKPOINT_DIR,
          checkpointDir);
  if(backupDir != null) {
    context.put(FileChannelConfiguration.BACKUP_CHECKPOINT_DIR, backupDir);
  }
  context.put(FileChannelConfiguration.DATA_DIRS, dataDir);
  context.put(FileChannelConfiguration.KEEP_ALIVE, String.valueOf(1));
  context.put(FileChannelConfiguration.CAPACITY, String.valueOf(10000));
  context.putAll(overrides);
  return context;
}
 

@Test
public void testOneEventWithDefaults() throws Exception {
  //Create a context without setting increment column and payload Column
  Map<String,String> ctxMap = new HashMap<String,String>();
  ctxMap.put("table", tableName);
  ctxMap.put("columnFamily", columnFamily);
  ctxMap.put("serializer",
          "org.apache.flume.sink.hbase.SimpleHbaseEventSerializer");
  Context tmpctx = new Context();
  tmpctx.putAll(ctxMap);

  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  HBaseSink sink = new HBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, tmpctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, new Context());
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  Event e = EventBuilder.withBody(
          Bytes.toBytes(valBase));
  channel.put(e);
  tx.commit();
  tx.close();

  sink.process();
  sink.stop();
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 1);
  byte[] out = results[0];
  Assert.assertArrayEquals(e.getBody(), out);
  out = results[1];
  Assert.assertArrayEquals(Longs.toByteArray(1), out);
  testUtility.deleteTable(tableName.getBytes());
}
 

@Test
public void testOneEventWithDefaults() throws Exception {
  Map<String,String> ctxMap = new HashMap<String,String>();
  ctxMap.put("table", tableName);
  ctxMap.put("columnFamily", columnFamily);
  ctxMap.put("serializer",
          "org.apache.flume.sink.hbase.SimpleAsyncHbaseEventSerializer");
  ctxMap.put("keep-alive", "0");
  ctxMap.put("timeout", "10000");
  Context tmpctx = new Context();
  tmpctx.putAll(ctxMap);

  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, tmpctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, tmpctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  Event e = EventBuilder.withBody(
          Bytes.toBytes(valBase));
  channel.put(e);
  tx.commit();
  tx.close();
  Assert.assertFalse(sink.isConfNull());
  sink.process();
  sink.stop();
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 1);
  byte[] out = results[0];
  Assert.assertArrayEquals(e.getBody(), out);
  out = results[1];
  Assert.assertArrayEquals(Longs.toByteArray(1), out);
}
 

@Test
public void testByteCapacityBufferEmptyingAfterRollback() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("byteCapacity", "2000");
  parms.put("byteCapacityBufferPercentage", "20");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  byte[] eventBody = new byte[405];

  Transaction tx = channel.getTransaction();
  tx.begin();
  channel.put(EventBuilder.withBody(eventBody));
  channel.put(EventBuilder.withBody(eventBody));
  channel.put(EventBuilder.withBody(eventBody));
  tx.rollback();
  tx.close();

  tx = channel.getTransaction();
  tx.begin();
  channel.put(EventBuilder.withBody(eventBody));
  channel.put(EventBuilder.withBody(eventBody));
  channel.put(EventBuilder.withBody(eventBody));
  tx.commit();
  tx.close();
}
 

@Test(expected=ChannelException.class)
public void testByteCapacityOverload() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("byteCapacity", "2000");
  parms.put("byteCapacityBufferPercentage", "20");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  byte[] eventBody = new byte[405];

  Transaction transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody(eventBody));
  channel.put(EventBuilder.withBody(eventBody));
  channel.put(EventBuilder.withBody(eventBody));
  transaction.commit();
  transaction.close();

  transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody(eventBody));
  channel.put(EventBuilder.withBody(eventBody));
  // this should kill  it
  transaction.commit();
  Assert.fail();

}
 

@Test
public void testCapacityBufferEmptyingAfterTakeCommit() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("capacity", "3");
  parms.put("transactionCapacity", "3");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  Transaction tx = channel.getTransaction();
  tx.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  tx.commit();
  tx.close();

  tx = channel.getTransaction();
  tx.begin();
  channel.take();
  channel.take();
  tx.commit();
  tx.close();

  tx = channel.getTransaction();
  tx.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  tx.commit();
  tx.close();
}
 

public static ChannelSelector create(List<Channel> channels,
    Map<String, String> config) {

  ChannelSelector selector = getSelectorForType(config.get(
      BasicConfigurationConstants.CONFIG_TYPE));

  selector.setChannels(channels);

  Context context = new Context();
  context.putAll(config);

  Configurables.configure(selector, context);
  return selector;
}
 

@SuppressWarnings("unchecked")
@Override
public void configure(Context context){
  tableName = context.getString(HBaseSinkConfigurationConstants.CONFIG_TABLE);
  String cf = context.getString(
      HBaseSinkConfigurationConstants.CONFIG_COLUMN_FAMILY);
  batchSize = context.getLong(
      HBaseSinkConfigurationConstants.CONFIG_BATCHSIZE, new Long(100));
  serializerContext = new Context();
  //If not specified, will use HBase defaults.
  eventSerializerType = context.getString(
      HBaseSinkConfigurationConstants.CONFIG_SERIALIZER);
  Preconditions.checkNotNull(tableName,
      "Table name cannot be empty, please specify in configuration file");
  Preconditions.checkNotNull(cf,
      "Column family cannot be empty, please specify in configuration file");
  //Check foe event serializer, if null set event serializer type
  if(eventSerializerType == null || eventSerializerType.isEmpty()) {
    eventSerializerType =
        "org.apache.flume.sink.hbase.SimpleHbaseEventSerializer";
    logger.info("No serializer defined, Will use default");
  }
  serializerContext.putAll(context.getSubProperties(
          HBaseSinkConfigurationConstants.CONFIG_SERIALIZER_PREFIX));
  columnFamily = cf.getBytes(Charsets.UTF_8);
  try {
    Class<? extends HbaseEventSerializer> clazz =
        (Class<? extends HbaseEventSerializer>)
        Class.forName(eventSerializerType);
    serializer = clazz.newInstance();
    serializer.configure(serializerContext);
  } catch (Exception e) {
    logger.error("Could not instantiate event serializer." , e);
    Throwables.propagate(e);
  }
  kerberosKeytab = context.getString(HBaseSinkConfigurationConstants.CONFIG_KEYTAB, "");
  kerberosPrincipal = context.getString(HBaseSinkConfigurationConstants.CONFIG_PRINCIPAL, "");

  enableWal = context.getBoolean(HBaseSinkConfigurationConstants
    .CONFIG_ENABLE_WAL, HBaseSinkConfigurationConstants.DEFAULT_ENABLE_WAL);
  logger.info("The write to WAL option is set to: " + String.valueOf(enableWal));
  if(!enableWal) {
    logger.warn("HBase Sink's enableWal configuration is set to false. All " +
      "writes to HBase will have WAL disabled, and any data in the " +
      "memstore of this region in the Region Server could be lost!");
  }
  sinkCounter = new SinkCounter(this.getName());
}
 

@Test
public void testChannelResize() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("capacity", "5");
  parms.put("transactionCapacity", "5");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  Transaction transaction = channel.getTransaction();
  transaction.begin();
  for(int i=0; i < 5; i++) {
    channel.put(EventBuilder.withBody(String.format("test event %d", i).getBytes()));
  }
  transaction.commit();
  transaction.close();

  /*
   * Verify overflow semantics
   */
  transaction = channel.getTransaction();
  boolean overflowed = false;
  try {
    transaction.begin();
    channel.put(EventBuilder.withBody("overflow event".getBytes()));
    transaction.commit();
  } catch (ChannelException e) {
    overflowed = true;
    transaction.rollback();
  } finally {
    transaction.close();
  }
  Assert.assertTrue(overflowed);

  /*
   * Reconfigure capacity down and add another event, shouldn't result in exception
   */
  parms.put("capacity", "6");
  context.putAll(parms);
  Configurables.configure(channel, context);
  transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody("extended capacity event".getBytes()));
  transaction.commit();
  transaction.close();

  /*
   * Attempt to reconfigure capacity to below current entry count and verify
   * it wasn't carried out
   */
  parms.put("capacity", "2");
  parms.put("transactionCapacity", "2");
  context.putAll(parms);
  Configurables.configure(channel, context);
  for(int i=0; i < 6; i++) {
    transaction = channel.getTransaction();
    transaction.begin();
    Assert.assertNotNull(channel.take());
    transaction.commit();
    transaction.close();
  }
}
 

public void configure(Context context) throws ConfigurationException {
  super.configure(context);
  try {
    String channelList = context.getString(
        BasicConfigurationConstants.CONFIG_CHANNELS);
    if (channelList != null) {
      this.channels =
          new HashSet<String>(Arrays.asList(channelList.split("\\s+")));
    }
    if (channels.isEmpty()) {
      errors.add(new FlumeConfigurationError(componentName,
          ComponentType.CHANNEL.getComponentType(),
          FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
          ErrorOrWarning.ERROR));
      throw new ConfigurationException("No channels set for "
          + this.getComponentName());
    }
    Map<String, String> selectorParams = context.getSubProperties(
            BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);
    String selType;
    if (selectorParams != null && !selectorParams.isEmpty()) {
      selType = selectorParams.get(BasicConfigurationConstants.CONFIG_TYPE);
    } else {
      selType = ChannelSelectorConfigurationType.REPLICATING.toString();
    }

    if (selType == null || selType.isEmpty()) {
      selType = ChannelSelectorConfigurationType.REPLICATING.toString();

    }
    ChannelSelectorType selectorType =
        this.getKnownChannelSelector(selType);
    Context selectorContext = new Context();
    selectorContext.putAll(selectorParams);
    String config = null;
    if (selectorType == null) {
      config = selectorContext.getString(
          BasicConfigurationConstants.CONFIG_CONFIG);
      if (config == null || config.isEmpty()) {
        config = "OTHER";
      }
    } else {
      config = selectorType.toString().toUpperCase();
    }

    this.selectorConf =
        (ChannelSelectorConfiguration) ComponentConfigurationFactory
            .create(ComponentType.CHANNELSELECTOR.getComponentType(), config,
                ComponentType.CHANNELSELECTOR);
    selectorConf.setChannels(channels);
    selectorConf.configure(selectorContext);
  } catch (Exception e) {
    errors.add(new FlumeConfigurationError(componentName,
        ComponentType.CHANNELSELECTOR.getComponentType(),
        FlumeConfigurationErrorType.CONFIG_ERROR,
        ErrorOrWarning.ERROR));
    throw new ConfigurationException("Failed to configure component!", e);
  }
}