Java Code Examples for org.apache.flume.Transaction#rollback()

@Test
public void testCapacity() throws Exception {
  Map<String, String> overrides = Maps.newHashMap();
  overrides.put(FileChannelConfiguration.CAPACITY, String.valueOf(5));
  overrides.put(FileChannelConfiguration.TRANSACTION_CAPACITY,
      String.valueOf(5));
  channel = createFileChannel(overrides);
  channel.start();
  Assert.assertTrue(channel.isOpen());
  fillChannel(channel, "fillup");
  // take an event, roll it back, and
  // then make sure a put fails
  Transaction transaction;
  transaction = channel.getTransaction();
  transaction.begin();
  Event event = channel.take();
  Assert.assertNotNull(event);
  transaction.rollback();
  transaction.close();
  // ensure the take the didn't change the state of the capacity
  Assert.assertEquals(0, fillChannel(channel, "capacity").size());
  // ensure we the events back
  Assert.assertEquals(5, takeEvents(channel, 1, 5).size());
}
 

@Test
public void testRollbackSimulatedCrash() throws Exception {
  channel.start();
  Assert.assertTrue(channel.isOpen());
  int numEvents = 50;
  Set<String> in = putEvents(channel, "rollback", 1, numEvents);

  Transaction transaction;
  // put an item we will rollback
  transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody("rolled back".getBytes(Charsets.UTF_8)));
  transaction.rollback();
  transaction.close();

  // simulate crash
  channel.stop();
  channel = createFileChannel();
  channel.start();
  Assert.assertTrue(channel.isOpen());

  // we should not get the rolled back item
  Set<String> out = takeEvents(channel, 1, numEvents);
  compareInputAndOut(in, out);
}
 

@Test
public void testPut2() throws Exception {
  Transaction transaction = channel.getTransaction();
  transaction.begin();
  channel.put(events.get(0));
  transaction.rollback();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        channel.put(events.get(0));
      }
    });

  transaction.close();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        channel.put(events.get(0));
      }
    });
}
 

@Test
public void testTake2() throws Exception {
  Transaction transaction = channel.getTransaction();
  transaction.begin();
  channel.take();
  transaction.rollback();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        channel.take();
      }
    });

  transaction.close();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        channel.take();
      }
    });
}
 

@Test
public void testCommit2() throws Exception {
  final Transaction transaction = channel.getTransaction();

  transaction.begin();
  transaction.rollback();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.commit();
      }
    });

  transaction.close();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.commit();
      }
    });
}
 

private void handleTransactionFailure(Transaction txn)
    throws EventDeliveryException {
  try {
    txn.rollback();
  } catch (Throwable e) {
    logger.error("Failed to commit transaction." +
        "Transaction rolled back.", e);
    if(e instanceof Error || e instanceof RuntimeException){
      logger.error("Failed to commit transaction." +
          "Transaction rolled back.", e);
      Throwables.propagate(e);
    } else {
      logger.error("Failed to commit transaction." +
          "Transaction rolled back.", e);
      throw new EventDeliveryException("Failed to commit transaction." +
          "Transaction rolled back.", e);
    }
  } finally {
    txn.close();
  }
}
 

@Override
public Status process() throws EventDeliveryException {
  Status status = Status.READY;

  Channel channel = getChannel();
  Transaction transaction = channel.getTransaction();
  Event event = null;
  long eventCounter = counterGroup.get("events.success");

  try {
    transaction.begin();
    int i = 0;
    for (i = 0; i < batchSize; i++) {
      event = channel.take();
      if (++eventCounter % logEveryNEvents == 0) {
        logger.info("Null sink {} successful processed {} events.", getName(), eventCounter);
      }
      if(event == null) {
        status = Status.BACKOFF;
        break;
      }
    }
    transaction.commit();
    counterGroup.addAndGet("events.success", (long) Math.min(batchSize, i));
    counterGroup.incrementAndGet("transaction.success");
  } catch (Exception ex) {
    transaction.rollback();
    counterGroup.incrementAndGet("transaction.failed");
    logger.error("Failed to deliver event. Exception follows.", ex);
    throw new EventDeliveryException("Failed to deliver event: " + event, ex);
  } finally {
    transaction.close();
  }

  return status;
}
 

@Test
public void testPut3() throws Exception {
  Transaction transaction = channel.getTransaction();
  transaction.begin();
  channel.put(events.get(0));

  final Transaction finalTransaction = transaction;
  testChannelException(new Runnable() {
      @Override
      public void run() {
        finalTransaction.commit();
      }
    });

  transaction.rollback();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        channel.put(events.get(0));
      }
    });

  transaction.close();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        channel.put(events.get(0));
      }
    });
}
 

@Test
public void testRollback1() throws Exception {
  final Transaction transaction = channel.getTransaction();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.rollback();
      }
    });

  transaction.begin();

  testWrongThread(new Runnable() {
      @Override
      public void run() {
        transaction.rollback();
      }
    });

  transaction.rollback();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.rollback();
      }
    });

  transaction.close();

  testIllegalState(new Runnable() {
      @Override
      public void run() {
        transaction.rollback();
      }
    });
}
 

@Override
public Status process() throws EventDeliveryException {
    List<Event> events;
    List<Map<String, Object>> parsedEvents;
    Status status = Status.BACKOFF;
    Transaction transaction = this.getChannel().getTransaction();
    try {
        transaction.begin();
        events = takeEventsFromChannel(this.getChannel(), batchSize);
        status = Status.READY;
        if (!events.isEmpty()) {
            updateSinkCounters(events);
            parsedEvents = eventParser.parse(events);
            sendEvents(parsedEvents);
            sinkCounter.addToEventDrainSuccessCount(events.size());
        } else {
            sinkCounter.incrementBatchEmptyCount();
        }
        transaction.commit();
        status = Status.READY;
    } catch (ChannelException e) {
        e.printStackTrace();
        transaction.rollback();
        status = Status.BACKOFF;
        this.sinkCounter.incrementConnectionFailedCount();
    } catch (Throwable t) {
        t.printStackTrace();
        transaction.rollback();
        status = Status.BACKOFF;
        if (t instanceof Error) {
            LOG.error(t.getMessage());
            throw new DruidSinkException("An error occurred during processing events to be stored in druid", t);

        }
    } finally {
        transaction.close();
    }
    return status;
}
 

public static Set<String> putEvents(Channel channel, String prefix,
        int batchSize, int numEvents, boolean untilCapacityIsReached)
            throws Exception {
  Set<String> result = Sets.newHashSet();
  for (int i = 0; i < numEvents; i += batchSize) {
    Transaction transaction = channel.getTransaction();
    transaction.begin();
    try {
      Set<String> batch = Sets.newHashSet();
      for (int j = 0; j < batchSize; j++) {
        String s = prefix + "-" + i + "-" + j + "-" + UUID.randomUUID();
        Event event = EventBuilder.withBody(s.getBytes(Charsets.UTF_8));
        channel.put(event);
        batch.add(s);
      }
      transaction.commit();
      result.addAll(batch);
    } catch (Exception ex) {
      transaction.rollback();
      if(untilCapacityIsReached && ex instanceof ChannelException &&
          ("The channel has reached it's capacity. "
              + "This might be the result of a sink on the channel having too "
              + "low of batch size, a downstream system running slower than "
              + "normal, or that the channel capacity is just too low. "
              + "[channel=" +channel.getName() + "]").
            equals(ex.getMessage())) {
        break;
      }
      throw ex;
    } finally {
      transaction.close();
    }
  }
  return result;
}
 

@Test
public void testReferenceCounts() throws Exception {
  Map<String, String> overrides = Maps.newHashMap();
  overrides.put(FileChannelConfiguration.CHECKPOINT_INTERVAL, "10000");
  overrides.put(FileChannelConfiguration.MAX_FILE_SIZE, "150");
  final FileChannel channel = createFileChannel(overrides);
  channel.start();
  putEvents(channel, "testing-reference-counting", 1, 15);
  Transaction tx = channel.getTransaction();
  takeWithoutCommit(channel, tx, 10);
  forceCheckpoint(channel);
  tx.rollback();
  //Since we did not commit the original transaction. now we should get 15
  //events back.
  final Set<String> takenEvents = Sets.newHashSet();
  Executors.newSingleThreadExecutor().submit(new Runnable() {
    @Override
    public void run() {
      try {
        takenEvents.addAll(takeEvents(channel, 15));
      } catch (Exception ex) {
        Throwables.propagate(ex);
      }
    }
  }).get();
  Assert.assertEquals(15, takenEvents.size());
}
 

@Override
public Status process() throws EventDeliveryException {
  logger.debug("processing...");
  Status status = Status.READY;
  Channel channel = getChannel();
  Transaction txn = channel.getTransaction();
  try {
    txn.begin();
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for (int i = 0; i < batchSize; i++) {
      Event event = channel.take();

      if (event == null) {
        break;
      }

      IndexRequestBuilder indexRequest =
          indexRequestFactory.createIndexRequest(
              client, indexName, indexType, event);

      if (ttlMs > 0) {
        indexRequest.setTTL(ttlMs);
      }

      bulkRequest.add(indexRequest);
    }

    int size = bulkRequest.numberOfActions();
    if (size <= 0) {
      sinkCounter.incrementBatchEmptyCount();
      counterGroup.incrementAndGet("channel.underflow");
      status = Status.BACKOFF;
    } else {
      if (size < batchSize) {
        sinkCounter.incrementBatchUnderflowCount();
        status = Status.BACKOFF;
      } else {
        sinkCounter.incrementBatchCompleteCount();
      }

      sinkCounter.addToEventDrainAttemptCount(size);

      BulkResponse bulkResponse = bulkRequest.execute().actionGet();
      if (bulkResponse.hasFailures()) {
        throw new EventDeliveryException(bulkResponse.buildFailureMessage());
      }
    }
    txn.commit();
    sinkCounter.addToEventDrainSuccessCount(size);
    counterGroup.incrementAndGet("transaction.success");
  } catch (Throwable ex) {
    try {
      txn.rollback();
      counterGroup.incrementAndGet("transaction.rollback");
    } catch (Exception ex2) {
      logger.error(
          "Exception in rollback. Rollback might not have been successful.",
          ex2);
    }

    if (ex instanceof Error || ex instanceof RuntimeException) {
      logger.error("Failed to commit transaction. Transaction rolled back.",
          ex);
      Throwables.propagate(ex);
    } else {
      logger.error("Failed to commit transaction. Transaction rolled back.",
          ex);
      throw new EventDeliveryException(
          "Failed to commit transaction. Transaction rolled back.", ex);
    }
  } finally {
    txn.close();
  }
  return status;
}
 

@Override
public Status process() {
    if (shouldBackOff.get()) {
        throw new NoNodeAvailableException("Check whether Elasticsearch is down or not.");
    }
    Channel channel = getChannel();
    Transaction txn = channel.getTransaction();
    txn.begin();
    try {
        Event event = channel.take();
        if (event != null) {
            String body = new String(event.getBody(), Charsets.UTF_8);
            if (!Strings.isNullOrEmpty(body)) {
                logger.debug("start to sink event [{}].", body);
                String index = indexBuilder.getIndex(event);
                String type = indexBuilder.getType(event);
                String id = indexBuilder.getId(event);
                XContentBuilder xContentBuilder = serializer.serialize(event);
                if (xContentBuilder != null) {
                    if (!(Strings.isNullOrEmpty(id))) {
                        bulkProcessor.add(new IndexRequest(index, type, id)
                                .source(xContentBuilder));
                    } else {
                        bulkProcessor.add(new IndexRequest(index, type)
                                .source(xContentBuilder));
                    }
                } else {
                    logger.error("Could not serialize the event body [{}] for index [{}], type[{}] and id [{}] ",
                            new Object[]{body, index, type, id});
                }
            }
            logger.debug("sink event [{}] successfully.", body);
        }
        txn.commit();
        return Status.READY;
    } catch (Throwable tx) {
        try {
            txn.rollback();
        } catch (Exception ex) {
            logger.error("exception in rollback.", ex);
        }
        logger.error("transaction rolled back.", tx);
        return Status.BACKOFF;
    } finally {
        txn.close();
    }
}
 

@Override
public Status process() throws EventDeliveryException {

    Status status = Status.READY;

    Channel channel = getChannel();
    Transaction tx = channel.getTransaction();
    try {
        tx.begin();
        
        List<KeyedMessage<String, String>> datas = new ArrayList<KeyedMessage<String, String>>();

        int txnEventCount = 0;
        for (txnEventCount = 0; txnEventCount < batchNumMessages; txnEventCount++) {
            Event event = channel.take();
            if (event == null) {
            	break;
            }         
            Map<String, String> headers = event.getHeaders();
            if(headers == null){
              logger.warn("headers are Null");
              continue;
            }
            
            String topic = headers.get("category");
            if(topic == null){
              logger.warn("headers do not contain entry of category");
              continue;
            }
            topic = topicPrefix + "." + topic;
            
            KeyedMessage<String, String> m = new KeyedMessage<String, String>(topic, new String(event.getBody()));
            datas.add(m);
        }

        producer.send(datas);  

        tx.commit();
    } catch (Exception e) {
        logger.error("can't process events, drop it!", e);
        tx.rollback();
        throw new EventDeliveryException(e);
    } finally {
    	tx.close();
    }
    return status;
}
 

@Test
public void testChannelResize() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("capacity", "5");
  parms.put("transactionCapacity", "5");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  Transaction transaction = channel.getTransaction();
  transaction.begin();
  for(int i=0; i < 5; i++) {
    channel.put(EventBuilder.withBody(String.format("test event %d", i).getBytes()));
  }
  transaction.commit();
  transaction.close();

  /*
   * Verify overflow semantics
   */
  transaction = channel.getTransaction();
  boolean overflowed = false;
  try {
    transaction.begin();
    channel.put(EventBuilder.withBody("overflow event".getBytes()));
    transaction.commit();
  } catch (ChannelException e) {
    overflowed = true;
    transaction.rollback();
  } finally {
    transaction.close();
  }
  Assert.assertTrue(overflowed);

  /*
   * Reconfigure capacity down and add another event, shouldn't result in exception
   */
  parms.put("capacity", "6");
  context.putAll(parms);
  Configurables.configure(channel, context);
  transaction = channel.getTransaction();
  transaction.begin();
  channel.put(EventBuilder.withBody("extended capacity event".getBytes()));
  transaction.commit();
  transaction.close();

  /*
   * Attempt to reconfigure capacity to below current entry count and verify
   * it wasn't carried out
   */
  parms.put("capacity", "2");
  parms.put("transactionCapacity", "2");
  context.putAll(parms);
  Configurables.configure(channel, context);
  for(int i=0; i < 6; i++) {
    transaction = channel.getTransaction();
    transaction.begin();
    Assert.assertNotNull(channel.take());
    transaction.commit();
    transaction.close();
  }
}
 

@Ignore("BasicChannelSemantics doesn't support re-entrant transactions")
@Test
public void testReEntTxnRollBack() throws InterruptedException,
    EventDeliveryException {
  Event event, event2;
  Context context = new Context();
  int putCounter = 0;

  context.put("keep-alive", "1");
  Configurables.configure(channel, context);

  Transaction transaction = channel.getTransaction();
  Assert.assertNotNull(transaction);

  // add events and rollback txn
  transaction.begin();
  for (putCounter = 0; putCounter < 10; putCounter++) {
    event = EventBuilder.withBody(("test event" + putCounter).getBytes());
    channel.put(event);
  }
  transaction.rollback();
  transaction.close();

  // verify that no events are stored due to rollback
  transaction = channel.getTransaction();
  transaction.begin();
  event2 = channel.take();
  Assert.assertNull("extra event found", event2);
  transaction.commit();
  transaction.close();

  // add events and commit
  transaction = channel.getTransaction();
  transaction.begin();
  for (putCounter = 0; putCounter < 10; putCounter++) {
    event = EventBuilder.withBody(("test event" + putCounter).getBytes());
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  transaction = channel.getTransaction();
  Assert.assertNotNull(transaction);

  // verify events are there, then rollback the take
  transaction.begin();
  for (int i = 0; i < 10; i++) {
    transaction.begin(); // inner begin
    event2 = channel.take();
    Assert.assertNotNull("lost an event", event2);
    Assert.assertArrayEquals(event2.getBody(), ("test event" + i).getBytes());
    transaction.commit(); // inner commit
  }
  event2 = channel.take();
  Assert.assertNull("extra event found", event2);

  transaction.rollback();
  transaction.close();

  // verify that the events were left in there due to rollback
  transaction = channel.getTransaction();
  transaction.begin();
  for (int i = 0; i < 10; i++) {
    event2 = channel.take();
    Assert.assertNotNull("lost an event", event2);
    Assert.assertArrayEquals(event2.getBody(), ("test event" + i).getBytes());
  }
  event2 = channel.take();
  Assert.assertNull("extra event found", event2);

  transaction.rollback();
  transaction.close();
}
 

private void putEventsAndCommit(final List<Row> actions, final List<Increment> incs,
    Transaction txn) throws EventDeliveryException {
  try {
    runPrivileged(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        for(Row r : actions) {
          if(r instanceof Put) {
            ((Put)r).setWriteToWAL(enableWal);
          }
          // Newer versions of HBase - Increment implements Row.
          if(r instanceof Increment) {
            ((Increment)r).setWriteToWAL(enableWal);
          }
        }
        table.batch(actions);
        return null;
      }
    });

    runPrivileged(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        for (final Increment i : incs) {
          i.setWriteToWAL(enableWal);
          table.increment(i);
        }
        return null;
      }
    });

    txn.commit();
    sinkCounter.addToEventDrainSuccessCount(actions.size());
  } catch (Throwable e) {
    try{
      txn.rollback();
    } catch (Exception e2) {
      logger.error("Exception in rollback. Rollback might not have been" +
          "successful." , e2);
    }
    logger.error("Failed to commit transaction." +
        "Transaction rolled back.", e);
    if(e instanceof Error || e instanceof RuntimeException){
      logger.error("Failed to commit transaction." +
          "Transaction rolled back.", e);
      Throwables.propagate(e);
    } else {
      logger.error("Failed to commit transaction." +
          "Transaction rolled back.", e);
      throw new EventDeliveryException("Failed to commit transaction." +
          "Transaction rolled back.", e);
    }
  } finally {
    txn.close();
  }
}
 

public Status process() throws EventDeliveryException {
    Status status = Status.BACKOFF;
    Transaction transaction = this.getChannel().getTransaction();
    try {
        transaction.begin();
        List<Event> eventList = this.takeEventsFromChannel(this.getChannel());
        status = Status.READY;
        if (!eventList.isEmpty()) {
            if (eventList.size() == this.batchsize) {
                this.sinkCounter.incrementBatchCompleteCount();
            } else {
                this.sinkCounter.incrementBatchUnderflowCount();
            }
            for (Event event : eventList) {
                List<ColumnNameValue> columnNameValueList = getColumnNameValueListFromEvent(event);

                if (event.getHeaders().containsKey(TOPIC_EVENT_HEADER)) {
                    // If we've defined the _topic header using in the event, we send the data to this topic
                    stratioStreamingAPI.insertData(this.streamName, columnNameValueList,
                            getEventDefinedTopicName(event.getHeaders().get(TOPIC_EVENT_HEADER)), false);

                } else if (!this.topic.isEmpty())   {
                    // If we've specified a topic in the properties file we send the data to that topic
                    stratioStreamingAPI.insertData(this.streamName, columnNameValueList, this.topic, false);
                }   else    {
                    // In other case we send the data to default topic, don't specifying any topic name
                    stratioStreamingAPI.insertData(this.streamName, columnNameValueList);
                }
            }
            this.sinkCounter.addToEventDrainSuccessCount(eventList.size());
        } else {
            this.sinkCounter.incrementBatchEmptyCount();
        }
        transaction.commit();
        status = Status.READY;
    } catch (ChannelException e) {
        e.printStackTrace();
        transaction.rollback();
        status = Status.BACKOFF;
        this.sinkCounter.incrementConnectionFailedCount();
    } catch (Throwable t) {
        t.printStackTrace();
        transaction.rollback();
        status = Status.BACKOFF;
        if (t instanceof Error) {
            throw new StratioDecisionSinkException(t);
        }
    } finally {
        transaction.close();
    }
    return status;
}
 

@Override
public Status process() throws EventDeliveryException {
  logger.debug("processing...");
  Status status = Status.READY;
  Channel channel = getChannel();
  Transaction txn = channel.getTransaction();
  try {
    txn.begin();
    int count;
    for (count = 0; count < batchSize; ++count) {
      Event event = channel.take();

      if (event == null) {
        break;
      }
      String realIndexType = BucketPath.escapeString(indexType, event.getHeaders());
      client.addEvent(event, indexNameBuilder, realIndexType, ttlMs);
    }

    if (count <= 0) {
      sinkCounter.incrementBatchEmptyCount();
      counterGroup.incrementAndGet("channel.underflow");
      status = Status.BACKOFF;
    } else {
      if (count < batchSize) {
        sinkCounter.incrementBatchUnderflowCount();
        status = Status.BACKOFF;
      } else {
        sinkCounter.incrementBatchCompleteCount();
      }

      sinkCounter.addToEventDrainAttemptCount(count);
      client.execute();
    }
    txn.commit();
    sinkCounter.addToEventDrainSuccessCount(count);
    counterGroup.incrementAndGet("transaction.success");
  } catch (Throwable ex) {
    try {
      txn.rollback();
      counterGroup.incrementAndGet("transaction.rollback");
    } catch (Exception ex2) {
      logger.error(
          "Exception in rollback. Rollback might not have been successful.",
          ex2);
    }

    if (ex instanceof Error || ex instanceof RuntimeException) {
      logger.error(FAILED_TO_COMMIT_TRANSACTION_TRANSACTION_ROLLED_BACK,
          ex);
      Throwables.propagate(ex);
    } else {
      logger.error(FAILED_TO_COMMIT_TRANSACTION_TRANSACTION_ROLLED_BACK,
          ex);
      throw new EventDeliveryException(
              FAILED_TO_COMMIT_TRANSACTION_TRANSACTION_ROLLED_BACK, ex);
    }
  } finally {
    txn.close();
  }
  return status;
}