org.apache.avro.ipc.Server Java Examples

The following examples show how to use org.apache.avro.ipc.Server. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RunLocalTest.java    From hadoop-arch-book with Apache License 2.0 6 votes vote down vote up
public static Server startTestFlumeServer(int port) {
  Responder responder = new SpecificResponder(AvroSourceProtocol.class,
          new OKAvroHandler());
  Server server = new NettyServer(responder,
            new InetSocketAddress("127.0.0.1", port));

  server.start();
  LOG.info("Server started on test flume server hostname: localhost, port: " + port);

  try {

    Thread.sleep(1000L);

  } catch (InterruptedException ex) {
    LOG.error("Thread interrupted. Exception follows.", ex);
    Thread.currentThread().interrupt();
  }

  return server;
}
 
Example #2
Source File: TestNettyAvroRpcClient.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
/**
 * First connect the client, then shut down the server, then send a request.
 * @throws FlumeException
 * @throws EventDeliveryException
 * @throws InterruptedException
 */
@Test(expected=EventDeliveryException.class)
public void testServerDisconnect() throws FlumeException,
    EventDeliveryException, InterruptedException {
  NettyAvroRpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcTestUtils.getStockLocalClient(server.getPort());
    server.close();
    Thread.sleep(1000L); // wait a second for the close to occur
    try {
      server.join();
    } catch (InterruptedException ex) {
      logger.warn("Thread interrupted during join()", ex);
      Thread.currentThread().interrupt();
    }
    try {
      client.append(EventBuilder.withBody("hello", Charset.forName("UTF8")));
    } finally {
      Assert.assertFalse("Client should not be active", client.isActive());
    }
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 
Example #3
Source File: TestNettyAvroRpcClient.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
/**
 * First connect the client, then close the client, then send a request.
 * @throws FlumeException
 * @throws EventDeliveryException
 */
@Test(expected=EventDeliveryException.class)
public void testClientClosedRequest() throws FlumeException,
    EventDeliveryException {
  NettyAvroRpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcTestUtils.getStockLocalClient(server.getPort());
    client.close();
    Assert.assertFalse("Client should not be active", client.isActive());
    System.out.println("Yaya! I am not active after client close!");
    client.append(EventBuilder.withBody("hello", Charset.forName("UTF8")));
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 
Example #4
Source File: AvroProtocol.java    From dubbox with Apache License 2.0 6 votes vote down vote up
@Override
protected <T> Runnable doExport(T impl, Class<T> type, URL url)
        throws RpcException {

    logger.info("impl => " + impl.getClass());
    logger.info("type => " + type.getName());
    logger.info("url => " + url);

    final Server server = new NettyServer(new ReflectResponder(type, impl),
            new InetSocketAddress(url.getHost(), url.getPort()));
    server.start();

    return new Runnable() {
        public void run() {
            try {
                logger.info("Close Avro Server");
                server.close();
            } catch (Throwable e) {
                logger.warn(e.getMessage(), e);
            }
        }
    };
}
 
Example #5
Source File: TestRpcClientFactory.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test
public void testThreeParamBatchAppend() throws FlumeException,
    EventDeliveryException {
  int batchSize = 7;
  RpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcClientFactory.getDefaultInstance(localhost, server.getPort(),
        batchSize);

    List<Event> events = new ArrayList<Event>();
    for (int i = 0; i < batchSize; i++) {
      events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
    }
    client.appendBatch(events);
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 
Example #6
Source File: TestRpcClientFactory.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test
public void testPropertiesBatchAppend() throws FlumeException,
    EventDeliveryException {
  int batchSize = 7;
  RpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    Properties p = new Properties();
    p.put("hosts", "host1");
    p.put("hosts.host1", localhost + ":" + String.valueOf(server.getPort()));
    p.put("batch-size", String.valueOf(batchSize));
    client = RpcClientFactory.getInstance(p);
    List<Event> events = new ArrayList<Event>();
    for (int i = 0; i < batchSize; i++) {
      events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
    }
    client.appendBatch(events);
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 
Example #7
Source File: TestRpcClientFactory.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test
public void testTwoParamBatchAppendOverflow() throws FlumeException,
    EventDeliveryException {
  RpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcClientFactory.getDefaultInstance(localhost, server.getPort());
    int batchSize = client.getBatchSize();
    int moreThanBatch = batchSize + 1;
    List<Event> events = new ArrayList<Event>();
    for (int i = 0; i < moreThanBatch; i++) {
      events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
    }
    client.appendBatch(events);
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 
Example #8
Source File: RpcTestUtils.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
/**
 * Helper method for testing simple (single) with compression level 6 appends on handlers
 * @param handler
 * @throws FlumeException
 * @throws EventDeliveryException
 */
public static void handlerSimpleAppendTest(AvroSourceProtocol handler, boolean enableServerCompression, boolean enableClientCompression, int compressionLevel)
    throws FlumeException, EventDeliveryException {
  NettyAvroRpcClient client = null;
  Server server = startServer(handler, 0, enableServerCompression);
  try {
    Properties starterProp = new Properties();
    if (enableClientCompression) {
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "deflate");
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_LEVEL, "" + compressionLevel);
    } else {
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "none");
    }
    client = getStockLocalClient(server.getPort(), starterProp);
    boolean isActive = client.isActive();
    Assert.assertTrue("Client should be active", isActive);
    client.append(EventBuilder.withBody("wheee!!!", Charset.forName("UTF8")));
  } finally {
    stopServer(server);
    if (client != null) client.close();
  }
}
 
Example #9
Source File: TestAvroSink.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test
public void testLifecycle() throws InterruptedException,
    InstantiationException, IllegalAccessException {
  setUp();
  Server server = createServer(new MockAvroServer());

  server.start();

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 
Example #10
Source File: TestLoadBalancingRpcClient.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test(expected=FlumeException.class)
public void testCreatingLbClientSingleHost() {
  Server server1 = null;
  RpcClient c = null;
  try {
    server1 = RpcTestUtils.startServer(new OKAvroHandler());
    Properties p = new Properties();
    p.put("host1", "127.0.0.1:" + server1.getPort());
    p.put("hosts", "host1");
    p.put("client.type", "default_loadbalance");
    RpcClientFactory.getInstance(p);
  } finally {
    if (server1 != null) server1.close();
    if (c != null) c.close();
  }
}
 
Example #11
Source File: TestLoadBalancingRpcClient.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
@Test
public void testLbDefaultClientTwoHostsBatch() throws Exception {
  Server s1 = null, s2 = null;
  RpcClient c = null;
  try{
    LoadBalancedAvroHandler h1 = new LoadBalancedAvroHandler();
    LoadBalancedAvroHandler h2 = new LoadBalancedAvroHandler();

    s1 = RpcTestUtils.startServer(h1);
    s2 = RpcTestUtils.startServer(h2);

    Properties p = new Properties();
    p.put("hosts", "h1 h2");
    p.put("client.type", "default_loadbalance");
    p.put("hosts.h1", "127.0.0.1:" + s1.getPort());
    p.put("hosts.h2", "127.0.0.1:" + s2.getPort());

    c = RpcClientFactory.getInstance(p);
    Assert.assertTrue(c instanceof LoadBalancingRpcClient);

    for (int i = 0; i < 100; i++) {
      c.appendBatch(getBatchedEvent(i));
    }

    Assert.assertEquals(50, h1.getAppendBatchCount());
    Assert.assertEquals(50, h2.getAppendBatchCount());
  } finally {
    if (s1 != null) s1.close();
    if (s2 != null) s2.close();
    if (c != null) c.close();
  }
}
 
Example #12
Source File: TestLoadBalancingRpcClient.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
@Test
public void testTwoHostFailoverBatch() throws Exception {
  Server s1 = null, s2 = null;
  RpcClient c = null;
  try{
    LoadBalancedAvroHandler h1 = new LoadBalancedAvroHandler();
    LoadBalancedAvroHandler h2 = new LoadBalancedAvroHandler();

    s1 = RpcTestUtils.startServer(h1);
    s2 = RpcTestUtils.startServer(h2);

    Properties p = new Properties();
    p.put("hosts", "h1 h2");
    p.put("client.type", "default_loadbalance");
    p.put("hosts.h1", "127.0.0.1:" + s1.getPort());
    p.put("hosts.h2", "127.0.0.1:" + s2.getPort());

    c = RpcClientFactory.getInstance(p);
    Assert.assertTrue(c instanceof LoadBalancingRpcClient);

    for (int i = 0; i < 100; i++) {
      if (i == 20) {
        h2.setFailed();
      } else if (i == 40) {
        h2.setOK();
      }

      c.appendBatch(getBatchedEvent(i));
    }

    Assert.assertEquals(60, h1.getAppendBatchCount());
    Assert.assertEquals(40, h2.getAppendBatchCount());
  } finally {
    if (s1 != null) s1.close();
    if (s2 != null) s2.close();
    if (c != null) c.close();
  }
}
 
Example #13
Source File: TestAvroSink.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
private Server createServer(AvroSourceProtocol protocol)
    throws IllegalAccessException, InstantiationException {

  Server server = new NettyServer(new SpecificResponder(
      AvroSourceProtocol.class, protocol), new InetSocketAddress(
      hostname, port));

  return server;
}
 
Example #14
Source File: TestAvroSink.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
@Test
public void testProcess() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();

  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createServer(new MockAvroServer());

  server.start();

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 
Example #15
Source File: RpcTestUtils.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
/**
 * Helper method for testing batch appends on handlers
 * @param handler
 * @throws FlumeException
 * @throws EventDeliveryException
 */
public static void handlerBatchAppendTest(AvroSourceProtocol handler, boolean enableServerCompression, boolean enableClientCompression, int compressionLevel)
    throws FlumeException, EventDeliveryException {
  NettyAvroRpcClient client = null;
  Server server = startServer(handler, 0 , enableServerCompression);
  try {

    Properties starterProp = new Properties();
    if (enableClientCompression) {
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "deflate");
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_LEVEL, "" + compressionLevel);
    } else {
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "none");
    }

    client = getStockLocalClient(server.getPort(), starterProp);
    boolean isActive = client.isActive();
    Assert.assertTrue("Client should be active", isActive);

    int batchSize = client.getBatchSize();
    List<Event> events = new ArrayList<Event>();
    for (int i = 0; i < batchSize; i++) {
      events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
    }
    client.appendBatch(events);

  } finally {
    stopServer(server);
    if (client != null) client.close();
  }
}
 
Example #16
Source File: RpcTestUtils.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
/**
 * Request that the specified Server stop, and attempt to wait for it to exit.
 * @param server A running NettyServer
 */
public static void stopServer(Server server) {
  try {
    server.close();
    server.join();
  } catch (InterruptedException ex) {
    logger.error("Thread interrupted. Exception follows.", ex);
    Thread.currentThread().interrupt();
  }
}
 
Example #17
Source File: TestNettyAvroRpcClient.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
/**
 * Send too many events at once. Should handle this case gracefully.
 * @throws FlumeException
 * @throws EventDeliveryException
 */
@Test
public void testBatchOverrun() throws FlumeException, EventDeliveryException {

  int batchSize = 10;
  int moreThanBatchSize = batchSize + 1;
  NettyAvroRpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  Properties props = new Properties();
  props.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS, "localhost");
  props.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + "localhost",
      localhost + ":" + server.getPort());
  props.setProperty(RpcClientConfigurationConstants.CONFIG_BATCH_SIZE, "" + batchSize);
  try {
    client = new NettyAvroRpcClient();
    client.configure(props);

    // send one more than the batch size
    List<Event> events = new ArrayList<Event>();
    for (int i = 0; i < moreThanBatchSize; i++) {
      events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
    }
    client.appendBatch(events);
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 
Example #18
Source File: TestLoadBalancingRpcClient.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
@Test
public void testLbDefaultClientTwoHosts() throws Exception {
  Server s1 = null, s2 = null;
  RpcClient c = null;
  try{
    LoadBalancedAvroHandler h1 = new LoadBalancedAvroHandler();
    LoadBalancedAvroHandler h2 = new LoadBalancedAvroHandler();

    s1 = RpcTestUtils.startServer(h1);
    s2 = RpcTestUtils.startServer(h2);

    Properties p = new Properties();
    p.put("hosts", "h1 h2");
    p.put("client.type", "default_loadbalance");
    p.put("hosts.h1", "127.0.0.1:" + s1.getPort());
    p.put("hosts.h2", "127.0.0.1:" + s2.getPort());

    c = RpcClientFactory.getInstance(p);
    Assert.assertTrue(c instanceof LoadBalancingRpcClient);

    for (int i = 0; i < 100; i++) {
      c.append(getEvent(i));
    }

    Assert.assertEquals(50, h1.getAppendCount());
    Assert.assertEquals(50, h2.getAppendCount());
  } finally {
    if (s1 != null) s1.close();
    if (s2 != null) s2.close();
    if (c != null) c.close();
  }
}
 
Example #19
Source File: TestLoadBalancingRpcClient.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
@Test(expected = EventDeliveryException.class)
public void testTwoHostFailoverThrowAfterClose() throws Exception {
  Server s1 = null, s2 = null;
  RpcClient c = null;
  try{
    LoadBalancedAvroHandler h1 = new LoadBalancedAvroHandler();
    LoadBalancedAvroHandler h2 = new LoadBalancedAvroHandler();

    s1 = RpcTestUtils.startServer(h1);
    s2 = RpcTestUtils.startServer(h2);

    Properties p = new Properties();
    p.put("hosts", "h1 h2");
    p.put("client.type", "default_loadbalance");
    p.put("hosts.h1", "127.0.0.1:" + s1.getPort());
    p.put("hosts.h2", "127.0.0.1:" + s2.getPort());

    c = RpcClientFactory.getInstance(p);
    Assert.assertTrue(c instanceof LoadBalancingRpcClient);

    for (int i = 0; i < 100; i++) {
      if (i == 20) {
        h2.setFailed();
      } else if (i == 40) {
        h2.setOK();
      }
      c.append(getEvent(i));
    }

    Assert.assertEquals(60, h1.getAppendCount());
    Assert.assertEquals(40, h2.getAppendCount());
    if (c != null) c.close();
    c.append(getEvent(3));
    Assert.fail();
  } finally {
    if (s1 != null) s1.close();
    if (s2 != null) s2.close();
  }
}
 
Example #20
Source File: TestLoadBalancingRpcClient.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
@Test
public void testTwoHostFailover() throws Exception {
  Server s1 = null, s2 = null;
  RpcClient c = null;
  try{
    LoadBalancedAvroHandler h1 = new LoadBalancedAvroHandler();
    LoadBalancedAvroHandler h2 = new LoadBalancedAvroHandler();

    s1 = RpcTestUtils.startServer(h1);
    s2 = RpcTestUtils.startServer(h2);

    Properties p = new Properties();
    p.put("hosts", "h1 h2");
    p.put("client.type", "default_loadbalance");
    p.put("hosts.h1", "127.0.0.1:" + s1.getPort());
    p.put("hosts.h2", "127.0.0.1:" + s2.getPort());

    c = RpcClientFactory.getInstance(p);
    Assert.assertTrue(c instanceof LoadBalancingRpcClient);

    for (int i = 0; i < 100; i++) {
      if (i == 20) {
        h2.setFailed();
      } else if (i == 40) {
        h2.setOK();
      }
      c.append(getEvent(i));
    }

    Assert.assertEquals(60, h1.getAppendCount());
    Assert.assertEquals(40, h2.getAppendCount());
  } finally {
    if (s1 != null) s1.close();
    if (s2 != null) s2.close();
    if (c != null) c.close();
  }
}
 
Example #21
Source File: TestFailoverRpcClient.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
/**
 * Try writing to some servers and then kill them all.
 *
 * @throws FlumeException
 * @throws EventDeliveryException
 */
@Test(
    expected = EventDeliveryException.class)
public void testFailedServers() throws FlumeException, EventDeliveryException {
  FailoverRpcClient client = null;
  Server server1 = RpcTestUtils.startServer(new OKAvroHandler());
  Server server2 = RpcTestUtils.startServer(new OKAvroHandler());
  Server server3 = RpcTestUtils.startServer(new OKAvroHandler());
  Properties props = new Properties();
  props.put("client.type", "default_failover");

  props.put("hosts", "host1 host2 host3");
  props.put("hosts.host1", "localhost:" + String.valueOf(server1.getPort()));
  props.put("hosts.host2", "localhost:" + String.valueOf(server2.getPort()));
  props.put("hosts.host3", " localhost:" + String.valueOf(server3.getPort()));
  client = (FailoverRpcClient) RpcClientFactory.getInstance(props);
  List<Event> events = new ArrayList<Event>();
  for (int i = 0; i < 50; i++) {
    events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
  }
  client.appendBatch(events);
  server1.close();
  server2.close();
  server3.close();
  events = new ArrayList<Event>();
  for (int i = 0; i < 50; i++) {
    events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
  }
  client.appendBatch(events);
}
 
Example #22
Source File: TestRpcClientFactory.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
@Test
public void testThreeParamDeprecatedAppend() throws FlumeException,
    EventDeliveryException {
  RpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcClientFactory.getInstance(localhost, server.getPort(), 3);
    Assert.assertEquals("Batch size was specified", 3, client.getBatchSize());
    client.append(EventBuilder.withBody("wheee!!!", Charset.forName("UTF8")));
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 
Example #23
Source File: TestRpcClientFactory.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
@Test
public void testTwoParamDeprecatedAppend() throws FlumeException,
    EventDeliveryException {
  RpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcClientFactory.getInstance(localhost, server.getPort());
    client.append(EventBuilder.withBody("wheee!!!", Charset.forName("UTF8")));
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 
Example #24
Source File: TestRpcClientFactory.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
@Test
public void testTwoParamSimpleAppend() throws FlumeException,
    EventDeliveryException {
  RpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcClientFactory.getDefaultInstance(localhost, server.getPort());
    client.append(EventBuilder.withBody("wheee!!!", Charset.forName("UTF8")));
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 
Example #25
Source File: RunLocalTest.java    From hadoop-arch-book with Apache License 2.0 5 votes vote down vote up
public static void stopTestFlumeServer(Server server) {
  try {
    server.close();
    server.join();
  } catch (InterruptedException ex) {
    LOG.error("Thread interrupted. Exception follows.", ex);
    Thread.currentThread().interrupt();
  }
}
 
Example #26
Source File: TestAvroSink.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
private Server createSslServer(AvroSourceProtocol protocol)
    throws IllegalAccessException, InstantiationException {
  Server server = new NettyServer(new SpecificResponder(
      AvroSourceProtocol.class, protocol), new InetSocketAddress(hostname, port),
          new NioServerSocketChannelFactory(
                  Executors.newCachedThreadPool(),
                  Executors.newCachedThreadPool()),
          new SSLChannelPipelineFactory(),
          null);

  return server;
}
 
Example #27
Source File: TestLoadBalancingRpcClient.java    From mt-flume with Apache License 2.0 4 votes vote down vote up
@Test
public void testLbClientTenHostRoundRobinDistributionBatch() throws Exception
{
  final int NUM_HOSTS = 10;
  final int NUM_EVENTS = 1000;
  Server[] s = new Server[NUM_HOSTS];
  LoadBalancedAvroHandler[] h = new LoadBalancedAvroHandler[NUM_HOSTS];
  RpcClient c = null;
  try{
    Properties p = new Properties();
    StringBuilder hostList = new StringBuilder("");
    for (int i = 0; i<NUM_HOSTS; i++) {
      h[i] = new LoadBalancedAvroHandler();
      s[i] = RpcTestUtils.startServer(h[i]);
      String name = "h" + i;
      p.put("hosts." + name, "127.0.0.1:" + s[i].getPort());
      hostList.append(name).append(" ");
    }

    p.put("hosts", hostList.toString().trim());
    p.put("client.type", "default_loadbalance");
    p.put("host-selector", "round_robin");

    c = RpcClientFactory.getInstance(p);
    Assert.assertTrue(c instanceof LoadBalancingRpcClient);

    for (int i = 0; i < NUM_EVENTS; i++) {
      c.appendBatch(getBatchedEvent(i));
    }

    Set<Integer> counts = new HashSet<Integer>();
    int total = 0;
    for (LoadBalancedAvroHandler handler : h) {
      total += handler.getAppendBatchCount();
      counts.add(handler.getAppendBatchCount());
    }

    Assert.assertTrue("Very unusual distribution", counts.size() == 1);
    Assert.assertTrue("Missing events", total == NUM_EVENTS);
  } finally {
    for (int i = 0; i<NUM_HOSTS; i++) {
      if (s[i] != null) s[i].close();
    }
  }
}
 
Example #28
Source File: TestAvroSink.java    From mt-flume with Apache License 2.0 4 votes vote down vote up
@Test
public void testSslSinkWithNonSslServer() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createServer(new MockAvroServer());

  server.start();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("trust-all-certs", String.valueOf(true));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  Configurables.configure(sink, context);

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  boolean failed = false;
  try {
    for (int i = 0; i < 5; i++) {
      sink.process();
      failed = true;
    }
  } catch (EventDeliveryException ex) {
    logger.info("Correctly failed to send event", ex);
  }


  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();

  if (failed) {
    Assert.fail("SSL-enabled sink successfully connected to a non-SSL-enabled server, that's wrong.");
  }
}
 
Example #29
Source File: TestAvroSink.java    From mt-flume with Apache License 2.0 4 votes vote down vote up
@Test
public void testSslProcessWithTrustStore() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createSslServer(new MockAvroServer());

  server.start();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("truststore", "src/test/resources/truststore.jks");
  context.put("truststore-password", "password");
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  Configurables.configure(sink, context);

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 
Example #30
Source File: TestAvroSink.java    From mt-flume with Apache License 2.0 4 votes vote down vote up
@Test
public void testSslProcess() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createSslServer(new MockAvroServer());

  server.start();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("trust-all-certs", String.valueOf(true));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  Configurables.configure(sink, context);

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}