com.twitter.util.Duration Java Examples

The following examples show how to use com.twitter.util.Duration. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestDistributedLogClientBuilder.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 60000)
public void testBuildClientsFromSameBuilder() throws Exception {
    DistributedLogClientBuilder builder = DistributedLogClientBuilder.newBuilder()
            .name("build-clients-from-same-builder")
            .clientId(ClientId$.MODULE$.apply("test-builder"))
            .finagleNameStr("inet!127.0.0.1:7001")
            .streamNameRegex(".*")
            .handshakeWithClientInfo(true)
            .clientBuilder(ClientBuilder.get()
                .hostConnectionLimit(1)
                .connectTimeout(Duration.fromSeconds(1))
                .tcpConnectTimeout(Duration.fromSeconds(1))
                .requestTimeout(Duration.fromSeconds(10)));
    DistributedLogClient client1 = builder.build();
    DistributedLogClient client2 = builder.build();
    assertFalse(client1 == client2);
}
 
Example #2
Source File: TerrapinServiceImplTest.java    From terrapin with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetEmptyClusters() {
  ByteBuffer key = ByteBuffer.wrap(KEY);
  TerrapinGetRequest request = prepareGetRequest();
  RequestOptions options = new RequestOptions();
  options.setSelectionPolicy(SelectionPolicy.PRIMARY_FIRST);
  request.setOptions(options);
  request.setClusterList(ImmutableList.copyOf(new String[]{}));

  TerrapinResponse response = new TerrapinResponse().setResponseMap(ImmutableMap.of(
      key, new TerrapinSingleResponse().setValue(ByteBuffer.wrap("value".getBytes()))));
  when(mockClient1.getMany(eq(FILESET), eq(Sets.newHashSet(key)))).thenReturn(
      Future.value(response));
  when(mockClient2.getManyNoRetries(eq(FILESET), eq(Sets.newHashSet(key)))).thenReturn(
      Future.value(response));

  Try<TerrapinSingleResponse> singleResponseTry = serviceIface.get(request).get(Duration.forever());
  assertTrue(singleResponseTry.isThrow());
  assertEquals(TerrapinGetErrorCode.INVALID_REQUEST,
      ((TerrapinGetException)((Throw)singleResponseTry).e()).getErrorCode());

}
 
Example #3
Source File: TerrapinServiceImplTest.java    From terrapin with Apache License 2.0 6 votes vote down vote up
@Test
public void testMultiGetEmptyClusters() {
  TerrapinMultiGetRequest request = prepareMultiGetRequest();
  RequestOptions options = new RequestOptions();
  options.setSelectionPolicy(SelectionPolicy.PRIMARY_FIRST);
  request.setOptions(options);
  request.setClusterList(ImmutableList.copyOf(new String[]{}));

  TerrapinResponse response = new TerrapinResponse().setResponseMap(ImmutableMap.of(
      ByteBuffer.wrap("key1".getBytes()),
      new TerrapinSingleResponse().setValue("value1".getBytes()),
      ByteBuffer.wrap("key2".getBytes()),
      new TerrapinSingleResponse().setErrorCode(TerrapinGetErrorCode.READ_ERROR)));
  Set<ByteBuffer> keys = Sets.newHashSet(ByteBuffer.wrap("key1".getBytes()),
      ByteBuffer.wrap("key2".getBytes()),
      ByteBuffer.wrap("key3".getBytes()));

  when(mockClient1.getMany(eq(FILESET), eq(keys))).thenReturn(Future.value(response));
  when(mockClient2.getManyNoRetries(eq(FILESET), eq(keys))).thenReturn(Future.value(response));

  Try<TerrapinResponse> returnResponseTry = serviceIface.multiGet(request).get(Duration.forever());
  assertTrue(returnResponseTry.isThrow());
  assertEquals(TerrapinGetErrorCode.INVALID_REQUEST,
      ((TerrapinGetException)((Throw)returnResponseTry).e()).getErrorCode());
}
 
Example #4
Source File: TerrapinServiceImplTest.java    From terrapin with Apache License 2.0 6 votes vote down vote up
@Test
public void testMultiGetError() {
  TerrapinMultiGetRequest request = prepareMultiGetRequest();
  Set<ByteBuffer> keys = Sets.newHashSet(ByteBuffer.wrap("key1".getBytes()),
      ByteBuffer.wrap("key2".getBytes()),
      ByteBuffer.wrap("key3".getBytes()));

  when (mockClient1.getMany(eq(FILESET), eq(keys))).thenReturn(
      Future.<TerrapinResponse>exception(new TerrapinGetException("Failed",
          TerrapinGetErrorCode.FILE_SET_NOT_FOUND)));
  Try<TerrapinResponse> responseTry = serviceIface.multiGet(request).get(Duration.forever());
  assertTrue(responseTry.isThrow());
  assertEquals(TerrapinGetErrorCode.FILE_SET_NOT_FOUND,
               ((TerrapinGetException)((Throw)responseTry).e()).getErrorCode());

  when(mockClient1.getMany(eq(FILESET), eq(keys))).thenThrow(
      new RuntimeException(new NullPointerException()));
  responseTry = serviceIface.multiGet(request).get(Duration.forever());
  assertTrue(responseTry.isThrow());
  assertEquals(TerrapinGetErrorCode.OTHER,
               ((TerrapinGetException)((Throw)responseTry).e()).getErrorCode());
}
 
Example #5
Source File: OstrichAdminService.java    From doctorkafka with Apache License 2.0 6 votes vote down vote up
public void startAdminHttpService() {
  try {
    Properties properties = new Properties();
    properties.load(this.getClass().getResource("build.properties").openStream());
    LOG.info("build.properties build_revision: {}",
        properties.getProperty("build_revision", "unknown"));
  } catch (Throwable t) {
    LOG.warn("Failed to load properties from build.properties", t);
  }
  Duration[] defaultLatchIntervals = {Duration.apply(1, TimeUnit.MINUTES)};
  Iterator<Duration> durationIterator = Arrays.asList(defaultLatchIntervals).iterator();
  @SuppressWarnings("deprecation")
  AdminServiceFactory adminServiceFactory = new AdminServiceFactory(
      this.port,
      20,
      List$.MODULE$.empty(),
      Option.empty(),
      List$.MODULE$.empty(),
      Map$.MODULE$.empty(),
      JavaConversions.asScalaIterator(durationIterator).toList());
  RuntimeEnvironment runtimeEnvironment = new RuntimeEnvironment(this);
  AdminHttpService service = adminServiceFactory.apply(runtimeEnvironment);
  for (Map.Entry<String, CustomHttpHandler> entry : this.customHttpHandlerMap.entrySet()) {
    service.httpServer().createContext(entry.getKey(), entry.getValue());
  }
}
 
Example #6
Source File: TerrapinThriftMain.java    From terrapin with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  final PropertiesConfiguration config = TerrapinUtil.readPropertiesExitOnFailure(
      System.getProperties().getProperty("terrapin.config", "thrift.properties"));

  OstrichStatsReceiver statsReceiver = new OstrichStatsReceiver(Stats.get(""));
  int listenPort = config.getInt("thrift_port", 9090);
  TerrapinServiceImpl serviceImpl = new TerrapinServiceImpl(config,
      (List) config.getList("cluster_list"));
  Service<byte[], byte[]> service = new TerrapinService.Service(serviceImpl,
      new TBinaryProtocol.Factory());
  Server server = ServerBuilder.safeBuild(
      service,
      ServerBuilder.get()
          .name("TERRAPIN_THRIFT")
          .codec(ThriftServerFramedCodec.get())
          .hostConnectionMaxIdleTime(Duration.apply(1, TimeUnit.MINUTES))
          .maxConcurrentRequests(3000)
          .reportTo(statsReceiver)
          .bindTo(new InetSocketAddress(listenPort)));
  new OstrichAdminService(config.getInt(Constants.OSTRICH_METRICS_PORT, 9999)).start();
  LOG.info("\n#######################################"
          + "\n#      Ready To Serve Requests.       #"
          + "\n#######################################");
}
 
Example #7
Source File: TerrapinServerInternalImplTest.java    From terrapin with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetErrorMultipleResources() throws Exception {
  TerrapinInternalGetRequest request = new TerrapinInternalGetRequest();
  MultiKey multiKey1 = new MultiKey().setResource("resource1").setPartition("1");
  multiKey1.addToKey(ByteBuffer.wrap("k1".getBytes()));
  MultiKey multiKey2 = new MultiKey().setResource("resource2").setPartition("1");
  multiKey2.addToKey(ByteBuffer.wrap("k2".getBytes()));
  request.addToKeyList(multiKey1);
  request.addToKeyList(multiKey2);

  Reader mockReader = mock(Reader.class);
  when(mockResourcePartitionMap.getReader(eq("resource1"), eq("1"))).thenReturn(mockReader);
  Try<TerrapinResponse> responseTry = serverImpl.get(request).get(Duration.forever());
  TerrapinGetException e = (TerrapinGetException)((Throw)responseTry).e();
  assertEquals(TerrapinGetErrorCode.INVALID_REQUEST, e.getErrorCode());
}
 
Example #8
Source File: TerrapinControllerHandler.java    From terrapin with Apache License 2.0 6 votes vote down vote up
private void startThriftServer(int thriftPort) throws UnknownHostException {
  TerrapinController.ServiceIface serviceImpl = new TerrapinControllerServiceImpl(
      this.configuration,
      this.zkManager,
      this.hdfsClient,
      this.helixAdmin,
      this.clusterName);
  TerrapinController.Service service =
      new TerrapinController.Service(serviceImpl, new TBinaryProtocol.Factory());

  this.server = ServerBuilder.safeBuild(
      service,
      ServerBuilder.get()
      .name("TerrapinController")
      .codec(ThriftServerFramedCodec.get())
      .hostConnectionMaxIdleTime(Duration.fromTimeUnit(
          configuration.getInt(Constants.THRIFT_CONN_MAX_IDLE_TIME, 1), TimeUnit.MINUTES))
      .maxConcurrentRequests(configuration.getInt(Constants.THRIFT_MAX_CONCURRENT_REQUESTS,
          100))
      .reportTo(new OstrichStatsReceiver(Stats.get("")))
      .bindTo(new InetSocketAddress(thriftPort)));
  new OstrichAdminService(configuration.getInt(Constants.OSTRICH_METRICS_PORT, 9999)).start();
}
 
Example #9
Source File: OstrichAdminService.java    From terrapin with Apache License 2.0 6 votes vote down vote up
public void start() {
  Duration[] defaultLatchIntervals = {Duration.apply(1, TimeUnit.MINUTES)};
  @SuppressWarnings("deprecation")
  AdminServiceFactory adminServiceFactory = new AdminServiceFactory(
      this.mPort,
      20,
      List$.MODULE$.<StatsFactory>empty(),
      Option.<String>empty(),
      List$.MODULE$.<Regex>empty(),
      Map$.MODULE$.<String, CustomHttpHandler>empty(),
      List.<Duration>fromArray(defaultLatchIntervals));
  RuntimeEnvironment runtimeEnvironment = new RuntimeEnvironment(this);
  AdminHttpService service = adminServiceFactory.apply(runtimeEnvironment);
  for (Map.Entry<String, CustomHttpHandler> entry: this.mCustomHttpHandlerMap.entrySet()) {
    service.httpServer().createContext(entry.getKey(), entry.getValue());
  }
}
 
Example #10
Source File: OstrichAdminService.java    From pinlater with Apache License 2.0 6 votes vote down vote up
public void start() {
  Duration[] defaultLatchIntervals = {Duration.apply(1, TimeUnit.MINUTES)};
  @SuppressWarnings("deprecation")
  AdminServiceFactory adminServiceFactory = new AdminServiceFactory(
      this.mPort,
      20,
      List$.MODULE$.<StatsFactory>empty(),
      Option.<String>empty(),
      List$.MODULE$.<Regex>empty(),
      Map$.MODULE$.<String, CustomHttpHandler>empty(),
      List.<Duration>fromArray(defaultLatchIntervals));
  RuntimeEnvironment runtimeEnvironment = new RuntimeEnvironment(this);
  AdminHttpService service = adminServiceFactory.apply(runtimeEnvironment);
  for (Map.Entry<String, CustomHttpHandler> entry : this.mCustomHttpHandlerMap.entrySet()) {
    service.httpServer().createContext(entry.getKey(), entry.getValue());
  }
}
 
Example #11
Source File: TestDistributedLogServerBase.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
/**
 * Sanity check to make sure both checksum flag values work.
 */
@Test(timeout = 60000)
public void testChecksumFlag() throws Exception {
    String name = "testChecksumFlag";
    LocalRoutingService routingService = LocalRoutingService.newBuilder().build();
    routingService.addHost(name, dlServer.getAddress());
    DistributedLogClientBuilder dlClientBuilder = DistributedLogClientBuilder.newBuilder()
        .name(name)
        .clientId(ClientId$.MODULE$.apply("test"))
        .routingService(routingService)
        .handshakeWithClientInfo(true)
        .clientBuilder(ClientBuilder.get()
            .hostConnectionLimit(1)
            .connectionTimeout(Duration.fromSeconds(1))
            .requestTimeout(Duration.fromSeconds(60)))
        .checksum(false);
    DistributedLogClient dlClient = dlClientBuilder.build();
    Await.result(dlClient.write(name, ByteBuffer.wrap(("1").getBytes())));
    dlClient.close();

    dlClient = dlClientBuilder.checksum(true).build();
    Await.result(dlClient.write(name, ByteBuffer.wrap(("2").getBytes())));
    dlClient.close();
}
 
Example #12
Source File: TerrapinServerHandler.java    From terrapin with Apache License 2.0 6 votes vote down vote up
private void startThriftServer(int thriftPort) {
  TerrapinServerInternal.ServiceIface serviceImpl = new TerrapinServerInternalImpl(configuration,
      resourcePartitionMap);
  TerrapinServerInternal.Service service =
      new TerrapinServerInternal.Service(serviceImpl, new TBinaryProtocol.Factory());

  this.server = ServerBuilder.safeBuild(
      service,
      ServerBuilder.get()
          .name("TerrapinServer")
          .codec(ThriftServerFramedCodec.get())
          .hostConnectionMaxIdleTime(Duration.fromTimeUnit(
                  configuration.getInt(Constants.THRIFT_CONN_MAX_IDLE_TIME, 1), TimeUnit.MINUTES))
          .maxConcurrentRequests(configuration.getInt(Constants.THRIFT_MAX_CONCURRENT_REQUESTS,
                  100))
          .reportTo(new OstrichStatsReceiver(Stats.get("")))
          .bindTo(new InetSocketAddress(thriftPort)));
    new OstrichAdminService(configuration.getInt(Constants.OSTRICH_METRICS_PORT, 9999)).start();  
}
 
Example #13
Source File: DistributedLogClientImpl.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
void send(SocketAddress address) {
    long elapsedMs = stopwatch.elapsed(TimeUnit.MILLISECONDS);
    if (clientConfig.getMaxRedirects() > 0
        && tries.get() >= clientConfig.getMaxRedirects()) {
        fail(address, new RequestTimeoutException(Duration.fromMilliseconds(elapsedMs),
                "Exhausted max redirects in " + elapsedMs + " ms"));
        return;
    } else if (shouldTimeout(elapsedMs)) {
        fail(address, new RequestTimeoutException(Duration.fromMilliseconds(elapsedMs),
                "Exhausted max request timeout " + clientConfig.getRequestTimeoutMs()
                        + " in " + elapsedMs + " ms"));
        return;
    }
    synchronized (this) {
        String addrStr = address.toString();
        if (ctx.isSetTriedHosts() && ctx.getTriedHosts().contains(addrStr)) {
            nextAddressToSend = address;
            dlTimer.newTimeout(this,
                    Math.min(clientConfig.getRedirectBackoffMaxMs(),
                            tries.get() * clientConfig.getRedirectBackoffStartMs()),
                    TimeUnit.MILLISECONDS);
        } else {
            doSend(address);
        }
    }
}
 
Example #14
Source File: DistributedLogServerTestCase.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
protected TwoRegionDLClient(String name, Map<SocketAddress, String> regionMap) {
    localRoutingService = new LocalRoutingService();
    remoteRoutingService = new LocalRoutingService();
    RegionsRoutingService regionsRoutingService =
            RegionsRoutingService.of(new DefaultRegionResolver(regionMap),
                    localRoutingService, remoteRoutingService);
    dlClientBuilder = DistributedLogClientBuilder.newBuilder()
                .name(name)
                .clientId(ClientId$.MODULE$.apply(name))
                .routingService(regionsRoutingService)
                .streamNameRegex(".*")
                .handshakeWithClientInfo(true)
                .maxRedirects(2)
                .clientBuilder(ClientBuilder.get()
                    .hostConnectionLimit(1)
                    .connectionTimeout(Duration.fromSeconds(1))
                    .requestTimeout(Duration.fromSeconds(10)));
    dlClient = (DistributedLogClientImpl) dlClientBuilder.build();
}
 
Example #15
Source File: DistributedLogServerTestCase.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
protected DLClient(String name,
                   String streamNameRegex,
                   Optional<String> serverSideRoutingFinagleName) {
    routingService = LocalRoutingService.newBuilder().build();
    dlClientBuilder = DistributedLogClientBuilder.newBuilder()
                .name(name)
                .clientId(ClientId$.MODULE$.apply(name))
                .routingService(routingService)
                .streamNameRegex(streamNameRegex)
                .handshakeWithClientInfo(true)
                .clientBuilder(ClientBuilder.get()
                    .hostConnectionLimit(1)
                    .connectionTimeout(Duration.fromSeconds(1))
                    .requestTimeout(Duration.fromSeconds(60)));
    if (serverSideRoutingFinagleName.isPresent()) {
        dlClientBuilder =
                dlClientBuilder.serverRoutingServiceFinagleNameStr(serverSideRoutingFinagleName.get());
    }
    dlClient = (DistributedLogClientImpl) dlClientBuilder.build();
}
 
Example #16
Source File: TestDistributedLogClientBuilder.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 60000)
public void testBuildClientsFromSameBuilder() throws Exception {
    DistributedLogClientBuilder builder = DistributedLogClientBuilder.newBuilder()
            .name("build-clients-from-same-builder")
            .clientId(ClientId$.MODULE$.apply("test-builder"))
            .finagleNameStr("inet!127.0.0.1:7001")
            .streamNameRegex(".*")
            .handshakeWithClientInfo(true)
            .clientBuilder(ClientBuilder.get()
                .hostConnectionLimit(1)
                .connectTimeout(Duration.fromSeconds(1))
                .tcpConnectTimeout(Duration.fromSeconds(1))
                .requestTimeout(Duration.fromSeconds(10)));
    DistributedLogClient client1 = builder.build();
    DistributedLogClient client2 = builder.build();
    assertFalse(client1 == client2);
}
 
Example #17
Source File: TestLeastLoadPlacementPolicy.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 10000)
public void testCalculateBalances() throws Exception {
    int numSevers = new Random().nextInt(20) + 1;
    int numStreams = new Random().nextInt(200) + 1;
    RoutingService mockRoutingService = mock(RoutingService.class);
    Namespace mockNamespace = mock(Namespace.class);
    LeastLoadPlacementPolicy leastLoadPlacementPolicy = new LeastLoadPlacementPolicy(
        new EqualLoadAppraiser(),
        mockRoutingService,
        mockNamespace,
        null,
        Duration.fromSeconds(600),
        new NullStatsLogger());
    TreeSet<ServerLoad> serverLoads =
        Await.result(leastLoadPlacementPolicy.calculate(generateServers(numSevers), generateStreams(numStreams)));
    long lowLoadPerServer = numStreams / numSevers;
    long highLoadPerServer = lowLoadPerServer + 1;
    for (ServerLoad serverLoad : serverLoads) {
        long load = serverLoad.getLoad();
        assertEquals(load, serverLoad.getStreamLoads().size());
        assertTrue(String.format("Load %d is not between %d and %d",
            load, lowLoadPerServer, highLoadPerServer), load == lowLoadPerServer || load == highLoadPerServer);
    }
}
 
Example #18
Source File: TestDistributedLogServerBase.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 60000)
public void testBulkWriteEmptyBuffer() throws Exception {
    String name = String.format("dlserver-bulk-write-%s", "empty");

    dlClient.routingService.addHost(name, dlServer.getAddress());

    List<ByteBuffer> writes = new ArrayList<ByteBuffer>();
    writes.add(ByteBuffer.wrap(("").getBytes()));
    writes.add(ByteBuffer.wrap(("").getBytes()));
    List<Future<DLSN>> futures = dlClient.dlClient.writeBulk(name, writes);
    assertEquals(2, futures.size());
    for (Future<DLSN> future : futures) {
        // No throw == pass
        DLSN dlsn = Await.result(future, Duration.fromSeconds(10));
    }
}
 
Example #19
Source File: LeastLoadPlacementPolicy.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
public LeastLoadPlacementPolicy(LoadAppraiser loadAppraiser, RoutingService routingService,
                                Namespace namespace, PlacementStateManager placementStateManager,
                                Duration refreshInterval, StatsLogger statsLogger) {
    super(loadAppraiser, routingService, namespace, placementStateManager, refreshInterval, statsLogger);
    statsLogger.registerGauge("placement/load.diff", new Gauge<Number>() {
        @Override
        public Number getDefaultValue() {
            return 0;
        }

        @Override
        public Number getSample() {
            if (serverLoads.size() > 0) {
                return serverLoads.last().getLoad() - serverLoads.first().getLoad();
            } else {
                return getDefaultValue();
            }
        }
    });
}
 
Example #20
Source File: PinLaterQueryIssuer.java    From pinlater with Apache License 2.0 5 votes vote down vote up
private void issueEnqueueRequests(PinLater.ServiceIface iface) throws InterruptedException {
  Preconditions.checkNotNull(queueName, "Queue was not specified.");
  final AtomicLong queriesIssued = new AtomicLong(0);
  final Semaphore permits = new Semaphore(concurrency);
  while (numQueries == -1 || queriesIssued.get() < numQueries) {
    final PinLaterEnqueueRequest request = new PinLaterEnqueueRequest();
    request.setQueueName(queueName);
    for (int i = 0; i < batchSize; i++) {
      PinLaterJob job = new PinLaterJob(ByteBuffer.wrap(
          new String("task_" + random.nextInt(Integer.MAX_VALUE)).getBytes()));
      job.setPriority(priority);
      request.addToJobs(job);
    }
    final long startTimeNanos = System.nanoTime();
    queriesIssued.incrementAndGet();
    permits.acquire();
    iface.enqueueJobs(REQUEST_CONTEXT, request).respond(
        new Function<Try<PinLaterEnqueueResponse>, BoxedUnit>() {
          @Override
          public BoxedUnit apply(Try<PinLaterEnqueueResponse> responseTry) {
            permits.release();
            statsLogger.requestComplete(
                Duration.fromNanoseconds(System.nanoTime() - startTimeNanos));
            if (responseTry.isThrow()) {
              LOG.info("Exception for request: " + request + " : " + ((Throw) responseTry).e());
            }
            return BoxedUnit.UNIT;
          }
        });
  }
  permits.acquire(concurrency);
  LOG.info("Enqueue queries issued: " + queriesIssued);
}
 
Example #21
Source File: DistributedLogClientImpl.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
void send(SocketAddress address) {
    long elapsedMs = stopwatch.elapsed(TimeUnit.MILLISECONDS);
    if (clientConfig.getMaxRedirects() > 0 &&
            tries.get() >= clientConfig.getMaxRedirects()) {
        fail(address, new RequestTimeoutException(Duration.fromMilliseconds(elapsedMs),
                "Exhausted max redirects in " + elapsedMs + " ms"));
        return;
    } else if (clientConfig.getRequestTimeoutMs() > 0 &&
            elapsedMs >= clientConfig.getRequestTimeoutMs()) {
        fail(address, new RequestTimeoutException(Duration.fromMilliseconds(elapsedMs),
                "Exhausted max request timeout " + clientConfig.getRequestTimeoutMs()
                        + " in " + elapsedMs + " ms"));
        return;
    }
    synchronized (this) {
        String addrStr = address.toString();
        if (ctx.isSetTriedHosts() && ctx.getTriedHosts().contains(addrStr)) {
            nextAddressToSend = address;
            dlTimer.newTimeout(this,
                    Math.min(clientConfig.getRedirectBackoffMaxMs(),
                            tries.get() * clientConfig.getRedirectBackoffStartMs()),
                    TimeUnit.MILLISECONDS);
        } else {
            doSend(address);
        }
    }
}
 
Example #22
Source File: TerrapinServiceImplTest.java    From terrapin with Apache License 2.0 5 votes vote down vote up
@Test
public void testMultiGetClusterNotFound() {
  TerrapinMultiGetRequest request = prepareMultiGetRequest().setClusterList(ImmutableList.of(
      "random-cluster"));

  Try<TerrapinResponse> responseTry =
  serviceIface.multiGet(request).get(Duration.forever());
  assertTrue(responseTry.isThrow());
  assertEquals(TerrapinGetErrorCode.CLUSTER_NOT_FOUND,
               ((TerrapinGetException)((Throw)responseTry).e()).getErrorCode());
}
 
Example #23
Source File: ReaderWithOffsets.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
private static void readLoop(final DistributedLogManager dlm,
                             final DLSN dlsn,
                             final AtomicReference<DLSN> lastDLSN)
        throws Exception {

    final CountDownLatch keepAliveLatch = new CountDownLatch(1);

    System.out.println("Wait for records starting from " + dlsn);
    final AsyncLogReader reader = FutureUtils.result(dlm.openAsyncLogReader(dlsn));
    final FutureEventListener<LogRecordWithDLSN> readListener = new FutureEventListener<LogRecordWithDLSN>() {
        @Override
        public void onFailure(Throwable cause) {
            System.err.println("Encountered error on reading records from stream " + dlm.getStreamName());
            cause.printStackTrace(System.err);
            keepAliveLatch.countDown();
        }

        @Override
        public void onSuccess(LogRecordWithDLSN record) {
            System.out.println("Received record " + record.getDlsn());
            System.out.println("\"\"\"");
            System.out.println(new String(record.getPayload(), UTF_8));
            System.out.println("\"\"\"");
            lastDLSN.set(record.getDlsn());
            reader.readNext().addEventListener(this);
        }
    };
    reader.readNext().addEventListener(readListener);

    keepAliveLatch.await();
    FutureUtils.result(reader.asyncClose(), Duration.apply(5, TimeUnit.SECONDS));
}
 
Example #24
Source File: ServiceShutdownHook.java    From pinlater with Apache License 2.0 5 votes vote down vote up
/**
 * Turn on graceful shutdown on the finagle server passed in with the grace period passed in.
 *
 * @param server the finagle server instance built by finagle ServerBuilder.
 * @param gracePeriod the time period the shutdown process will wait for till the existing
 *                    requests drain. If the existing requests are not being drain after grace
 *                    period expires, the server will be forcefully shutdown.
 */
public static void register(final Server server, final Duration gracePeriod) {
  Runtime.getRuntime().addShutdownHook(new Thread() {
    @Override
    public void run() {
      LOG.info("Try to shut down the server gracefully: {}", gracePeriod);
      server.close(gracePeriod);
      LOG.info("Finished server graceful shutdown");
    }
  });
}
 
Example #25
Source File: MovingAverageRateFactory.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
public MovingAverageRateFactory(Timer timer) {
    this.avgs = new CopyOnWriteArrayList<SampledMovingAverageRate>();
    this.timer = timer;
    Function0<BoxedUnit> sampleTask = new Function0<BoxedUnit>() {
        public BoxedUnit apply() {
            sampleAll();
            return null;
        }
    };
    this.timerTask = timer.schedulePeriodically(
        Time.now(), Duration.fromSeconds(DEFAULT_INTERVAL_SECS), sampleTask);
}
 
Example #26
Source File: ZKSessionLock.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Override
public void unlock() {
    Future<BoxedUnit> unlockResult = asyncUnlock();
    try {
        Await.result(unlockResult, Duration.fromMilliseconds(lockOpTimeout));
    } catch (TimeoutException toe) {
        // This shouldn't happen unless we lose a watch, and may result in a leaked lock.
        LOG.error("Timeout unlocking {} owned by {} : ", new Object[] { lockPath, lockId, toe });
    } catch (Exception e) {
        LOG.warn("{} failed to unlock {} : ", new Object[] { lockId, lockPath, e });
    }
}
 
Example #27
Source File: TestBKLogReadHandler.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testLockStreamSameSubscriber() throws Exception {
    String streamName = runtime.getMethodName();
    BKDistributedLogManager bkdlm = createNewDLM(conf, streamName);
    DLMTestUtil.generateLogSegmentNonPartitioned(bkdlm, 0, 5, 1);
    BKLogReadHandler readHandler = bkdlm.createReadHandler();
    Await.result(readHandler.lockStream());

    // same subscrbiers couldn't lock stream in parallel
    BKDistributedLogManager bkdlm10 = createNewDLM(conf, streamName);
    BKLogReadHandler s10Handler =
            bkdlm10.createReadHandler(Optional.of("s1"));
    Await.result(s10Handler.lockStream());

    BKDistributedLogManager bkdlm11 = createNewDLM(conf, streamName);
    BKLogReadHandler s11Handler =
            bkdlm11.createReadHandler(Optional.of("s1"));
    try {
        Await.result(s11Handler.lockStream(), Duration.apply(10000, TimeUnit.MILLISECONDS));
        fail("Should fail lock stream using same subscriber id");
    } catch (OwnershipAcquireFailedException oafe) {
        // expected
    } catch (TimeoutException te) {
        // expected.
    }

    readHandler.asyncClose();
    bkdlm.close();
    s10Handler.asyncClose();
    bkdlm10.close();
    s11Handler.asyncClose();
    bkdlm11.close();
}
 
Example #28
Source File: TerrapinServiceImplTest.java    From terrapin with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetError() {
  // Test the case where We get back an error set through an error code set in
  // TerrapinSingleResponse.
  ByteBuffer key = ByteBuffer.wrap(KEY);
  TerrapinGetRequest request = prepareGetRequest();

  TerrapinResponse response = new TerrapinResponse().setResponseMap(ImmutableMap.of(
      key, new TerrapinSingleResponse().setErrorCode(TerrapinGetErrorCode.OTHER)));
  when(mockClient1.getMany(eq(FILESET), eq(Sets.newHashSet(key)))).thenReturn(
      Future.value(response));

  Try<TerrapinSingleResponse> singleResponseTry = serviceIface.get(request).get(
      Duration.forever());
  assertTrue(singleResponseTry.isThrow());
  assertEquals(TerrapinGetErrorCode.OTHER,
      ((TerrapinGetException)((Throw)singleResponseTry).e()).getErrorCode());

  // Test the case where the call to the client library itself bails out due to a
  // legit error.
  when(mockClient1.getMany(eq(FILESET), eq(Sets.newHashSet(key)))).thenReturn(
          Future.<TerrapinResponse>exception(new TerrapinGetException("Failed.",
                  TerrapinGetErrorCode.FILE_SET_NOT_FOUND)));
  singleResponseTry = serviceIface.get(request).get(Duration.forever());
  assertTrue(singleResponseTry.isThrow());
  assertEquals(TerrapinGetErrorCode.FILE_SET_NOT_FOUND,
      ((TerrapinGetException)((Throw)singleResponseTry).e()).getErrorCode());

  // Test the case where the call to the client library bails out due to a runtime
  // exception.
  when(mockClient1.getMany(eq(FILESET), eq(Sets.newHashSet(key)))).thenThrow(
      new RuntimeException(new NullPointerException()));
  singleResponseTry = serviceIface.get(request).get(Duration.forever());
  assertTrue(singleResponseTry.isThrow());
  assertEquals(TerrapinGetErrorCode.OTHER,
      ((TerrapinGetException)((Throw)singleResponseTry).e()).getErrorCode());
}
 
Example #29
Source File: TestDistributedLogServer.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
void validateFailedAsLogRecordTooLong(Future<DLSN> future) {
    try {
        DLSN dlsn = Await.result(future, Duration.fromSeconds(10));
        fail("should have failed");
    } catch (DLException dle) {
        assertEquals(StatusCode.TOO_LARGE_RECORD, dle.getCode());
    } catch (Exception ex) {
        failDueToWrongException(ex);
    }
}
 
Example #30
Source File: ProxyTool.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Override
protected int runCmd(CommandLine commandLine) throws Exception {
    try {
        parseCommandLine(commandLine);
    } catch (ParseException pe) {
        System.err.println("ERROR: failed to parse commandline : '" + pe.getMessage() + "'");
        printUsage();
        return -1;
    }

    DistributedLogClientBuilder clientBuilder = DistributedLogClientBuilder.newBuilder()
            .name("proxy_tool")
            .clientId(ClientId$.MODULE$.apply("proxy_tool"))
            .maxRedirects(2)
            .host(address)
            .clientBuilder(ClientBuilder.get()
                    .connectionTimeout(Duration.fromSeconds(2))
                    .tcpConnectTimeout(Duration.fromSeconds(2))
                    .requestTimeout(Duration.fromSeconds(10))
                    .hostConnectionLimit(1)
                    .hostConnectionCoresize(1)
                    .keepAlive(true)
                    .failFast(false));
    Pair<DistributedLogClient, MonitorServiceClient> clientPair =
            ClientUtils.buildClient(clientBuilder);
    try {
        return runCmd(clientPair);
    } finally {
        clientPair.getLeft().close();
    }
}