org.apache.reef.io.network.naming.NameResolver Java Examples

@Inject
private GrpcMessageEnvironment(
  final LocalAddressProvider localAddressProvider,
  final NameResolver nameResolver,
  final IdentifierFactory idFactory,
  @Parameter(MessageParameters.SenderId.class) final String localSenderId) {
  this.nameResolver = nameResolver;
  this.idFactory = idFactory;
  this.grpcServer = new GrpcMessageServer(localAddressProvider, nameResolver, idFactory, localSenderId);
  this.localSenderId = localSenderId;

  try {
    this.grpcServer.start();
  } catch (final Exception e) {
    LOG.warn("Failed to start grpc server", e);
    throw new RuntimeException(e);
  }
}
 

@Inject
private GrpcMessageEnvironment(
    final LocalAddressProvider localAddressProvider,
    final NameResolver nameResolver,
    final IdentifierFactory idFactory,
    @Parameter(MessageParameters.SenderId.class) final String localSenderId) {
  this.nameResolver = nameResolver;
  this.idFactory = idFactory;
  this.grpcServer = new GrpcMessageServer(localAddressProvider, nameResolver, idFactory, localSenderId);

  try {
    this.grpcServer.start();
  } catch (final Exception e) {
    LOG.warn("Failed to start grpc server", e);
    throw new RuntimeException(e);
  }
}
 

/**
 * Constructor.
 *
 * @param nameResolver name resolver.
 * @param idFactory    identifier factory.
 * @param receiverId   id of the receiver.
 */
GrpcMessageClient(final NameResolver nameResolver,
                  final IdentifierFactory idFactory,
                  final String receiverId) {
  this.nameResolver = nameResolver;
  this.idFactory = idFactory;
  this.receiverId = receiverId;
}
 

/**
 * Constructor.
 *
 * @param localAddressProvider local address provider.
 * @param nameResolver         name resolver.
 * @param idFactory            identifier factory.
 * @param localSenderId        id of the local sender.
 */
GrpcMessageServer(final LocalAddressProvider localAddressProvider,
                  final NameResolver nameResolver,
                  final IdentifierFactory idFactory,
                  final String localSenderId) {
  this.localAddressProvider = localAddressProvider;
  this.nameResolver = nameResolver;
  this.idFactory = idFactory;
  this.localSenderId = localSenderId;
  this.listenerMap = new ConcurrentHashMap<>();
}
 

/**
 * Constructor.
 * @param nameResolver name resolver.
 * @param idFactory identifier factory.
 * @param receiverId id of the receiver.
 */
GrpcMessageClient(final NameResolver nameResolver,
                  final IdentifierFactory idFactory,
                  final String receiverId) {
  this.nameResolver = nameResolver;
  this.idFactory = idFactory;
  this.receiverId = receiverId;
}
 

/**
 * Constructor.
 * @param localAddressProvider local address provider.
 * @param nameResolver name resolver.
 * @param idFactory identifier factory.
 * @param localSenderId id of the local sender.
 */
GrpcMessageServer(final LocalAddressProvider localAddressProvider,
                  final NameResolver nameResolver,
                  final IdentifierFactory idFactory,
                  final String localSenderId) {
  this.localAddressProvider = localAddressProvider;
  this.nameResolver = nameResolver;
  this.idFactory = idFactory;
  this.localSenderId = localSenderId;
  this.listenerMap = new ConcurrentHashMap<>();
}
 

@Inject
private NetworkConnectionServiceImpl(
    @Parameter(NetworkConnectionServiceIdFactory.class) final IdentifierFactory idFactory,
    @Parameter(NetworkConnectionServicePort.class) final int nsPort,
    final TransportFactory transportFactory,
    final NameResolver nameResolver) {
  this.idFactory = idFactory;
  this.connFactoryMap = new ConcurrentHashMap<>();
  this.nsCodec = new NetworkConnectionServiceMessageCodec(idFactory, connFactoryMap);
  this.nsLinkListener = new NetworkConnectionServiceLinkListener(connFactoryMap);
  final EventHandler<TransportEvent> recvHandler =
      new NetworkConnectionServiceReceiveHandler(connFactoryMap, nsCodec);
  this.nameResolver = nameResolver;
  this.transport = transportFactory.newInstance(nsPort, recvHandler, recvHandler,
      new NetworkConnectionServiceExceptionHandler());

  this.nameServiceRegisteringStage = new SingleThreadStage<>(
      "NameServiceRegisterer", new EventHandler<Tuple<Identifier, InetSocketAddress>>() {
        @Override
        public void onNext(final Tuple<Identifier, InetSocketAddress> tuple) {
          try {
            nameResolver.register(tuple.getKey(), tuple.getValue());
            LOG.log(Level.FINEST, "Registered {0} with nameservice", tuple.getKey());
          } catch (final Exception ex) {
            final String msg = "Unable to register " + tuple.getKey() + " with name service";
            LOG.log(Level.WARNING, msg, ex);
            throw new RuntimeException(msg, ex);
          }
        }
      }, 5);

  this.nameServiceUnregisteringStage = new SingleThreadStage<>(
      "NameServiceRegisterer", new EventHandler<Identifier>() {
        @Override
        public void onNext(final Identifier id) {
          try {
            nameResolver.unregister(id);
            LOG.log(Level.FINEST, "Unregistered {0} with nameservice", id);
          } catch (final Exception ex) {
            final String msg = "Unable to unregister " + id + " with name service";
            LOG.log(Level.WARNING, msg, ex);
            throw new RuntimeException(msg, ex);
          }
        }
      }, 5);

  this.isClosed = new AtomicBoolean();
}
 

@Inject
private NetworkService(
    @Parameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class) final IdentifierFactory factory,
    @Parameter(NetworkServiceParameters.NetworkServicePort.class) final int nsPort,
    final NameResolver nameResolver,
    @Parameter(NetworkServiceParameters.NetworkServiceCodec.class) final Codec<T> codec,
    @Parameter(NetworkServiceParameters.NetworkServiceTransportFactory.class) final TransportFactory tpFactory,
    @Parameter(NetworkServiceParameters.NetworkServiceHandler.class) final EventHandler<Message<T>> recvHandler,
    @Parameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class)
    final EventHandler<Exception> exHandler) {
  this.factory = factory;
  this.codec = codec;
  this.transport = tpFactory.newInstance(nsPort,
      new LoggingEventHandler<TransportEvent>(),
      new MessageHandler<T>(recvHandler, codec, factory), exHandler);

  this.nameResolver = nameResolver;

  this.nameServiceRegisteringStage = new SingleThreadStage<>(
      "NameServiceRegisterer", new EventHandler<Tuple<Identifier, InetSocketAddress>>() {
        @Override
        public void onNext(final Tuple<Identifier, InetSocketAddress> tuple) {
          try {
            nameResolver.register(tuple.getKey(), tuple.getValue());
            LOG.log(Level.FINEST, "Registered {0} with nameservice", tuple.getKey());
          } catch (final Exception ex) {
            final String msg = "Unable to register " + tuple.getKey() + "with name service";
            LOG.log(Level.WARNING, msg, ex);
            throw new RuntimeException(msg, ex);
          }
        }
      }, 5);

  this.nameServiceUnregisteringStage = new SingleThreadStage<>(
      "NameServiceRegisterer", new EventHandler<Identifier>() {
        @Override
        public void onNext(final Identifier id) {
          try {
            nameResolver.unregister(id);
            LOG.log(Level.FINEST, "Unregistered {0} with nameservice", id);
          } catch (final Exception ex) {
            final String msg = "Unable to unregister " + id + " with name service";
            LOG.log(Level.WARNING, msg, ex);
            throw new RuntimeException(msg, ex);
          }
        }
      }, 5);
}
 

/**
 * NetworkService messaging test.
 */
@Test
public void testMessagingNetworkService() throws Exception {
  LOG.log(Level.FINEST, name.getMethodName());

  final IdentifierFactory factory = new StringIdentifierFactory();

  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);

  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int nameServerPort = server.getPort();

    final int numMessages = 10;
    final Monitor monitor = new Monitor();

    // network service
    final String name2 = "task2";
    final String name1 = "task1";
    final Configuration nameResolverConf =
        Tang.Factory.getTang().newConfigurationBuilder(NameResolverConfiguration.CONF
        .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, this.localAddress)
        .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServerPort)
        .build())
        .build();

    final Injector injector2 = Tang.Factory.getTang().newInjector(nameResolverConf);

    LOG.log(Level.FINEST, "=== Test network service receiver start");
    LOG.log(Level.FINEST, "=== Test network service sender start");
    try (NameResolver nameResolver = injector2.getInstance(NameResolver.class)) {
      injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class, factory);
      injector2.bindVolatileInstance(NameResolver.class, nameResolver);
      injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class, new StringCodec());
      injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceTransportFactory.class,
          injector.getInstance(MessagingTransportFactory.class));
      injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class,
          new ExceptionHandler());

      final Injector injectorNs2 = injector2.forkInjector();
      injectorNs2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
          new MessageHandler<String>(name2, monitor, numMessages));
      final NetworkService<String> ns2 = injectorNs2.getInstance(NetworkService.class);

      final Injector injectorNs1 = injector2.forkInjector();
      injectorNs1.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
          new MessageHandler<String>(name1, null, 0));
      final NetworkService<String> ns1 = injectorNs1.getInstance(NetworkService.class);

      ns2.registerId(factory.getNewInstance(name2));
      final int port2 = ns2.getTransport().getListeningPort();
      server.register(factory.getNewInstance("task2"), new InetSocketAddress(this.localAddress, port2));

      ns1.registerId(factory.getNewInstance(name1));
      final int port1 = ns1.getTransport().getListeningPort();
      server.register(factory.getNewInstance("task1"), new InetSocketAddress(this.localAddress, port1));

      final Identifier destId = factory.getNewInstance(name2);

      try (Connection<String> conn = ns1.newConnection(destId)) {
        conn.open();
        for (int count = 0; count < numMessages; ++count) {
          conn.write("hello! " + count);
        }
        monitor.mwait();

      } catch (final NetworkException e) {
        e.printStackTrace();
        throw new RuntimeException(e);
      }
    }
  }
}
 

/**
 * NetworkService messaging rate benchmark.
 */
@Test
public void testMessagingNetworkServiceRate() throws Exception {

  Assume.assumeFalse("Use log level INFO to run benchmarking", LOG.isLoggable(Level.FINEST));

  LOG.log(Level.FINEST, name.getMethodName());

  final IdentifierFactory factory = new StringIdentifierFactory();

  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);

  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int nameServerPort = server.getPort();

    final int[] messageSizes = {1, 16, 32, 64, 512, 64 * 1024, 1024 * 1024};

    for (final int size : messageSizes) {
      final int numMessages = 300000 / (Math.max(1, size / 512));
      final Monitor monitor = new Monitor();

      // network service
      final String name2 = "task2";
      final String name1 = "task1";
      final Configuration nameResolverConf =
          Tang.Factory.getTang().newConfigurationBuilder(NameResolverConfiguration.CONF
          .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, this.localAddress)
          .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServerPort)
          .build())
          .build();

      final Injector injector2 = Tang.Factory.getTang().newInjector(nameResolverConf);

      LOG.log(Level.FINEST, "=== Test network service receiver start");
      LOG.log(Level.FINEST, "=== Test network service sender start");
      try (NameResolver nameResolver = injector2.getInstance(NameResolver.class)) {
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class, factory);
        injector2.bindVolatileInstance(NameResolver.class, nameResolver);
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class, new StringCodec());
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceTransportFactory.class,
            injector.getInstance(MessagingTransportFactory.class));
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class,
            new ExceptionHandler());

        final Injector injectorNs2 = injector2.forkInjector();
        injectorNs2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name2, monitor, numMessages));
        final NetworkService<String> ns2 = injectorNs2.getInstance(NetworkService.class);

        final Injector injectorNs1 = injector2.forkInjector();
        injectorNs1.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name1, null, 0));
        final NetworkService<String> ns1 = injectorNs1.getInstance(NetworkService.class);

        ns2.registerId(factory.getNewInstance(name2));
        final int port2 = ns2.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task2"), new InetSocketAddress(this.localAddress, port2));

        ns1.registerId(factory.getNewInstance(name1));
        final int port1 = ns1.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task1"), new InetSocketAddress(this.localAddress, port1));

        final Identifier destId = factory.getNewInstance(name2);
        final String message = StringUtils.repeat('1', size);

        final long start = System.currentTimeMillis();
        try (Connection<String> conn = ns1.newConnection(destId)) {
          conn.open();
          for (int i = 0; i < numMessages; i++) {
            conn.write(message);
          }
          monitor.mwait();
        } catch (final NetworkException e) {
          e.printStackTrace();
          throw new RuntimeException(e);
        }
        final long end = System.currentTimeMillis();
        final double runtime = ((double) end - start) / 1000;
        LOG.log(Level.FINEST, "size: " + size + "; messages/s: " + numMessages / runtime +
            " bandwidth(bytes/s): " + ((double) numMessages * 2 * size) / runtime); // x2 for unicode chars
      }
    }
  }
}
 

@Test
public void testMultithreadedSharedConnMessagingNetworkServiceRate() throws Exception {

  Assume.assumeFalse("Use log level INFO to run benchmarking", LOG.isLoggable(Level.FINEST));

  LOG.log(Level.FINEST, name.getMethodName());

  final IdentifierFactory factory = new StringIdentifierFactory();

  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);
  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int nameServerPort = server.getPort();

    final int[] messageSizes = {2000}; // {1,16,32,64,512,64*1024,1024*1024};

    for (final int size : messageSizes) {
      final int numMessages = 300000 / (Math.max(1, size / 512));
      final int numThreads = 2;
      final int totalNumMessages = numMessages * numThreads;
      final Monitor monitor = new Monitor();


      // network service
      final String name2 = "task2";
      final String name1 = "task1";
      final Configuration nameResolverConf =
          Tang.Factory.getTang().newConfigurationBuilder(NameResolverConfiguration.CONF
          .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, this.localAddress)
          .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServerPort)
          .build())
          .build();

      final Injector injector2 = Tang.Factory.getTang().newInjector(nameResolverConf);

      LOG.log(Level.FINEST, "=== Test network service receiver start");
      LOG.log(Level.FINEST, "=== Test network service sender start");
      try (NameResolver nameResolver = injector2.getInstance(NameResolver.class)) {
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class, factory);
        injector2.bindVolatileInstance(NameResolver.class, nameResolver);
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class, new StringCodec());
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceTransportFactory.class,
            injector.getInstance(MessagingTransportFactory.class));
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class,
            new ExceptionHandler());

        final Injector injectorNs2 = injector2.forkInjector();
        injectorNs2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name2, monitor, totalNumMessages));
        final NetworkService<String> ns2 = injectorNs2.getInstance(NetworkService.class);

        final Injector injectorNs1 = injector2.forkInjector();
        injectorNs1.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name1, null, 0));
        final NetworkService<String> ns1 = injectorNs1.getInstance(NetworkService.class);

        ns2.registerId(factory.getNewInstance(name2));
        final int port2 = ns2.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task2"), new InetSocketAddress(this.localAddress, port2));

        ns1.registerId(factory.getNewInstance(name1));
        final int port1 = ns1.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task1"), new InetSocketAddress(this.localAddress, port1));

        final Identifier destId = factory.getNewInstance(name2);

        try (Connection<String> conn = ns1.newConnection(destId)) {
          conn.open();

          final String message = StringUtils.repeat('1', size);
          final ExecutorService e = Executors.newCachedThreadPool();

          final long start = System.currentTimeMillis();
          for (int i = 0; i < numThreads; i++) {
            e.submit(new Runnable() {

              @Override
              public void run() {
                for (int i = 0; i < numMessages; i++) {
                  conn.write(message);
                }
              }
            });
          }


          e.shutdown();
          e.awaitTermination(30, TimeUnit.SECONDS);
          monitor.mwait();

          final long end = System.currentTimeMillis();
          final double runtime = ((double) end - start) / 1000;

          LOG.log(Level.FINEST, "size: " + size + "; messages/s: " + totalNumMessages / runtime + 
              " bandwidth(bytes/s): " + ((double) totalNumMessages * 2 * size) / runtime); // x2 for unicode chars
        }
      }
    }
  }
}
 

/**
 * NetworkService messaging rate benchmark.
 */
@Test
public void testMessagingNetworkServiceBatchingRate() throws Exception {

  Assume.assumeFalse("Use log level INFO to run benchmarking", LOG.isLoggable(Level.FINEST));

  LOG.log(Level.FINEST, name.getMethodName());

  final IdentifierFactory factory = new StringIdentifierFactory();

  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);

  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int nameServerPort = server.getPort();

    final int batchSize = 1024 * 1024;
    final int[] messageSizes = {32, 64, 512};

    for (final int size : messageSizes) {
      final int numMessages = 300 / (Math.max(1, size / 512));
      final Monitor monitor = new Monitor();

      // network service
      final String name2 = "task2";
      final String name1 = "task1";
      final Configuration nameResolverConf =
          Tang.Factory.getTang().newConfigurationBuilder(NameResolverConfiguration.CONF
          .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, this.localAddress)
          .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServerPort)
          .build())
          .build();

      final Injector injector2 = Tang.Factory.getTang().newInjector(nameResolverConf);

      LOG.log(Level.FINEST, "=== Test network service receiver start");
      LOG.log(Level.FINEST, "=== Test network service sender start");
      try (NameResolver nameResolver = injector2.getInstance(NameResolver.class)) {
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class, factory);
        injector2.bindVolatileInstance(NameResolver.class, nameResolver);
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class, new StringCodec());
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceTransportFactory.class,
            injector.getInstance(MessagingTransportFactory.class));
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class,
            new ExceptionHandler());

        final Injector injectorNs2 = injector2.forkInjector();
        injectorNs2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name2, monitor, numMessages));
        final NetworkService<String> ns2 = injectorNs2.getInstance(NetworkService.class);

        final Injector injectorNs1 = injector2.forkInjector();
        injectorNs1.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name1, null, 0));
        final NetworkService<String> ns1 = injectorNs1.getInstance(NetworkService.class);

        ns2.registerId(factory.getNewInstance(name2));
        final int port2 = ns2.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task2"), new InetSocketAddress(this.localAddress, port2));

        ns1.registerId(factory.getNewInstance(name1));
        final int port1 = ns1.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task1"), new InetSocketAddress(this.localAddress, port1));

        final Identifier destId = factory.getNewInstance(name2);
        final String message = StringUtils.repeat('1', batchSize);

        final long start = System.currentTimeMillis();
        try (Connection<String> conn = ns1.newConnection(destId)) {
          conn.open();
          for (int i = 0; i < numMessages; i++) {
            conn.write(message);
          }
          monitor.mwait();
        } catch (final NetworkException e) {
          e.printStackTrace();
          throw new RuntimeException(e);
        }
        final long end = System.currentTimeMillis();
        final double runtime = ((double) end - start) / 1000;
        final long numAppMessages = numMessages * batchSize / size;
        LOG.log(Level.FINEST, "size: " + size + "; messages/s: " + numAppMessages / runtime +
            " bandwidth(bytes/s): " + ((double) numAppMessages * 2 * size) / runtime); // x2 for unicode chars
      }
    }
  }
}