org.apache.reef.io.network.naming.NameServer Java Examples

The following examples show how to use org.apache.reef.io.network.naming.NameServer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DataTransferTest.java    From incubator-nemo with Apache License 2.0 6 votes vote down vote up
private Injector createNameClientInjector() {
  try {
    final Configuration configuration = TANG.newConfigurationBuilder()
      .bindImplementation(IdentifierFactory.class, StringIdentifierFactory.class)
      .build();
    final Injector injector = TANG.newInjector(configuration);
    final LocalAddressProvider localAddressProvider = injector.getInstance(LocalAddressProvider.class);
    final NameServer nameServer = injector.getInstance(NameServer.class);
    final Configuration nameClientConfiguration = NameResolverConfiguration.CONF
      .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, localAddressProvider.getLocalAddress())
      .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServer.getPort())
      .build();
    return injector.forkInjector(nameClientConfiguration);
  } catch (final InjectionException e) {
    throw new RuntimeException(e);
  }
}
 
Example #2
Source File: NemoDriver.java    From nemo with Apache License 2.0 6 votes vote down vote up
@Inject
private NemoDriver(final UserApplicationRunner userApplicationRunner,
                   final RuntimeMaster runtimeMaster,
                   final NameServer nameServer,
                   final LocalAddressProvider localAddressProvider,
                   final JobMessageObserver client,
                   @Parameter(JobConf.ExecutorJsonContents.class) final String resourceSpecificationString,
                   @Parameter(JobConf.JobId.class) final String jobId,
                   @Parameter(JobConf.FileDirectory.class) final String localDirectory,
                   @Parameter(JobConf.GlusterVolumeDirectory.class) final String glusterDirectory) {
  IdManager.setInDriver();
  this.userApplicationRunner = userApplicationRunner;
  this.runtimeMaster = runtimeMaster;
  this.nameServer = nameServer;
  this.localAddressProvider = localAddressProvider;
  this.resourceSpecificationString = resourceSpecificationString;
  this.jobId = jobId;
  this.localDirectory = localDirectory;
  this.glusterDirectory = glusterDirectory;
  this.client = client;
  this.handler = new RemoteClientMessageLoggingHandler(client);
}
 
Example #3
Source File: DataTransferTest.java    From nemo with Apache License 2.0 6 votes vote down vote up
private Injector createNameClientInjector() {
  try {
    final Configuration configuration = TANG.newConfigurationBuilder()
        .bindImplementation(IdentifierFactory.class, StringIdentifierFactory.class)
        .build();
    final Injector injector = TANG.newInjector(configuration);
    final LocalAddressProvider localAddressProvider = injector.getInstance(LocalAddressProvider.class);
    final NameServer nameServer = injector.getInstance(NameServer.class);
    final Configuration nameClientConfiguration = NameResolverConfiguration.CONF
        .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, localAddressProvider.getLocalAddress())
        .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServer.getPort())
        .build();
    return injector.forkInjector(nameClientConfiguration);
  } catch (final InjectionException e) {
    throw new RuntimeException(e);
  }
}
 
Example #4
Source File: NetworkMessagingTestService.java    From reef with Apache License 2.0 6 votes vote down vote up
public NetworkMessagingTestService(final String localAddress) throws InjectionException {
  // name server
  final Injector injector = Tang.Factory.getTang().newInjector();
  this.nameServer = injector.getInstance(NameServer.class);
  final Configuration netConf = NameResolverConfiguration.CONF
      .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, localAddress)
      .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServer.getPort())
      .build();

  LOG.log(Level.FINEST, "=== Test network connection service receiver start");
  // network service for receiver
  final Injector injectorReceiver = injector.forkInjector(netConf);
  this.receiverNetworkConnService = injectorReceiver.getInstance(NetworkConnectionService.class);
  this.factory = injectorReceiver.getNamedInstance(NetworkConnectionServiceIdFactory.class);

  // network service for sender
  LOG.log(Level.FINEST, "=== Test network connection service sender start");
  final Injector injectorSender = injector.forkInjector(netConf);
  senderNetworkConnService = injectorSender.getInstance(NetworkConnectionService.class);
}
 
Example #5
Source File: NemoDriver.java    From incubator-nemo with Apache License 2.0 5 votes vote down vote up
@Inject
private NemoDriver(final UserApplicationRunner userApplicationRunner,
                   final RuntimeMaster runtimeMaster,
                   final NameServer nameServer,
                   final LocalAddressProvider localAddressProvider,
                   final JobMessageObserver client,
                   final ClientRPC clientRPC,
                   final DataPlaneConf dataPlaneConf,
                   @Parameter(JobConf.ExecutorJSONContents.class) final String resourceSpecificationString,
                   @Parameter(JobConf.BandwidthJSONContents.class) final String bandwidthString,
                   @Parameter(JobConf.JobId.class) final String jobId,
                   @Parameter(JobConf.FileDirectory.class) final String localDirectory,
                   @Parameter(JobConf.GlusterVolumeDirectory.class) final String glusterDirectory) {
  IdManager.setInDriver();
  this.userApplicationRunner = userApplicationRunner;
  this.runtimeMaster = runtimeMaster;
  this.nameServer = nameServer;
  this.localAddressProvider = localAddressProvider;
  this.resourceSpecificationString = resourceSpecificationString;
  this.jobId = jobId;
  this.localDirectory = localDirectory;
  this.glusterDirectory = glusterDirectory;
  this.handler = new RemoteClientMessageLoggingHandler(client);
  this.clientRPC = clientRPC;
  this.dataPlaneConf = dataPlaneConf;
  // TODO #69: Support job-wide execution property
  ResourceSitePass.setBandwidthSpecificationString(bandwidthString);
  clientRPC.registerHandler(ControlMessage.ClientToDriverMessageType.Notification, this::handleNotification);
  clientRPC.registerHandler(ControlMessage.ClientToDriverMessageType.LaunchDAG, message -> {
    startSchedulingUserDAG(message.getLaunchDAG().getDag());
    final Map<Serializable, Object> broadcastVars =
      SerializationUtils.deserialize(message.getLaunchDAG().getBroadcastVars().toByteArray());
    BroadcastManagerMaster.registerBroadcastVariablesFromClient(broadcastVars);
  });
  clientRPC.registerHandler(ControlMessage.ClientToDriverMessageType.DriverShutdown, message -> shutdown());
  // Send DriverStarted message to the client
  clientRPC.send(ControlMessage.DriverToClientMessage.newBuilder()
    .setType(ControlMessage.DriverToClientMessageType.DriverStarted).build());
}
 
Example #6
Source File: JobDriver.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Job driver constructor.
 * All parameters are injected from TANG automatically.
 *
 * @param clock                      Wake clock to schedule and check up running jobs.
 * @param jobMessageObserver         is used to send messages back to the client.
 * @param evaluatorRequestor         is used to request Evaluators.
 * @param activeContextBridgeFactory
 */
@Inject
JobDriver(final Clock clock,
          final HttpServer httpServer,
          final NameServer nameServer,
          final JobMessageObserver jobMessageObserver,
          final EvaluatorRequestor evaluatorRequestor,
          final DriverStatusManager driverStatusManager,
          final LoggingScopeFactory loggingScopeFactory,
          final LocalAddressProvider localAddressProvider,
          final ActiveContextBridgeFactory activeContextBridgeFactory,
          final REEFFileNames reefFileNames,
          final AllocatedEvaluatorBridgeFactory allocatedEvaluatorBridgeFactory,
          final CLRProcessFactory clrProcessFactory,
          @Parameter(DefinedRuntimes.class) final Set<String> definedRuntimes) {
  this.clock = clock;
  this.httpServer = httpServer;
  this.jobMessageObserver = jobMessageObserver;
  this.evaluatorRequestor = evaluatorRequestor;
  this.nameServer = nameServer;
  this.driverStatusManager = driverStatusManager;
  this.activeContextBridgeFactory = activeContextBridgeFactory;
  this.allocatedEvaluatorBridgeFactory = allocatedEvaluatorBridgeFactory;
  this.nameServerInfo = localAddressProvider.getLocalAddress() + ":" + this.nameServer.getPort();
  this.loggingScopeFactory = loggingScopeFactory;
  this.reefFileNames = reefFileNames;
  this.localAddressProvider = localAddressProvider;
  this.clrProcessFactory = clrProcessFactory;
  this.definedRuntimes = definedRuntimes;
}
 
Example #7
Source File: GroupCommDriverImpl.java    From reef with Apache License 2.0 4 votes vote down vote up
@Inject
private GroupCommDriverImpl(final ConfigurationSerializer confSerializer,
                           @Parameter(DriverIdentifier.class) final String driverId,
                           @Parameter(TreeTopologyFanOut.class) final int fanOut,
                           final LocalAddressProvider localAddressProvider,
                           final TransportFactory tpFactory,
                           final NameServer nameService) {
  assert SingletonAsserter.assertSingleton(getClass());
  this.fanOut = fanOut;
  this.nameService = nameService;
  this.nameServiceAddr = localAddressProvider.getLocalAddress();
  this.nameServicePort = nameService.getPort();
  this.confSerializer = confSerializer;
  this.groupCommRunningTaskHandler = new BroadcastingEventHandler<>();
  this.groupCommRunningTaskStage = new SyncStage<>("GroupCommRunningTaskStage", groupCommRunningTaskHandler);
  this.groupCommFailedTaskHandler = new BroadcastingEventHandler<>();
  this.groupCommFailedTaskStage = new SyncStage<>("GroupCommFailedTaskStage", groupCommFailedTaskHandler);
  this.groupCommFailedEvaluatorHandler = new BroadcastingEventHandler<>();
  this.groupCommFailedEvaluatorStage = new SyncStage<>("GroupCommFailedEvaluatorStage",
      groupCommFailedEvaluatorHandler);
  this.groupCommMessageHandler = new GroupCommMessageHandler();
  this.groupCommMessageStage = new SyncStage<>("GroupCommMessageStage", groupCommMessageHandler);

  final Configuration nameResolverConf = Tang.Factory.getTang().newConfigurationBuilder(NameResolverConfiguration.CONF
      .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, nameServiceAddr)
      .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServicePort)
      .build())
      .build();

  final NameResolver nameResolver;
  try {
    nameResolver = Tang.Factory.getTang().newInjector(nameResolverConf).getInstance(NameResolver.class);
  } catch (final InjectionException e) {
    throw new RuntimeException("Failed to instantiate NameResolver", e);
  }

  try {
    final Injector injector = TANG.newInjector();
    injector.bindVolatileParameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class, idFac);
    injector.bindVolatileInstance(NameResolver.class, nameResolver);
    injector.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class,
        new GroupCommunicationMessageCodec());
    injector.bindVolatileParameter(NetworkServiceParameters.NetworkServiceTransportFactory.class, tpFactory);
    injector.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
        new EventHandler<Message<GroupCommunicationMessage>>() {
          @Override
          public void onNext(final Message<GroupCommunicationMessage> msg) {
            groupCommMessageStage.onNext(Utils.getGCM(msg));
          }
        });
    injector.bindVolatileParameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class,
        new LoggingEventHandler<Exception>());
    this.netService = injector.getInstance(NetworkService.class);
  } catch (final InjectionException e) {
    throw new RuntimeException("Failed to instantiate NetworkService", e);
  }
  this.netService.registerId(idFac.getNewInstance(driverId));
  final EStage<GroupCommunicationMessage> senderStage
      = new ThreadPoolStage<>("SrcCtrlMsgSender", new CtrlMsgSender(idFac, netService), 5);

  final Injector injector = TANG.newInjector();
  injector.bindVolatileParameter(GroupCommSenderStage.class, senderStage);
  injector.bindVolatileParameter(DriverIdentifier.class, driverId);
  injector.bindVolatileParameter(GroupCommRunningTaskHandler.class, groupCommRunningTaskHandler);
  injector.bindVolatileParameter(GroupCommFailedTaskHandler.class, groupCommFailedTaskHandler);
  injector.bindVolatileParameter(GroupCommFailedEvalHandler.class, groupCommFailedEvaluatorHandler);
  injector.bindVolatileInstance(GroupCommMessageHandler.class, groupCommMessageHandler);

  try {
    commGroupDriverFactory = injector.getInstance(CommunicationGroupDriverFactory.class);
  } catch (final InjectionException e) {
    throw new RuntimeException(e);
  }
}
 
Example #8
Source File: NameServiceCloseHandler.java    From reef with Apache License 2.0 4 votes vote down vote up
@Inject
public NameServiceCloseHandler(final NameServer toClose) {
  this.toClose = toClose;
}
 
Example #9
Source File: NetworkServiceTest.java    From reef with Apache License 2.0 4 votes vote down vote up
/**
 * NetworkService messaging test.
 */
@Test
public void testMessagingNetworkService() throws Exception {
  LOG.log(Level.FINEST, name.getMethodName());

  final IdentifierFactory factory = new StringIdentifierFactory();

  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);

  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int nameServerPort = server.getPort();

    final int numMessages = 10;
    final Monitor monitor = new Monitor();

    // network service
    final String name2 = "task2";
    final String name1 = "task1";
    final Configuration nameResolverConf =
        Tang.Factory.getTang().newConfigurationBuilder(NameResolverConfiguration.CONF
        .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, this.localAddress)
        .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServerPort)
        .build())
        .build();

    final Injector injector2 = Tang.Factory.getTang().newInjector(nameResolverConf);

    LOG.log(Level.FINEST, "=== Test network service receiver start");
    LOG.log(Level.FINEST, "=== Test network service sender start");
    try (NameResolver nameResolver = injector2.getInstance(NameResolver.class)) {
      injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class, factory);
      injector2.bindVolatileInstance(NameResolver.class, nameResolver);
      injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class, new StringCodec());
      injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceTransportFactory.class,
          injector.getInstance(MessagingTransportFactory.class));
      injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class,
          new ExceptionHandler());

      final Injector injectorNs2 = injector2.forkInjector();
      injectorNs2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
          new MessageHandler<String>(name2, monitor, numMessages));
      final NetworkService<String> ns2 = injectorNs2.getInstance(NetworkService.class);

      final Injector injectorNs1 = injector2.forkInjector();
      injectorNs1.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
          new MessageHandler<String>(name1, null, 0));
      final NetworkService<String> ns1 = injectorNs1.getInstance(NetworkService.class);

      ns2.registerId(factory.getNewInstance(name2));
      final int port2 = ns2.getTransport().getListeningPort();
      server.register(factory.getNewInstance("task2"), new InetSocketAddress(this.localAddress, port2));

      ns1.registerId(factory.getNewInstance(name1));
      final int port1 = ns1.getTransport().getListeningPort();
      server.register(factory.getNewInstance("task1"), new InetSocketAddress(this.localAddress, port1));

      final Identifier destId = factory.getNewInstance(name2);

      try (Connection<String> conn = ns1.newConnection(destId)) {
        conn.open();
        for (int count = 0; count < numMessages; ++count) {
          conn.write("hello! " + count);
        }
        monitor.mwait();

      } catch (final NetworkException e) {
        e.printStackTrace();
        throw new RuntimeException(e);
      }
    }
  }
}
 
Example #10
Source File: NetworkServiceTest.java    From reef with Apache License 2.0 4 votes vote down vote up
/**
 * NetworkService messaging rate benchmark.
 */
@Test
public void testMessagingNetworkServiceRate() throws Exception {

  Assume.assumeFalse("Use log level INFO to run benchmarking", LOG.isLoggable(Level.FINEST));

  LOG.log(Level.FINEST, name.getMethodName());

  final IdentifierFactory factory = new StringIdentifierFactory();

  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);

  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int nameServerPort = server.getPort();

    final int[] messageSizes = {1, 16, 32, 64, 512, 64 * 1024, 1024 * 1024};

    for (final int size : messageSizes) {
      final int numMessages = 300000 / (Math.max(1, size / 512));
      final Monitor monitor = new Monitor();

      // network service
      final String name2 = "task2";
      final String name1 = "task1";
      final Configuration nameResolverConf =
          Tang.Factory.getTang().newConfigurationBuilder(NameResolverConfiguration.CONF
          .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, this.localAddress)
          .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServerPort)
          .build())
          .build();

      final Injector injector2 = Tang.Factory.getTang().newInjector(nameResolverConf);

      LOG.log(Level.FINEST, "=== Test network service receiver start");
      LOG.log(Level.FINEST, "=== Test network service sender start");
      try (NameResolver nameResolver = injector2.getInstance(NameResolver.class)) {
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class, factory);
        injector2.bindVolatileInstance(NameResolver.class, nameResolver);
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class, new StringCodec());
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceTransportFactory.class,
            injector.getInstance(MessagingTransportFactory.class));
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class,
            new ExceptionHandler());

        final Injector injectorNs2 = injector2.forkInjector();
        injectorNs2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name2, monitor, numMessages));
        final NetworkService<String> ns2 = injectorNs2.getInstance(NetworkService.class);

        final Injector injectorNs1 = injector2.forkInjector();
        injectorNs1.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name1, null, 0));
        final NetworkService<String> ns1 = injectorNs1.getInstance(NetworkService.class);

        ns2.registerId(factory.getNewInstance(name2));
        final int port2 = ns2.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task2"), new InetSocketAddress(this.localAddress, port2));

        ns1.registerId(factory.getNewInstance(name1));
        final int port1 = ns1.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task1"), new InetSocketAddress(this.localAddress, port1));

        final Identifier destId = factory.getNewInstance(name2);
        final String message = StringUtils.repeat('1', size);

        final long start = System.currentTimeMillis();
        try (Connection<String> conn = ns1.newConnection(destId)) {
          conn.open();
          for (int i = 0; i < numMessages; i++) {
            conn.write(message);
          }
          monitor.mwait();
        } catch (final NetworkException e) {
          e.printStackTrace();
          throw new RuntimeException(e);
        }
        final long end = System.currentTimeMillis();
        final double runtime = ((double) end - start) / 1000;
        LOG.log(Level.FINEST, "size: " + size + "; messages/s: " + numMessages / runtime +
            " bandwidth(bytes/s): " + ((double) numMessages * 2 * size) / runtime); // x2 for unicode chars
      }
    }
  }
}
 
Example #11
Source File: NetworkServiceTest.java    From reef with Apache License 2.0 4 votes vote down vote up
@Test
public void testMultithreadedSharedConnMessagingNetworkServiceRate() throws Exception {

  Assume.assumeFalse("Use log level INFO to run benchmarking", LOG.isLoggable(Level.FINEST));

  LOG.log(Level.FINEST, name.getMethodName());

  final IdentifierFactory factory = new StringIdentifierFactory();

  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);
  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int nameServerPort = server.getPort();

    final int[] messageSizes = {2000}; // {1,16,32,64,512,64*1024,1024*1024};

    for (final int size : messageSizes) {
      final int numMessages = 300000 / (Math.max(1, size / 512));
      final int numThreads = 2;
      final int totalNumMessages = numMessages * numThreads;
      final Monitor monitor = new Monitor();


      // network service
      final String name2 = "task2";
      final String name1 = "task1";
      final Configuration nameResolverConf =
          Tang.Factory.getTang().newConfigurationBuilder(NameResolverConfiguration.CONF
          .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, this.localAddress)
          .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServerPort)
          .build())
          .build();

      final Injector injector2 = Tang.Factory.getTang().newInjector(nameResolverConf);

      LOG.log(Level.FINEST, "=== Test network service receiver start");
      LOG.log(Level.FINEST, "=== Test network service sender start");
      try (NameResolver nameResolver = injector2.getInstance(NameResolver.class)) {
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class, factory);
        injector2.bindVolatileInstance(NameResolver.class, nameResolver);
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class, new StringCodec());
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceTransportFactory.class,
            injector.getInstance(MessagingTransportFactory.class));
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class,
            new ExceptionHandler());

        final Injector injectorNs2 = injector2.forkInjector();
        injectorNs2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name2, monitor, totalNumMessages));
        final NetworkService<String> ns2 = injectorNs2.getInstance(NetworkService.class);

        final Injector injectorNs1 = injector2.forkInjector();
        injectorNs1.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name1, null, 0));
        final NetworkService<String> ns1 = injectorNs1.getInstance(NetworkService.class);

        ns2.registerId(factory.getNewInstance(name2));
        final int port2 = ns2.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task2"), new InetSocketAddress(this.localAddress, port2));

        ns1.registerId(factory.getNewInstance(name1));
        final int port1 = ns1.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task1"), new InetSocketAddress(this.localAddress, port1));

        final Identifier destId = factory.getNewInstance(name2);

        try (Connection<String> conn = ns1.newConnection(destId)) {
          conn.open();

          final String message = StringUtils.repeat('1', size);
          final ExecutorService e = Executors.newCachedThreadPool();

          final long start = System.currentTimeMillis();
          for (int i = 0; i < numThreads; i++) {
            e.submit(new Runnable() {

              @Override
              public void run() {
                for (int i = 0; i < numMessages; i++) {
                  conn.write(message);
                }
              }
            });
          }


          e.shutdown();
          e.awaitTermination(30, TimeUnit.SECONDS);
          monitor.mwait();

          final long end = System.currentTimeMillis();
          final double runtime = ((double) end - start) / 1000;

          LOG.log(Level.FINEST, "size: " + size + "; messages/s: " + totalNumMessages / runtime + 
              " bandwidth(bytes/s): " + ((double) totalNumMessages * 2 * size) / runtime); // x2 for unicode chars
        }
      }
    }
  }
}
 
Example #12
Source File: NetworkServiceTest.java    From reef with Apache License 2.0 4 votes vote down vote up
/**
 * NetworkService messaging rate benchmark.
 */
@Test
public void testMessagingNetworkServiceBatchingRate() throws Exception {

  Assume.assumeFalse("Use log level INFO to run benchmarking", LOG.isLoggable(Level.FINEST));

  LOG.log(Level.FINEST, name.getMethodName());

  final IdentifierFactory factory = new StringIdentifierFactory();

  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);

  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int nameServerPort = server.getPort();

    final int batchSize = 1024 * 1024;
    final int[] messageSizes = {32, 64, 512};

    for (final int size : messageSizes) {
      final int numMessages = 300 / (Math.max(1, size / 512));
      final Monitor monitor = new Monitor();

      // network service
      final String name2 = "task2";
      final String name1 = "task1";
      final Configuration nameResolverConf =
          Tang.Factory.getTang().newConfigurationBuilder(NameResolverConfiguration.CONF
          .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, this.localAddress)
          .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServerPort)
          .build())
          .build();

      final Injector injector2 = Tang.Factory.getTang().newInjector(nameResolverConf);

      LOG.log(Level.FINEST, "=== Test network service receiver start");
      LOG.log(Level.FINEST, "=== Test network service sender start");
      try (NameResolver nameResolver = injector2.getInstance(NameResolver.class)) {
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class, factory);
        injector2.bindVolatileInstance(NameResolver.class, nameResolver);
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class, new StringCodec());
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceTransportFactory.class,
            injector.getInstance(MessagingTransportFactory.class));
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class,
            new ExceptionHandler());

        final Injector injectorNs2 = injector2.forkInjector();
        injectorNs2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name2, monitor, numMessages));
        final NetworkService<String> ns2 = injectorNs2.getInstance(NetworkService.class);

        final Injector injectorNs1 = injector2.forkInjector();
        injectorNs1.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name1, null, 0));
        final NetworkService<String> ns1 = injectorNs1.getInstance(NetworkService.class);

        ns2.registerId(factory.getNewInstance(name2));
        final int port2 = ns2.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task2"), new InetSocketAddress(this.localAddress, port2));

        ns1.registerId(factory.getNewInstance(name1));
        final int port1 = ns1.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task1"), new InetSocketAddress(this.localAddress, port1));

        final Identifier destId = factory.getNewInstance(name2);
        final String message = StringUtils.repeat('1', batchSize);

        final long start = System.currentTimeMillis();
        try (Connection<String> conn = ns1.newConnection(destId)) {
          conn.open();
          for (int i = 0; i < numMessages; i++) {
            conn.write(message);
          }
          monitor.mwait();
        } catch (final NetworkException e) {
          e.printStackTrace();
          throw new RuntimeException(e);
        }
        final long end = System.currentTimeMillis();
        final double runtime = ((double) end - start) / 1000;
        final long numAppMessages = numMessages * batchSize / size;
        LOG.log(Level.FINEST, "size: " + size + "; messages/s: " + numAppMessages / runtime +
            " bandwidth(bytes/s): " + ((double) numAppMessages * 2 * size) / runtime); // x2 for unicode chars
      }
    }
  }
}