org.apache.reef.exception.evaluator.NetworkException Java Examples

@Override
public void send(final List<T> elements, final Integer... counts)
    throws NetworkException, InterruptedException {
  LOG.entering("ScatterSender", "send");

  initializeGroup();
  if (counts.length != commGroupClient.getActiveSlaveTasks().size()) {
    throw new RuntimeException("Parameter 'counts' has length " + counts.length
        + ", but number of slaves is " + commGroupClient.getActiveSlaveTasks().size());
  }

  send(elements,
      Arrays.asList(counts),
      commGroupClient.getActiveSlaveTasks());

  LOG.exiting("ScatterSender", "send");
}
 

private void runMessagingNetworkConnectionService(final Codec<String> codec) throws Exception {
  final int numMessages = 2000;
  final Monitor monitor = new Monitor();
  try (NetworkMessagingTestService messagingTestService = new NetworkMessagingTestService(localAddress)) {
    messagingTestService.registerTestConnectionFactory(groupCommClientId, numMessages, monitor, codec);

    try (Connection<String> conn = messagingTestService.getConnectionFromSenderToReceiver(groupCommClientId)) {
      try {
        conn.open();
        for (int count = 0; count < numMessages; ++count) {
          // send messages to the receiver.
          conn.write("hello" + count);
        }
        monitor.mwait();
      } catch (final NetworkException e) {
        e.printStackTrace();
        throw new RuntimeException(e);
      }
    }
  }
}
 

@Override
public <T> Future<MessageSender<T>> asyncConnect(final String receiverId, final String listenerId) {
  // If the connection toward the receiver exists already, reuses it.
  final Connection connection;

  if (receiverToConnectionMap.containsKey(receiverId)) {
    connection = receiverToConnectionMap.get(receiverId);
  } else {
    connection = connectionFactory.newConnection(idFactory.getNewInstance(receiverId));
    try {
      connection.open();
    } catch (final NetworkException e) {
      try {
        connection.close();
      } catch (final NetworkException exceptionToIgnore) {
        LOG.info("Can't close the broken connection.", exceptionToIgnore);
      }

      final CompletableFuture<MessageSender<T>> failedFuture = new CompletableFuture<>();
      failedFuture.completeExceptionally(e);
      return failedFuture;
    }
  }

  return CompletableFuture.completedFuture((MessageSender) new NcsMessageSender(connection, replyFutureMap));
}
 

@Override
public void send(final T myData) throws NetworkException, InterruptedException {
  LOG.entering("ReduceSender", "send", this);
  LOG.fine("I am " + this);

  if (init.compareAndSet(false, true)) {
    commGroupClient.initialize();
  }
  // I am an intermediate node or leaf.
  LOG.finest("Waiting for children");
  // Wait for children to send
  try {
    final T reducedValueOfChildren = topology.recvFromChildren(reduceFunction, dataCodec);
    final List<T> vals = new ArrayList<>(2);
    vals.add(myData);
    if (reducedValueOfChildren != null) {
      vals.add(reducedValueOfChildren);
    }
    final T reducedValue = reduceFunction.apply(vals);
    topology.sendToParent(dataCodec.encode(reducedValue), ReefNetworkGroupCommProtos.GroupCommMessage.Type.Reduce);
  } catch (final ParentDeadException e) {
    throw new RuntimeException("ParentDeadException", e);
  }
  LOG.exiting("ReduceSender", "send", this);
}
 

@Override
public void send(final T element) throws NetworkException, InterruptedException {
  LOG.entering("BroadcastSender", "send", this);
  LOG.fine("I am " + this);

  if (init.compareAndSet(false, true)) {
    LOG.fine(this + " Communication group initializing");
    commGroupClient.initialize();
    LOG.fine(this + " Communication group initialized");
  }

  try {
    topology.sendToChildren(dataCodec.encode(element), ReefNetworkGroupCommProtos.GroupCommMessage.Type.Broadcast);
  } catch (final ParentDeadException e) {
    throw new RuntimeException("ParentDeadException", e);
  }
  LOG.exiting("BroadcastSender", "send", this);
}
 

@Override
public List<T> receive(final List<? extends Identifier> order) throws NetworkException, InterruptedException {
  LOG.entering("GatherReceiver", "receive");
  final Map<String, T> mapOfTaskIdToData = receiveMapOfTaskIdToData();

  LOG.log(Level.FINE, "{0} Sorting data according to specified order of task identifiers.", this);
  final List<T> retList = new LinkedList<>();
  for (final Identifier key : order) {
    final String keyString = key.toString();
    if (mapOfTaskIdToData.containsKey(keyString)) {
      retList.add(mapOfTaskIdToData.get(key.toString()));
    } else {
      LOG.warning(this + " Received no data from " + keyString + ". Adding null.");
      retList.add(null);
    }
  }

  LOG.exiting("GatherReceiver", "receive");
  return retList;
}
 

@Override
public T reduce() throws InterruptedException, NetworkException {
  LOG.entering("ReduceReceiver", "reduce", this);
  LOG.fine("I am " + this);

  if (init.compareAndSet(false, true)) {
    commGroupClient.initialize();
  }
  // I am root
  LOG.fine(this + " Waiting to receive reduced value");
  // Wait for children to send
  final T redVal;
  try {
    redVal = topology.recvFromChildren(reduceFunction, dataCodec);
  } catch (final ParentDeadException e) {
    throw new RuntimeException("ParentDeadException", e);
  }
  LOG.exiting("ReduceReceiver", "reduce", this);
  return redVal;
}
 

@Override
public void onNext(final GroupCommunicationMessage srcCtrlMsg) {
  LOG.entering("CtrlMsgSender", "onNext", srcCtrlMsg);
  final Identifier id = idFac.getNewInstance(srcCtrlMsg.getDestid());
  final Connection<GroupCommunicationMessage> link = netService.newConnection(id);
  try {
    link.open();
    link.write(srcCtrlMsg);
  } catch (final NetworkException e) {
    throw new RuntimeException("Unable to send ctrl task msg to parent " + id, e);
  }
  LOG.exiting("CtrlMsgSender", "onNext", srcCtrlMsg);
}
 

@Override
public void send(final List<T> elements, final List<Integer> counts, final List<? extends Identifier> order)
    throws NetworkException, InterruptedException {
  LOG.entering("ScatterSender", "send");

  if (counts.size() != order.size()) {
    throw new RuntimeException("Parameter 'counts' has size " + counts.size()
        + ", but parameter 'order' has size " + order.size() + ".");
  }
  initializeGroup();

  // I am root.
  LOG.fine("I am " + this);

  LOG.fine(this + " Encoding data and determining which Tasks receive which elements.");
  final Map<String, byte[]> mapOfChildIdToBytes = scatterEncoder.encode(elements, counts, order, dataCodec);

  try {
    LOG.fine(this + " Sending " + elements.size() + " elements.");
    topology.sendToChildren(mapOfChildIdToBytes, ReefNetworkGroupCommProtos.GroupCommMessage.Type.Scatter);

  } catch (final ParentDeadException e) {
    throw new RuntimeException("ParentDeadException during OperatorTopology.sendToChildren()", e);
  }

  LOG.exiting("ScatterSender", "send");
}
 

@Override
public void send(final List<T> elements) throws NetworkException, InterruptedException {
  LOG.entering("ScatterSender", "send");

  initializeGroup();
  send(elements,
      ScatterHelper.getUniformCounts(elements.size(), commGroupClient.getActiveSlaveTasks().size()),
      commGroupClient.getActiveSlaveTasks());

  LOG.exiting("ScatterSender", "send");
}
 

@Override
public void send(final List<T> elements, final List<? extends Identifier> order)
    throws NetworkException, InterruptedException {
  LOG.entering("ScatterSender", "send");

  initializeGroup();
  send(elements,
      ScatterHelper.getUniformCounts(elements.size(), order.size()),
      order);

  LOG.exiting("ScatterSender", "send");
}
 

@Override
public T receive() throws NetworkException, InterruptedException {
  LOG.entering("BroadcastReceiver", "receive", this);
  LOG.fine("I am " + this);

  if (init.compareAndSet(false, true)) {
    LOG.fine(this + " Communication group initializing");
    commGroupClient.initialize();
    LOG.fine(this + " Communication group initialized");
  }
  // I am an intermediate node or leaf.

  final T retVal;
  // Wait for parent to send
  LOG.fine(this + " Waiting to receive broadcast");
  final byte[] data;
  try {
    data = topology.recvFromParent(ReefNetworkGroupCommProtos.GroupCommMessage.Type.Broadcast);
    // TODO: Should receive the identity element instead of null
    if (data == null) {
      LOG.fine(this + " Received null. Perhaps one of my ancestors is dead.");
      retVal = null;
    } else {
      LOG.finest("Using " + dataCodec.getClass().getSimpleName() + " as codec");
      retVal = dataCodec.decode(data);
      LOG.finest("Decoded msg successfully");
      LOG.finest(this + " Sending to children.");
    }

    topology.sendToChildren(data, ReefNetworkGroupCommProtos.GroupCommMessage.Type.Broadcast);
  } catch (final ParentDeadException e) {
    throw new RuntimeException("ParentDeadException", e);
  }
  LOG.exiting("BroadcastReceiver", "receive", this);
  return retVal;
}
 

public void send(final GroupCommunicationMessage msg, final String dest) throws NetworkException {
  LOG.entering("Sender", "send", msg);
  final Identifier destId = idFac.getNewInstance(dest);
  final Connection<GroupCommunicationMessage> link = netService.newConnection(destId);
  link.open();
  link.write(msg);
  LOG.exiting("Sender", "send", msg);
}
 

@Override
public List<T> receive() throws NetworkException, InterruptedException {
  LOG.entering("GatherReceiver", "receive");
  final Map<String, T> mapOfTaskIdToData = receiveMapOfTaskIdToData();

  LOG.log(Level.FINE, "{0} Sorting data according to lexicographical order of task identifiers.", this);
  final TreeMap<String, T> sortedMapOfTaskIdToData = new TreeMap<>(mapOfTaskIdToData);
  final List<T> retList = new LinkedList<>(sortedMapOfTaskIdToData.values());

  LOG.exiting("GatherReceiver", "receive");
  return retList;
}
 

/**
 * Open a channel for destination identifier of NetworkConnectionService.
 * @param connectionFactoryId
 * @param remoteEndPointId
 * @throws NetworkException
 */
<T> Link<NetworkConnectionServiceMessage<T>> openLink(
    final Identifier connectionFactoryId, final Identifier remoteEndPointId) throws NetworkException {
  final Identifier remoteId = getEndPointIdWithConnectionFactoryId(connectionFactoryId, remoteEndPointId);
  try {
    final SocketAddress address = nameResolver.lookup(remoteId);
    if (address == null) {
      throw new NetworkException("Lookup " + remoteId + " is null");
    }
    return transport.open(address, nsCodec, nsLinkListener);
  } catch(final Exception e) {
    throw new NetworkException(e);
  }
}
 

public <T> void registerTestConnectionFactory(final Identifier connFactoryId,
                                              final int numMessages, final Monitor monitor,
                                              final Codec<T> codec) throws NetworkException {
  final Identifier receiverEndPointId = factory.getNewInstance("receiver");
  final Identifier senderEndPointId = factory.getNewInstance("sender");
  receiverNetworkConnService.registerConnectionFactory(connFactoryId, codec,
      new MessageHandler<T>(monitor, numMessages, senderEndPointId, receiverEndPointId),
      new TestListener<T>(), receiverEndPointId);
  senderNetworkConnService.registerConnectionFactory(connFactoryId, codec,
      new MessageHandler<T>(monitor, numMessages, receiverEndPointId, senderEndPointId),
      new TestListener<T>(), senderEndPointId);
}
 

private void updateModel(final Vector descentDirection) throws NetworkException, InterruptedException {
  try (Timer t = new Timer("GetDescentDirection + FindMinEta + UpdateModel")) {
    final Vector lineSearchEvals = lineSearch(descentDirection);
    minEta = findMinEta(model, descentDirection, lineSearchEvals);
    model.multAdd(minEta, descentDirection);
  }

  LOG.log(Level.INFO, "OUT: New Model = {0}", model);
}
 

private Vector lineSearch(final Vector descentDirection) throws NetworkException, InterruptedException {
  Vector lineSearchResults = null;
  boolean allDead = false;
  do {
    try (Timer t = new Timer("LineSearch - Broadcast("
        + (sendModel ? "ModelAndDescentDirection" : "DescentDirection") + ") + Reduce(LossEvalsInLineSearch)")) {
      if (sendModel) {
        LOG.log(Level.INFO, "OUT: DoLineSearchWithModel");
        controlMessageBroadcaster.send(ControlMessages.DoLineSearchWithModel);
        modelAndDescentDirectionBroadcaster.send(new Pair<>(model, descentDirection));
      } else {
        LOG.log(Level.INFO, "OUT: DoLineSearch");
        controlMessageBroadcaster.send(ControlMessages.DoLineSearch);
        descentDriectionBroadcaster.send(descentDirection);
      }
      final Pair<Vector, Integer> lineSearchEvals = lineSearchEvaluationsReducer.reduce();
      if (lineSearchEvals != null) {
        final int numExamples = lineSearchEvals.getSecond();
        lineSearchResults = lineSearchEvals.getFirst();
        lineSearchResults.scale(1.0 / numExamples);
        LOG.log(Level.INFO, "OUT: #Examples: {0}", numExamples);
        LOG.log(Level.INFO, "OUT: LineSearchEvals: {0}", lineSearchResults);
        allDead = false;
      } else {
        allDead = true;
      }
    }

    sendModel = chkAndUpdate();
  } while (allDead || !ignoreAndContinue && sendModel);
  return lineSearchResults;
}
 

private Pair<Double, Vector> computeLossAndGradient() throws NetworkException, InterruptedException {
  Pair<Double, Vector> returnValue = null;
  boolean allDead = false;
  do {
    try (Timer t = new Timer("Broadcast(" + (sendModel ? "Model" : "MinEta") + ") + Reduce(LossAndGradient)")) {
      if (sendModel) {
        LOG.log(Level.INFO, "OUT: ComputeGradientWithModel");
        controlMessageBroadcaster.send(ControlMessages.ComputeGradientWithModel);
        modelBroadcaster.send(model);
      } else {
        LOG.log(Level.INFO, "OUT: ComputeGradientWithMinEta");
        controlMessageBroadcaster.send(ControlMessages.ComputeGradientWithMinEta);
        minEtaBroadcaster.send(minEta);
      }
      final Pair<Pair<Double, Integer>, Vector> lossAndGradient = lossAndGradientReducer.reduce();

      if (lossAndGradient != null) {
        final int numExamples = lossAndGradient.getFirst().getSecond();
        LOG.log(Level.INFO, "OUT: #Examples: {0}", numExamples);
        final double lossPerExample = lossAndGradient.getFirst().getFirst() / numExamples;
        LOG.log(Level.INFO, "OUT: Loss: {0}", lossPerExample);
        final double objFunc = (lambda / 2) * model.norm2Sqr() + lossPerExample;
        LOG.log(Level.INFO, "OUT: Objective Func Value: {0}", objFunc);
        final Vector gradient = lossAndGradient.getSecond();
        gradient.scale(1.0 / numExamples);
        LOG.log(Level.INFO, "OUT: Gradient: {0}", gradient);
        returnValue = new Pair<>(objFunc, gradient);
        allDead = false;
      } else {
        allDead = true;
      }
    }
    sendModel = chkAndUpdate();
  } while (allDead || !ignoreAndContinue && sendModel);
  return returnValue;
}
 

@Override
public <U> void reply(final U replyMessage) {
  final Connection connection = connectionFactory.newConnection(idFactory.getNewInstance(senderId));
  try {
    connection.open();
    connection.write(replyMessage);
    // We do not call connection.close since NCS caches connection.
  } catch (final NetworkException e) {
    // TODO #140: Properly classify and handle each RPC failure
    // Not logging the stacktrace here, as it's not very useful.
    LOG.error("NCS Exception");
  }
}
 

@Override
public <U> void reply(final U replyMessage) {
  LOG.debug("[REPLY]: {}", replyMessage);
  final Connection connection = connectionFactory.newConnection(idFactory.getNewInstance(senderId));
  try {
    connection.open();
  } catch (final NetworkException e) {
    throw new RuntimeException("Cannot connect to " + senderId, e);
  }

  connection.write(replyMessage);
}
 

@Override
public List<T> receive() throws NetworkException, InterruptedException {
  LOG.entering("ScatterReceiver", "receive");
  // I am intermediate node or leaf.
  LOG.fine("I am " + this);

  if (init.compareAndSet(false, true)) {
    LOG.fine(this + " Communication group initializing.");
    commGroupClient.initialize();
    LOG.fine(this + " Communication group initialized.");
  }

  try {
    LOG.fine(this + " Waiting to receive scatter from parent.");
    final byte[] data = topology.recvFromParent(ReefNetworkGroupCommProtos.GroupCommMessage.Type.Scatter);

    if (data == null) {
      LOG.fine(this + " Received null. Perhaps one of my ancestors is dead.");
      LOG.exiting("ScatterSender", "receive", null);
      return null;
    }

    LOG.fine(this + " Successfully received scattered data.");
    final ScatterData scatterData = scatterDecoder.decode(data);

    LOG.fine(this + " Trying to propagate messages to children.");
    topology.sendToChildren(scatterData.getChildrenData(), ReefNetworkGroupCommProtos.GroupCommMessage.Type.Scatter);

    LOG.fine(this + " Decoding data elements sent to me.");
    final List<T> retList = new LinkedList<>();
    for (final byte[] singleData : scatterData.getMyData()) {
      retList.add(dataCodec.decode(singleData));
    }

    LOG.exiting("ScatterSender", "receive", retList);
    return retList;

  } catch (final ParentDeadException e) {
    throw new RuntimeException("ParentDeadException", e);
  }
}
 

/**
 * NetworkService messaging rate benchmark.
 */
@Test
public void testMessagingNetworkServiceBatchingRate() throws Exception {

  Assume.assumeFalse("Use log level INFO to run benchmarking", LOG.isLoggable(Level.FINEST));

  LOG.log(Level.FINEST, name.getMethodName());

  final IdentifierFactory factory = new StringIdentifierFactory();

  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);

  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int nameServerPort = server.getPort();

    final int batchSize = 1024 * 1024;
    final int[] messageSizes = {32, 64, 512};

    for (final int size : messageSizes) {
      final int numMessages = 300 / (Math.max(1, size / 512));
      final Monitor monitor = new Monitor();

      // network service
      final String name2 = "task2";
      final String name1 = "task1";
      final Configuration nameResolverConf =
          Tang.Factory.getTang().newConfigurationBuilder(NameResolverConfiguration.CONF
          .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, this.localAddress)
          .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServerPort)
          .build())
          .build();

      final Injector injector2 = Tang.Factory.getTang().newInjector(nameResolverConf);

      LOG.log(Level.FINEST, "=== Test network service receiver start");
      LOG.log(Level.FINEST, "=== Test network service sender start");
      try (NameResolver nameResolver = injector2.getInstance(NameResolver.class)) {
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class, factory);
        injector2.bindVolatileInstance(NameResolver.class, nameResolver);
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class, new StringCodec());
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceTransportFactory.class,
            injector.getInstance(MessagingTransportFactory.class));
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class,
            new ExceptionHandler());

        final Injector injectorNs2 = injector2.forkInjector();
        injectorNs2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name2, monitor, numMessages));
        final NetworkService<String> ns2 = injectorNs2.getInstance(NetworkService.class);

        final Injector injectorNs1 = injector2.forkInjector();
        injectorNs1.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name1, null, 0));
        final NetworkService<String> ns1 = injectorNs1.getInstance(NetworkService.class);

        ns2.registerId(factory.getNewInstance(name2));
        final int port2 = ns2.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task2"), new InetSocketAddress(this.localAddress, port2));

        ns1.registerId(factory.getNewInstance(name1));
        final int port1 = ns1.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task1"), new InetSocketAddress(this.localAddress, port1));

        final Identifier destId = factory.getNewInstance(name2);
        final String message = StringUtils.repeat('1', batchSize);

        final long start = System.currentTimeMillis();
        try (Connection<String> conn = ns1.newConnection(destId)) {
          conn.open();
          for (int i = 0; i < numMessages; i++) {
            conn.write(message);
          }
          monitor.mwait();
        } catch (final NetworkException e) {
          e.printStackTrace();
          throw new RuntimeException(e);
        }
        final long end = System.currentTimeMillis();
        final double runtime = ((double) end - start) / 1000;
        final long numAppMessages = numMessages * batchSize / size;
        LOG.log(Level.FINEST, "size: " + size + "; messages/s: " + numAppMessages / runtime +
            " bandwidth(bytes/s): " + ((double) numAppMessages * 2 * size) / runtime); // x2 for unicode chars
      }
    }
  }
}
 

/**
 * NetworkService messaging rate benchmark.
 */
@Test
public void testMessagingNetworkServiceRate() throws Exception {

  Assume.assumeFalse("Use log level INFO to run benchmarking", LOG.isLoggable(Level.FINEST));

  LOG.log(Level.FINEST, name.getMethodName());

  final IdentifierFactory factory = new StringIdentifierFactory();

  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);

  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int nameServerPort = server.getPort();

    final int[] messageSizes = {1, 16, 32, 64, 512, 64 * 1024, 1024 * 1024};

    for (final int size : messageSizes) {
      final int numMessages = 300000 / (Math.max(1, size / 512));
      final Monitor monitor = new Monitor();

      // network service
      final String name2 = "task2";
      final String name1 = "task1";
      final Configuration nameResolverConf =
          Tang.Factory.getTang().newConfigurationBuilder(NameResolverConfiguration.CONF
          .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, this.localAddress)
          .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServerPort)
          .build())
          .build();

      final Injector injector2 = Tang.Factory.getTang().newInjector(nameResolverConf);

      LOG.log(Level.FINEST, "=== Test network service receiver start");
      LOG.log(Level.FINEST, "=== Test network service sender start");
      try (NameResolver nameResolver = injector2.getInstance(NameResolver.class)) {
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class, factory);
        injector2.bindVolatileInstance(NameResolver.class, nameResolver);
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class, new StringCodec());
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceTransportFactory.class,
            injector.getInstance(MessagingTransportFactory.class));
        injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class,
            new ExceptionHandler());

        final Injector injectorNs2 = injector2.forkInjector();
        injectorNs2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name2, monitor, numMessages));
        final NetworkService<String> ns2 = injectorNs2.getInstance(NetworkService.class);

        final Injector injectorNs1 = injector2.forkInjector();
        injectorNs1.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
            new MessageHandler<String>(name1, null, 0));
        final NetworkService<String> ns1 = injectorNs1.getInstance(NetworkService.class);

        ns2.registerId(factory.getNewInstance(name2));
        final int port2 = ns2.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task2"), new InetSocketAddress(this.localAddress, port2));

        ns1.registerId(factory.getNewInstance(name1));
        final int port1 = ns1.getTransport().getListeningPort();
        server.register(factory.getNewInstance("task1"), new InetSocketAddress(this.localAddress, port1));

        final Identifier destId = factory.getNewInstance(name2);
        final String message = StringUtils.repeat('1', size);

        final long start = System.currentTimeMillis();
        try (Connection<String> conn = ns1.newConnection(destId)) {
          conn.open();
          for (int i = 0; i < numMessages; i++) {
            conn.write(message);
          }
          monitor.mwait();
        } catch (final NetworkException e) {
          e.printStackTrace();
          throw new RuntimeException(e);
        }
        final long end = System.currentTimeMillis();
        final double runtime = ((double) end - start) / 1000;
        LOG.log(Level.FINEST, "size: " + size + "; messages/s: " + numMessages / runtime +
            " bandwidth(bytes/s): " + ((double) numMessages * 2 * size) / runtime); // x2 for unicode chars
      }
    }
  }
}
 

/**
 * NetworkService messaging test.
 */
@Test
public void testMessagingNetworkService() throws Exception {
  LOG.log(Level.FINEST, name.getMethodName());

  final IdentifierFactory factory = new StringIdentifierFactory();

  final Injector injector = Tang.Factory.getTang().newInjector();
  injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory);
  injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider);

  try (NameServer server = injector.getInstance(NameServer.class)) {
    final int nameServerPort = server.getPort();

    final int numMessages = 10;
    final Monitor monitor = new Monitor();

    // network service
    final String name2 = "task2";
    final String name1 = "task1";
    final Configuration nameResolverConf =
        Tang.Factory.getTang().newConfigurationBuilder(NameResolverConfiguration.CONF
        .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, this.localAddress)
        .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServerPort)
        .build())
        .build();

    final Injector injector2 = Tang.Factory.getTang().newInjector(nameResolverConf);

    LOG.log(Level.FINEST, "=== Test network service receiver start");
    LOG.log(Level.FINEST, "=== Test network service sender start");
    try (NameResolver nameResolver = injector2.getInstance(NameResolver.class)) {
      injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceIdentifierFactory.class, factory);
      injector2.bindVolatileInstance(NameResolver.class, nameResolver);
      injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class, new StringCodec());
      injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceTransportFactory.class,
          injector.getInstance(MessagingTransportFactory.class));
      injector2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceExceptionHandler.class,
          new ExceptionHandler());

      final Injector injectorNs2 = injector2.forkInjector();
      injectorNs2.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
          new MessageHandler<String>(name2, monitor, numMessages));
      final NetworkService<String> ns2 = injectorNs2.getInstance(NetworkService.class);

      final Injector injectorNs1 = injector2.forkInjector();
      injectorNs1.bindVolatileParameter(NetworkServiceParameters.NetworkServiceHandler.class,
          new MessageHandler<String>(name1, null, 0));
      final NetworkService<String> ns1 = injectorNs1.getInstance(NetworkService.class);

      ns2.registerId(factory.getNewInstance(name2));
      final int port2 = ns2.getTransport().getListeningPort();
      server.register(factory.getNewInstance("task2"), new InetSocketAddress(this.localAddress, port2));

      ns1.registerId(factory.getNewInstance(name1));
      final int port1 = ns1.getTransport().getListeningPort();
      server.register(factory.getNewInstance("task1"), new InetSocketAddress(this.localAddress, port1));

      final Identifier destId = factory.getNewInstance(name2);

      try (Connection<String> conn = ns1.newConnection(destId)) {
        conn.open();
        for (int count = 0; count < numMessages; ++count) {
          conn.write("hello! " + count);
        }
        monitor.mwait();

      } catch (final NetworkException e) {
        e.printStackTrace();
        throw new RuntimeException(e);
      }
    }
  }
}
 

/**
 * NetworkService messaging rate benchmark.
 */
@Test
public void testMessagingNetworkConnServiceBatchingRate() throws Exception {

  Assume.assumeFalse("Use log level INFO to run benchmarking", LOG.isLoggable(Level.FINEST));

  LOG.log(Level.FINEST, name.getMethodName());

  final int batchSize = 1024 * 1024;
  final int[] messageSizes = {32, 64, 512};

  for (final int size : messageSizes) {
    final String message = StringUtils.repeat('1', batchSize);
    final int numMessages = 300 / (Math.max(1, size / 512));
    final Monitor monitor = new Monitor();
    final Codec<String> codec = new StringCodec();
    try (NetworkMessagingTestService messagingTestService = new NetworkMessagingTestService(localAddress)) {
      messagingTestService.registerTestConnectionFactory(groupCommClientId, numMessages, monitor, codec);
      try (Connection<String> conn =
               messagingTestService.getConnectionFromSenderToReceiver(groupCommClientId)) {
        final long start = System.currentTimeMillis();
        try {
          conn.open();
          for (int i = 0; i < numMessages; i++) {
            conn.write(message);
          }
          monitor.mwait();
        } catch (final NetworkException e) {
          e.printStackTrace();
          throw new RuntimeException(e);
        }

        final long end = System.currentTimeMillis();
        final double runtime = ((double) end - start) / 1000;
        final long numAppMessages = numMessages * batchSize / size;
        LOG.log(Level.INFO, "size: " + size + "; messages/s: " + numAppMessages / runtime +
            " bandwidth(bytes/s): " + ((double) numAppMessages * 2 * size) / runtime); // x2 for unicode chars
      }
    }
  }
}
 

/**
 * NetworkService messaging rate benchmark.
 */
@Test
public void testMessagingNetworkConnServiceRate() throws Exception {

  Assume.assumeFalse("Use log level INFO to run benchmarking", LOG.isLoggable(Level.FINEST));

  LOG.log(Level.FINEST, name.getMethodName());

  final int[] messageSizes = {1, 16, 32, 64, 512, 64 * 1024, 1024 * 1024};

  for (final int size : messageSizes) {
    final String message = StringUtils.repeat('1', size);
    final int numMessages = 300000 / (Math.max(1, size / 512));
    final Monitor monitor = new Monitor();
    final Codec<String> codec = new StringCodec();
    try (NetworkMessagingTestService messagingTestService = new NetworkMessagingTestService(localAddress)) {
      messagingTestService.registerTestConnectionFactory(groupCommClientId, numMessages, monitor, codec);

      try (Connection<String> conn =
               messagingTestService.getConnectionFromSenderToReceiver(groupCommClientId)) {

        final long start = System.currentTimeMillis();
        try {
          conn.open();
          for (int count = 0; count < numMessages; ++count) {
            // send messages to the receiver.
            conn.write(message);
          }
          monitor.mwait();
        } catch (final NetworkException e) {
          e.printStackTrace();
          throw new RuntimeException(e);
        }
        final long end = System.currentTimeMillis();

        final double runtime = ((double) end - start) / 1000;
        LOG.log(Level.INFO, "size: " + size + "; messages/s: " + numMessages / runtime +
            " bandwidth(bytes/s): " + ((double) numMessages * 2 * size) / runtime); // x2 for unicode chars
      }
    }
  }
}
 

/**
 * Closes the connection and unregisters it from the service.
 */
@Override
public void close() throws NetworkException {
  this.service.remove(this.destId);
}
 

Link<NetworkConnectionServiceMessage<T>> openLink(final Identifier remoteId) throws NetworkException {
  return networkService.openLink(connectionFactoryId, remoteId);
}
 

@Override
public void open() throws NetworkException {
  link = connFactory.openLink(destId);
}