io.prestosql.spi.HostAddress Java Examples

/**
 * Helper method to determine if a split is local to a node irrespective of whether splitAddresses contain port information or not
 */
private static boolean isSplitLocal(List<HostAddress> splitAddresses, HostAddress nodeAddress, SetMultimap<InetAddress, InternalNode> nodesByHost)
{
    for (HostAddress address : splitAddresses) {
        if (nodeAddress.equals(address)) {
            return true;
        }
        InetAddress inetAddress;
        try {
            inetAddress = address.toInetAddress();
        }
        catch (UnknownHostException e) {
            continue;
        }
        if (!address.hasPort()) {
            Set<InternalNode> localNodes = nodesByHost.get(inetAddress);
            return localNodes.stream()
                    .anyMatch(node -> node.getHostAndPort().equals(nodeAddress));
        }
    }
    return false;
}
 

@JsonCreator
public AccumuloSplit(
        @JsonProperty("ranges") List<WrappedRange> ranges,
        @JsonProperty("hostPort") Optional<String> hostPort)
{
    this.hostPort = requireNonNull(hostPort, "hostPort is null");
    this.ranges = ImmutableList.copyOf(requireNonNull(ranges, "ranges is null"));

    // Parse the host address into a list of addresses, this would be an Accumulo Tablet server or some localhost thing
    if (hostPort.isPresent()) {
        addresses = ImmutableList.of(HostAddress.fromString(hostPort.get()));
    }
    else {
        addresses = ImmutableList.of();
    }
}
 

@Test
public void testRefresh()
        throws Exception
{
    try (TempFile tempFile = new TempFile()) {
        Files.copy(topologyFile, tempFile.file());

        TestingTicker ticker = new TestingTicker();
        FileBasedNetworkTopology topology = new FileBasedNetworkTopology(tempFile.file(), Duration.valueOf("1d"), ticker);

        assertEquals(topology.locate(HostAddress.fromString("not-exist.example.com")), new NetworkLocation());
        assertEquals(topology.locate(HostAddress.fromString("192.168.0.1")), new NetworkLocation("region1", "rack1", "machine1"));
        assertEquals(topology.locate(HostAddress.fromString("192.168.0.2")), new NetworkLocation("region1", "rack1", "machine2"));
        assertEquals(topology.locate(HostAddress.fromString("192.168.0.3")), new NetworkLocation());

        assertEquals(topology.locate(HostAddress.fromString("new")), new NetworkLocation());
        Files.copy(topologyNewFile, tempFile.file());
        ticker.increment(1, TimeUnit.DAYS);

        assertEquals(topology.locate(HostAddress.fromString("new")), new NetworkLocation("new", "rack", "machine"));
        assertEquals(topology.locate(HostAddress.fromString("not-exist.example.com")), new NetworkLocation());
        assertEquals(topology.locate(HostAddress.fromString("192.168.0.1")), new NetworkLocation("region1", "rack1", "machine5"));
        assertEquals(topology.locate(HostAddress.fromString("192.168.0.2")), new NetworkLocation());
        assertEquals(topology.locate(HostAddress.fromString("192.168.0.3")), new NetworkLocation("region1", "rack1", "machine6"));
    }
}
 

private void updateRowsOnHosts(long tableId, Collection<Slice> fragments)
{
    TableInfo info = tables.get(tableId);
    checkState(
            info != null,
            "Uninitialized tableId [%s.%s]",
            info.getSchemaName(),
            info.getTableName());

    Map<HostAddress, MemoryDataFragment> dataFragments = new HashMap<>(info.getDataFragments());
    for (Slice fragment : fragments) {
        MemoryDataFragment memoryDataFragment = MemoryDataFragment.fromSlice(fragment);
        dataFragments.merge(memoryDataFragment.getHostAddress(), memoryDataFragment, MemoryDataFragment::merge);
    }

    tables.put(tableId, new TableInfo(tableId, info.getSchemaName(), info.getTableName(), info.getColumns(), dataFragments));
}
 

@JsonCreator
public KafkaSplit(
        @JsonProperty("topicName") String topicName,
        @JsonProperty("keyDataFormat") String keyDataFormat,
        @JsonProperty("messageDataFormat") String messageDataFormat,
        @JsonProperty("keyDataSchemaContents") Optional<String> keyDataSchemaContents,
        @JsonProperty("messageDataSchemaContents") Optional<String> messageDataSchemaContents,
        @JsonProperty("partitionId") int partitionId,
        @JsonProperty("messagesRange") Range messagesRange,
        @JsonProperty("leader") HostAddress leader)
{
    this.topicName = requireNonNull(topicName, "topicName is null");
    this.keyDataFormat = requireNonNull(keyDataFormat, "dataFormat is null");
    this.messageDataFormat = requireNonNull(messageDataFormat, "messageDataFormat is null");
    this.keyDataSchemaContents = keyDataSchemaContents;
    this.messageDataSchemaContents = messageDataSchemaContents;
    this.partitionId = partitionId;
    this.messagesRange = requireNonNull(messagesRange, "messagesRange is null");
    this.leader = requireNonNull(leader, "leader is null");
}
 

@JsonCreator
public TpcdsSplit(
        @JsonProperty("partNumber") int partNumber,
        @JsonProperty("totalParts") int totalParts,
        @JsonProperty("addresses") List<HostAddress> addresses,
        @JsonProperty("noSexism") boolean noSexism)
{
    checkState(partNumber >= 0, "partNumber must be >= 0");
    checkState(totalParts >= 1, "totalParts must be >= 1");
    checkState(totalParts > partNumber, "totalParts must be > partNumber");
    requireNonNull(addresses, "addresses is null");

    this.partNumber = partNumber;
    this.totalParts = totalParts;
    this.addresses = ImmutableList.copyOf(addresses);
    this.noSexism = noSexism;
}
 

@Test
public void testToHostAddressList()
        throws Exception
{
    Set<Host> hosts = ImmutableSet.of(
            new TestHost(
                    new InetSocketAddress(
                            InetAddress.getByAddress(new byte[] {
                                    1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16
                            }),
                            3000)),
            new TestHost(new InetSocketAddress(InetAddress.getByAddress(new byte[] {1, 2, 3, 4}), 3000)));

    HostAddressFactory hostAddressFactory = new HostAddressFactory();
    List<HostAddress> list = hostAddressFactory.toHostAddressList(hosts);

    assertEquals(list.toString(), "[[102:304:506:708:90a:b0c:d0e:f10], 1.2.3.4]");
}
 

@JsonCreator
public RedisSplit(
        @JsonProperty("schemaName") String schemaName,
        @JsonProperty("tableName") String tableName,
        @JsonProperty("keyDataFormat") String keyDataFormat,
        @JsonProperty("valueDataFormat") String valueDataFormat,
        @JsonProperty("keyName") String keyName,
        @JsonProperty("start") long start,
        @JsonProperty("end") long end,
        @JsonProperty("nodes") List<HostAddress> nodes)
{
    this.schemaName = requireNonNull(schemaName, "schemaName is null");
    this.tableName = requireNonNull(tableName, "tableName is null");
    this.keyDataFormat = requireNonNull(keyDataFormat, "keyDataFormat is null");
    this.valueDataFormat = requireNonNull(valueDataFormat, "valueDataFormat is null");
    this.keyName = keyName;
    this.nodes = ImmutableList.copyOf(requireNonNull(nodes, "nodes is null"));
    this.start = start;
    this.end = end;
    this.valueDataType = toRedisDataType(valueDataFormat);
    this.keyDataType = toRedisDataType(keyDataFormat);
}
 

@Test
public void testAddresses()
{
    // http split with default port
    PrometheusSplit httpSplit = new PrometheusSplit(URI.create("http://prometheus.com/prometheus"));
    assertEquals(httpSplit.getAddresses(), ImmutableList.of(HostAddress.fromString("prometheus.com")));
    assertEquals(httpSplit.isRemotelyAccessible(), true);

    // http split with custom port
    httpSplit = new PrometheusSplit(URI.create("http://prometheus.com:8080/prometheus"));
    assertEquals(httpSplit.getAddresses(), ImmutableList.of(HostAddress.fromParts("prometheus.com", 8080)));
    assertEquals(httpSplit.isRemotelyAccessible(), true);

    // http split with default port
    PrometheusSplit httpsSplit = new PrometheusSplit(URI.create("https://prometheus.com/prometheus"));
    assertEquals(httpsSplit.getAddresses(), ImmutableList.of(HostAddress.fromString("prometheus.com")));
    assertEquals(httpsSplit.isRemotelyAccessible(), true);

    // http split with custom port
    httpsSplit = new PrometheusSplit(URI.create("https://prometheus.com:8443/prometheus"));
    assertEquals(httpsSplit.getAddresses(), ImmutableList.of(HostAddress.fromParts("prometheus.com", 8443)));
    assertEquals(httpsSplit.isRemotelyAccessible(), true);
}
 

@JsonCreator
public IcebergSplit(
        @JsonProperty("path") String path,
        @JsonProperty("start") long start,
        @JsonProperty("length") long length,
        @JsonProperty("fileFormat") FileFormat fileFormat,
        @JsonProperty("addresses") List<HostAddress> addresses,
        @JsonProperty("partitionKeys") Map<Integer, String> partitionKeys)
{
    this.path = requireNonNull(path, "path is null");
    this.start = start;
    this.length = length;
    this.fileFormat = requireNonNull(fileFormat, "fileFormat is null");
    this.addresses = ImmutableList.copyOf(requireNonNull(addresses, "addresses is null"));
    this.partitionKeys = Collections.unmodifiableMap(requireNonNull(partitionKeys, "partitionKeys is null"));
}
 

@Override
public State getState(HostAddress hostAddress)
{
    for (MonitoringTask task : tasks.values()) {
        if (hostAddress.equals(fromUri(task.uri))) {
            if (!task.isFailed()) {
                return ALIVE;
            }

            Exception lastFailureException = task.getStats().getLastFailureException();
            if (lastFailureException instanceof ConnectException) {
                return GONE;
            }
            if (lastFailureException instanceof SocketTimeoutException) {
                // TODO: distinguish between process unresponsiveness (e.g GC pause) and host reboot
                return UNRESPONSIVE;
            }

            return UNKNOWN;
        }
    }

    return UNKNOWN;
}
 

private List<ConnectorSplit> getSplitsByTokenRange(CassandraTable table, String partitionId, Optional<Long> sessionSplitsPerNode)
{
    String schema = table.getTableHandle().getSchemaName();
    String tableName = table.getTableHandle().getTableName();
    String tokenExpression = table.getTokenExpression();

    ImmutableList.Builder<ConnectorSplit> builder = ImmutableList.builder();
    List<CassandraTokenSplitManager.TokenSplit> tokenSplits = tokenSplitMgr.getSplits(schema, tableName, sessionSplitsPerNode);
    for (CassandraTokenSplitManager.TokenSplit tokenSplit : tokenSplits) {
        String condition = buildTokenCondition(tokenExpression, tokenSplit.getStartToken(), tokenSplit.getEndToken());
        List<HostAddress> addresses = new HostAddressFactory().hostAddressNamesToHostAddressList(tokenSplit.getHosts());
        CassandraSplit split = new CassandraSplit(partitionId, condition, addresses);
        builder.add(split);
    }

    return builder.build();
}
 

@Test
public void testAddresses()
{
    // http split with default port
    ExampleSplit httpSplit = new ExampleSplit(URI.create("http://example.com/example"));
    assertEquals(httpSplit.getAddresses(), ImmutableList.of(HostAddress.fromString("example.com")));
    assertEquals(httpSplit.isRemotelyAccessible(), true);

    // http split with custom port
    httpSplit = new ExampleSplit(URI.create("http://example.com:8080/example"));
    assertEquals(httpSplit.getAddresses(), ImmutableList.of(HostAddress.fromParts("example.com", 8080)));
    assertEquals(httpSplit.isRemotelyAccessible(), true);

    // http split with default port
    ExampleSplit httpsSplit = new ExampleSplit(URI.create("https://example.com/example"));
    assertEquals(httpsSplit.getAddresses(), ImmutableList.of(HostAddress.fromString("example.com")));
    assertEquals(httpsSplit.isRemotelyAccessible(), true);

    // http split with custom port
    httpsSplit = new ExampleSplit(URI.create("https://example.com:8443/example"));
    assertEquals(httpsSplit.getAddresses(), ImmutableList.of(HostAddress.fromParts("example.com", 8443)));
    assertEquals(httpsSplit.isRemotelyAccessible(), true);
}
 

@JsonCreator
public PhoenixSplit(
        @JsonProperty("addresses") List<HostAddress> addresses,
        @JsonProperty("phoenixInputSplit") WrappedPhoenixInputSplit wrappedPhoenixInputSplit,
        @JsonProperty("constraint") TupleDomain<ColumnHandle> constraint)
{
    super(Optional.empty());
    this.addresses = requireNonNull(addresses, "addresses is null");
    this.phoenixInputSplit = requireNonNull(wrappedPhoenixInputSplit, "wrappedPhoenixInputSplit is null");
    this.constraint = requireNonNull(constraint, "constraint is null");
}
 

public RaptorSplit(
        UUID shardUuid,
        List<HostAddress> addresses,
        OptionalLong transactionId)
{
    this(ImmutableSet.of(shardUuid), OptionalInt.empty(), addresses, transactionId);
}
 

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableHandle table, SplitSchedulingStrategy splitSchedulingStrategy)
{
    List<HostAddress> localAddress = ImmutableList.of(nodeManager.getCurrentNode().getHostAndPort());
    ConnectorSplit split = new InformationSchemaSplit(localAddress);
    return new FixedSplitSource(ImmutableList.of(split));
}
 

private NodeMap createNodeMap(Optional<CatalogName> catalogName)
{
    Set<InternalNode> nodes = catalogName
            .map(nodeManager::getActiveConnectorNodes)
            .orElseGet(() -> nodeManager.getNodes(ACTIVE));

    Set<String> coordinatorNodeIds = nodeManager.getCoordinators().stream()
            .map(InternalNode::getNodeIdentifier)
            .collect(toImmutableSet());

    ImmutableSetMultimap.Builder<HostAddress, InternalNode> byHostAndPort = ImmutableSetMultimap.builder();
    ImmutableSetMultimap.Builder<InetAddress, InternalNode> byHost = ImmutableSetMultimap.builder();
    ImmutableSetMultimap.Builder<NetworkLocation, InternalNode> workersByNetworkPath = ImmutableSetMultimap.builder();
    for (InternalNode node : nodes) {
        if (includeCoordinator || !coordinatorNodeIds.contains(node.getNodeIdentifier())) {
            NetworkLocation location = networkTopology.locate(node.getHostAndPort());
            for (int i = 0; i <= location.getSegments().size(); i++) {
                workersByNetworkPath.put(location.subLocation(0, i), node);
            }
        }
        try {
            byHostAndPort.put(node.getHostAndPort(), node);

            InetAddress host = InetAddress.getByName(node.getInternalUri().getHost());
            byHost.put(host, node);
        }
        catch (UnknownHostException e) {
            if (inaccessibleNodeLogCache.getIfPresent(node) == null) {
                inaccessibleNodeLogCache.put(node, true);
                LOG.warn(e, "Unable to resolve host name for node: %s", node);
            }
        }
    }

    return new NodeMap(byHostAndPort.build(), byHost.build(), workersByNetworkPath.build(), coordinatorNodeIds);
}
 

private List<HostAddress> getSplitAddresses(PhoenixInputSplit split)
{
    try {
        return ImmutableList.of(HostAddress.fromString(split.getLocations()[0]));
    }
    catch (IOException | InterruptedException e) {
        if (e instanceof InterruptedException) {
            Thread.currentThread().interrupt();
        }
        throw new PrestoException(PHOENIX_INTERNAL_ERROR, "Exception when getting split addresses", e);
    }
}
 

@JsonCreator
public MemoryDataFragment(
        @JsonProperty("hostAddress") HostAddress hostAddress,
        @JsonProperty("rows") long rows)
{
    this.hostAddress = requireNonNull(hostAddress, "hostAddress is null");
    checkArgument(rows >= 0, "Rows number cannot be negative");
    this.rows = rows;
}
 

private NodeMap createNodeMap(Optional<CatalogName> catalogName)
{
    Set<InternalNode> nodes = catalogName
            .map(nodeManager::getActiveConnectorNodes)
            .orElseGet(() -> nodeManager.getNodes(ACTIVE));

    Set<String> coordinatorNodeIds = nodeManager.getCoordinators().stream()
            .map(InternalNode::getNodeIdentifier)
            .collect(toImmutableSet());

    ImmutableSetMultimap.Builder<HostAddress, InternalNode> byHostAndPort = ImmutableSetMultimap.builder();
    ImmutableSetMultimap.Builder<InetAddress, InternalNode> byHost = ImmutableSetMultimap.builder();
    for (InternalNode node : nodes) {
        try {
            byHostAndPort.put(node.getHostAndPort(), node);

            InetAddress host = InetAddress.getByName(node.getInternalUri().getHost());
            byHost.put(host, node);
        }
        catch (UnknownHostException e) {
            if (inaccessibleNodeLogCache.getIfPresent(node) == null) {
                inaccessibleNodeLogCache.put(node, true);
                LOG.warn(e, "Unable to resolve host name for node: %s", node);
            }
        }
    }

    return new NodeMap(byHostAndPort.build(), byHost.build(), ImmutableSetMultimap.of(), coordinatorNodeIds);
}
 

private static ImmutableSet<HostAddress> parseNodes(String nodes)
{
    Splitter splitter = Splitter.on(',').omitEmptyStrings().trimResults();
    return StreamSupport.stream(splitter.split(nodes).spliterator(), false)
            .map(KafkaConfig::toHostAddress)
            .collect(toImmutableSet());
}
 

public LocalFileRecordCursor(LocalFileTables localFileTables, List<LocalFileColumnHandle> columns, SchemaTableName tableName, HostAddress address, TupleDomain<LocalFileColumnHandle> predicate)
{
    this.columns = requireNonNull(columns, "columns is null");
    this.address = requireNonNull(address, "address is null");

    fieldToColumnIndex = new int[columns.size()];
    for (int i = 0; i < columns.size(); i++) {
        LocalFileColumnHandle columnHandle = columns.get(i);
        fieldToColumnIndex[i] = columnHandle.getOrdinalPosition();
    }
    this.includeServer = isThisServerIncluded(address, predicate, localFileTables.getTable(tableName));
    this.reader = includeServer ? getFilesReader(localFileTables, predicate, tableName) : null;
}
 

public HostAddress toHostAddress(String hostAddressName)
{
    HostAddress address = hostMap.get(hostAddressName);
    if (address == null) {
        address = HostAddress.fromString(hostAddressName);
        hostMap.put(hostAddressName, address);
    }
    return address;
}
 

void updateTaskStatus(TaskStatus newValue)
{
    // change to new value if old value is not changed and new value has a newer version
    AtomicBoolean taskMismatch = new AtomicBoolean();
    taskStatus.setIf(newValue, oldValue -> {
        // did the task instance id change
        if (!isNullOrEmpty(oldValue.getTaskInstanceId()) && !oldValue.getTaskInstanceId().equals(newValue.getTaskInstanceId())) {
            taskMismatch.set(true);
            return false;
        }

        if (oldValue.getState().isDone()) {
            // never update if the task has reached a terminal state
            return false;
        }
        if (newValue.getVersion() < oldValue.getVersion()) {
            // don't update to an older version (same version is ok)
            return false;
        }
        return true;
    });

    if (taskMismatch.get()) {
        // This will also set the task status to FAILED state directly.
        // Additionally, this will issue a DELETE for the task to the worker.
        // While sending the DELETE is not required, it is preferred because a task was created by the previous request.
        onFail.accept(new PrestoException(REMOTE_TASK_MISMATCH, format("%s (%s)", REMOTE_TASK_MISMATCH_ERROR, HostAddress.fromUri(getTaskStatus().getSelf()))));
    }
}
 

public InternalHiveBlock(long start, long end, List<HostAddress> addresses)
{
    checkArgument(start <= end, "block end cannot be before block start");
    this.start = start;
    this.end = end;
    this.addresses = ImmutableList.copyOf(addresses);
}
 

@Test
public void testSerialization()
{
    SystemSplit expected = new SystemSplit(HostAddress.fromParts("127.0.0.1", 0), TupleDomain.all());

    JsonCodec<SystemSplit> codec = jsonCodec(SystemSplit.class);
    SystemSplit actual = codec.fromJson(codec.toJson(expected));

    assertEquals(actual.getAddresses(), expected.getAddresses());
    assertEquals(actual.getConstraint(), expected.getConstraint());
}
 

@Test
public void testErrorCodes()
{
    assertEquals(new PageTooLargeException().getErrorCode(), PAGE_TOO_LARGE.toErrorCode());
    assertEquals(new PageTransportErrorException("").getErrorCode(), PAGE_TRANSPORT_ERROR.toErrorCode());
    assertEquals(new PageTransportTimeoutException(HostAddress.fromParts("127.0.0.1", 8080), "", null).getErrorCode(), PAGE_TRANSPORT_TIMEOUT.toErrorCode());
}
 

private RaptorSplit(
        Set<UUID> shardUuids,
        OptionalInt bucketNumber,
        List<HostAddress> addresses,
        OptionalLong transactionId)
{
    this.shardUuids = ImmutableSet.copyOf(requireNonNull(shardUuids, "shardUuid is null"));
    this.bucketNumber = requireNonNull(bucketNumber, "bucketNumber is null");
    this.addresses = ImmutableList.copyOf(requireNonNull(addresses, "addresses is null"));
    this.transactionId = requireNonNull(transactionId, "transactionId is null");
}
 

@Override
public NetworkLocation locate(HostAddress address)
{
    List<String> parts = new ArrayList<>(ImmutableList.copyOf(Splitter.on(".").split(address.getHostText())));
    Collections.reverse(parts);
    return new NetworkLocation(parts);
}
 

@JsonCreator
public TpchSplit(
        @JsonProperty("partNumber") int partNumber,
        @JsonProperty("totalParts") int totalParts,
        @JsonProperty("addresses") List<HostAddress> addresses)
{
    checkState(partNumber >= 0, "partNumber must be >= 0");
    checkState(totalParts >= 1, "totalParts must be >= 1");
    checkState(totalParts > partNumber, "totalParts must be > partNumber");

    this.partNumber = partNumber;
    this.totalParts = totalParts;
    this.addresses = ImmutableList.copyOf(requireNonNull(addresses, "addresses is null"));
}