Java Code Examples for com.google.common.hash.Hashing#consistentHash()

@Override
public QueuePartition getPartition(final FlowFileRecord flowFile, final QueuePartition[] partitions,  final QueuePartition localPartition) {
    final int hash = hash(flowFile);

    // The consistentHash method appears to always return a bucket of '1' if there are 2 possible buckets,
    // so in this case we will just use modulo division to avoid this. I suspect this is a bug with the Guava
    // implementation, but it's not clear at this point.
    final int index;
    if (partitions.length < 3) {
        index = Math.floorMod(hash, partitions.length);
    } else {
        index = Hashing.consistentHash(hash, partitions.length);
    }

    if (logger.isDebugEnabled()) {
        final List<String> partitionDescriptions = new ArrayList<>(partitions.length);
        for (final QueuePartition partition : partitions) {
            partitionDescriptions.add(partition.getSwapPartitionName());
        }

        logger.debug("Assigning Partition {} to {} based on {}", index, flowFile.getAttribute(CoreAttributes.UUID.key()), partitionDescriptions);
    }

    return partitions[index];
}
 

@Override
public RouterInfo route(Context context, InboundPackage inboundPackage) {
    List<RouterInfo> routeList =  getRouterInfoList(context,inboundPackage);

    if(routeList==null
    || 0 == routeList.size()){
        return null;
    }
    int idx = 0;
    RouteType routeType = getRouteType();
    switch (routeType){
        case RANDOM_ROUTE:{
            idx = ThreadLocalRandom.current().nextInt(routeList.size());
            break;
        }
        case CONSISTENT_HASH_ROUTE:{
            idx = Hashing.consistentHash(context.getRouteBasis(), routeList.size());
            break;
        }
        default:{
            // to use the first one.
            break;
        }
    }
    RouterInfo routerInfo = routeList.get(idx);

    context.setRouterInfo(routerInfo);

    logger.info("caseid {} get route info {}:{}", context.getCaseId(),routerInfo.getHost(),routerInfo.getPort());

    return routerInfo;
}
 

@Override
public Server choose(Object key) {
	
	//使用一致性哈希进行分发
	//TODO 此处使用了GUAVA中简单的一致性哈希算法选择服务，但这里存在性能缺陷：当reachableServers中间某一个服务节点失效了
	//那么后续节点的一致性哈希结果将会不匹配，后续需要使用更完善的哈希环 加上 虚拟节点 的形式解决本问题
	List<Server> reachableServers = getLoadBalancer().getReachableServers();
	if(reachableServers != null && reachableServers.size() != 0){
		int serverSeq = Hashing.consistentHash(Thread.currentThread().getId(), reachableServers.size());
		return reachableServers.get(serverSeq);
	} else {
		return super.choose(key);
	}
}
 

@Test
public void checkUniformDistribution() {
    final long samples = 100000L;
    final int rounds = 5;
    final double percentMarginOfError = 0.5;
    final int numKeys = 100;
    final Random random = new Random();

    for (int round = 0; round < rounds; round++) {
        logger.info(String.format("checkUniformDistribution - round %s: %d samples", round + 1, samples));

        for (final HashFunction hash: new HashFunction[]{md5(), murmur3_128(), sipHash24(), sha256()}) {
            long sum = 0L;
            final long initialTime = System.currentTimeMillis();
            for (Integer counter = 0; counter < samples; counter++) {
                final int chosen = (int) (random.nextFloat() * (numKeys - Float.MIN_VALUE));
                sum += Hashing.consistentHash(hash.hashInt(chosen), (int) numBackends);
            }

            final long finishTime = System.currentTimeMillis();

            final double result = (numBackends * (numBackends - 1) / 2.0) * (samples / numBackends);

            logger.info(String.format("-> checkUniformDistribution (%s): Time spent (ms): %d. NonUniformDistRatio (smaller is better): %.4f%%",
                    hash, finishTime - initialTime, Math.abs(100.0 * (result-sum) / result)));

            final double topLimit = sum * (1.0 + percentMarginOfError);
            final double bottomLimit = sum * (1.0 - percentMarginOfError);

            try {
                assertThat(result).isGreaterThanOrEqualTo(bottomLimit).isLessThanOrEqualTo(topLimit);
            } catch (AssertionError e) {
                logger.error("Error when testing " + hash);
                throw e;
            }
        }
    }
}
 

@Override
public Integer calculate(String columnValue)  {
    try
    {
        long targetTime = formatter.get().parse(
                columnValue).getTime();
        int targetPartition = (int) ((targetTime - beginDate) / partionTime);
        int innerIndex =  Hashing.consistentHash(targetTime,intGroupPartionSize);
        return targetPartition * intGroupPartionSize + innerIndex;

    } catch (ParseException e)
    {
        throw new IllegalArgumentException(new StringBuilder().append("columnValue:").append(columnValue).append(" Please check if the format satisfied.").toString(),e);
    }
}
 

@Override
public int calculateIndex(String columnValue) {
  long targetTime = formatter.parse(columnValue).get(ChronoField.DAY_OF_YEAR);
  int targetPartition = (int) ((targetTime - beginDate) / partionDay);
  int innerIndex = Hashing.consistentHash(targetTime, groupPartionSize);
  return targetPartition * groupPartionSize + innerIndex;
}
 

public int getNodeIndex(int numNodes, String key)
{
  HashFunction hf = Hashing.md5();
  HashCode hc = hf.hashString(key, Charsets.UTF_8);
  int initialNodeIndex = Hashing.consistentHash(hc, numNodes);
  int finalNodeIndex = initialNodeIndex;
  if (hc.asInt() % 2 == 0) {
    finalNodeIndex = getNextRunningNodeIndex(initialNodeIndex);
  }
  else {
    finalNodeIndex = getPreviousRunningNodeIndex(initialNodeIndex);
  }

  return finalNodeIndex;
}
 

@Override
public Endpoint selectNow(ClientRequestContext ctx) {

    final List<Endpoint> endpoints = group().endpoints();
    if (endpoints.isEmpty()) {
        return null;
    }

    final long key = requestContextHasher.applyAsLong(ctx);
    final int nearest = Hashing.consistentHash(key, endpoints.size());
    return endpoints.get(nearest);
}
 

public static int hash(long str,int size)
{
return     Hashing.consistentHash(str,size)      ;
}
 

public int hashToBucket(String str, int numBuckets) {
    return Hashing.consistentHash(hash.hashUnencodedChars(str), numBuckets);
}
 

public int hashToBucket(UUID uuid, int numBuckets) {
    return Hashing.consistentHash(
            Hashing.combineOrdered(Arrays.asList(hash.hashLong(uuid.getMostSignificantBits()), hash.hashLong(uuid.getLeastSignificantBits()))),
            numBuckets);
}
 

public int hashToBucket(byte[] array, int numBuckets) {
    return Hashing.consistentHash(hash.hashBytes(array), numBuckets);
}
 

public int hashToBucket(BufferView bufferView, int numBuckets) {
    HashBuilder builder = newBuilder();
    bufferView.collect(builder::put);
    return Hashing.consistentHash(builder.getHashCode(), numBuckets);
}
 

/**
 * Deterministic function that returns a bucket id based on the resource's roid.
 * NB: At the moment, nothing depends on this being deterministic, so we have the ability to
 * change the number of buckets and utilize a random distribution once we do.
 */
private static long getBucketIdFromEppResource(Key<? extends EppResource> resourceKey) {
  int numBuckets = getEppResourceIndexBucketCount();
  // IDs can't be 0, so add 1 to the hash.
  return Hashing.consistentHash(resourceKey.getName().hashCode(), numBuckets) + 1L;
}
 

/**
 * Returns the lock index for a given refreshItem.
 *
 * <p>We hash the second level domain domain for all records, to group in-balliwick hosts (the
 * only ones we refresh DNS for) with their superordinate domains. We use consistent hashing to
 * determine the lock index because it gives us [0,N) bucketing properties out of the box, then
 * add 1 to make indexes within [1,N].
 */
private int getLockIndex(String tld, int numPublishLocks, RefreshItem refreshItem) {
  String domain = getSecondLevelDomain(refreshItem.name(), tld);
  return Hashing.consistentHash(hashFunction.hashString(domain, UTF_8), numPublishLocks) + 1;
}
 

/**
 * Locate the bucket number given the value, the funnel and the total buckets.
 *
 * Assigns to {@code hashCode} a "bucket" in the range {@code [0, buckets)}, in a uniform manner that minimizes the
 * need for remapping as {@code buckets} grows. That is, {@code consistentHash(h, n)} equals:
 *
 * <ul> <li>{@code n - 1}, with approximate probability {@code 1/n} <li>{@code consistentHash(h, n - 1)}, otherwise
 * (probability {@code 1 - 1/n}) </ul>
 *
 * <p>See the <a href="http://en.wikipedia.org/wiki/Consistent_hashing">wikipedia article on consistent hashing</a>
 * for more information.
 *
 * <p>See <a href="http://arxiv.org/pdf/1406.2294v1.pdf">this paper</a> for more details on the algorithm</p>
 *
 *
 * Note that after testing, increasing buckets does NOT yield the expected results.  You will need an algorithm
 * that manually walks a tree.  See
 *
 */
public int getBucket( T value ) {

    final HashCode hashCode = HASHER.hashObject( value, funnel );

    int owningIndex = Hashing.consistentHash( hashCode, totalBuckets );

    return owningIndex;
}