Java Code Examples for com.google.common.collect.Sets#intersection()

The following examples show how to use com.google.common.collect.Sets#intersection() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: _EventArgumentLinking.java    From tac-kbp-eal with MIT License 6 votes vote down vote up
@Value.Check
protected void check() {
  for (final TypeRoleFillerRealisSet eventFrame : eventFrames()) {
    final Set<TypeRoleFillerRealis> intersection =
        Sets.intersection(eventFrame.asSet(), incomplete());
    checkArgument(intersection.isEmpty(), "A TRFR cannot be both incomplete and linked: %s",
        intersection);
  }
  if (idsToEventFrames().isPresent()) {
    for (final String id : idsToEventFrames().get().keySet()) {
      checkArgument(!id.contains("-"), "Event frame IDs may not contain -s");
      checkArgument(!id.contains("\t"), "Event frame IDs may not contain tabs");
    }
    CollectionUtils.assertSameElementsOrIllegalArgument(eventFrames(),
        idsToEventFrames().get().values(), "Event frames did not match IDs",
        "Event frames in list", "Event frames in ID map");
  }
}
 
Example 2
Source File: RemoveRedundantTypeFilterRule.java    From fdb-record-layer with Apache License 2.0 6 votes vote down vote up
@Override
public void onMatch(@Nonnull PlannerRuleCall call) {
    LogicalTypeFilterExpression typeFilter = call.get(root);
    ExpressionRef<RelationalExpression> child = call.get(childMatcher);

    Set<String> childRecordTypes = RecordTypesProperty.evaluate(call.getContext(), child);
    Set<String> filterRecordTypes = Sets.newHashSet(typeFilter.getRecordTypes());
    if (filterRecordTypes.containsAll(childRecordTypes)) {
        // type filter is completely redundant, so remove it entirely
        call.yield(child);
    } else {
        // otherwise, keep a logical filter on record types which the child might produce and are included in the filter
        Set<String> unsatisfiedTypeFilters = Sets.intersection(childRecordTypes, filterRecordTypes);
        if (!unsatisfiedTypeFilters.equals(filterRecordTypes)) {
            // there were some unnecessary filters, so remove them
            call.yield(GroupExpressionRef.of(new LogicalTypeFilterExpression(unsatisfiedTypeFilters, child)));
        } // otherwise, nothing changes
    }
}
 
Example 3
Source File: DetailReport.java    From q with Apache License 2.0 6 votes vote down vote up
public void updateReport(Map<String, Set<String>> queryToIds, String q, Set<String> results, String testName, Map<String, String> titleIdToName, Map<ResultType, Integer> counters)
{
    if (!results.equals(queryToIds.get(q))) {
        String expectedTitles = getTitles(queryToIds.get(q), titleIdToName);
        if (results.size() > 0) {
            Set<String> intersection = Sets.intersection(queryToIds.get(q), results);
            Set<String> uniqExpected = new HashSet<String>(queryToIds.get(q));
            uniqExpected.removeAll(intersection);
            expectedTitles = getTitles(uniqExpected, titleIdToName);
            Set<String> uniqActual = new HashSet<String>(results);
            uniqActual.removeAll(intersection);
            String actualTitles = getTitles(uniqActual, titleIdToName);
            if (results.containsAll(queryToIds.get(q))) {
            	getItems().add(new DetailReportItem(testName, ResultType.supersetResultsFailed, q, expectedTitles, actualTitles));
                updateCounter(counters, ResultType.supersetResultsFailed);
            } else {
            	getItems().add(new DetailReportItem(testName, ResultType.differentResultsFailed, q, expectedTitles, actualTitles));
                updateCounter(counters, ResultType.differentResultsFailed);
            }
        } else {
        	getItems().add(new DetailReportItem(testName, ResultType.noResultsFailed, q, expectedTitles, NONE));
            updateCounter(counters, ResultType.noResultsFailed);
        }
    } else
        updateCounter(counters, ResultType.successQ);
}
 
Example 4
Source File: CollectionCompareTests.java    From java_in_examples with Apache License 2.0 6 votes vote down vote up
private static void testIntersect() {
    Collection<String> collection1 = Lists.newArrayList("a1", "a2", "a3", "a1");
    Collection<String> collection2 = Lists.newArrayList("a4", "a8", "a3", "a5");
    Set<String> set1 = Sets.newHashSet("a1", "a2", "a3", "a1");
    Set<String> set2 = Sets.newHashSet("a4", "a8", "a3", "a5");
    MutableSet<String> mutableSet1 = UnifiedSet.newSetWith("a1", "a2", "a3", "a1");
    MutableSet<String> mutableSet2 = UnifiedSet.newSetWith("a4", "a8", "a3", "a5");

    // Get all common elements in two collection
    Set<String> jdk = new HashSet<>(set1); // using JDK
    jdk.retainAll(set2);
    Set<String> guava = Sets.intersection(set1, set2); // using guava
    Collection<String> apache = CollectionUtils.intersection(collection1, collection2);  // using Apache
    Set<String> gs = mutableSet1.intersect(mutableSet2); // using GS
    System.out.println("intersect = " + jdk + ":" + guava + ":" + apache + ":" + gs); // print intersect = [a3]:[a3]:[a3]:[a3]
}
 
Example 5
Source File: AbstractPermissionManager.java    From onetwo with Apache License 2.0 6 votes vote down vote up
public void syncMenuToDatabase(MenuInfoParser<P> menuInfoParser){
//		Class<?> rootMenuClass = this.menuInfoParser.getMenuInfoable().getRootMenuClass();
//		Class<?> permClass = this.menuInfoParser.getMenuInfoable().getIPermissionClass();
		Optional<P> rootPermissionOpt = menuInfoParser.getRootMenu();
		if(!rootPermissionOpt.isPresent()){
			this.removeRootMenu(menuInfoParser);
			return ;
		}
		P rootPermission = rootPermissionOpt.get();
//		List<? extends IPermission> permList = (List<? extends IPermission>)this.baseEntityManager.findByProperties(permClass, "code:like", rootCode+"%");
//		Set<P> dbPermissions = findExistsPermission(rootPermission.getCode());
		Map<String, P> dbPermissionMap = findExistsPermission(rootPermission.getCode());
		Map<String, P> menuNodeMap = menuInfoParser.getPermissionMap();
		Set<P> memoryPermissions = new HashSet<>(menuNodeMap.values());

		Set<P> dbPermissions = new HashSet<P>(dbPermissionMap.values());
		Set<P> adds = Sets.difference(memoryPermissions, dbPermissions);
		Set<P> deletes = Sets.difference(dbPermissions, memoryPermissions);
		Set<P> intersections = Sets.intersection(memoryPermissions, dbPermissions);
		
		this.updatePermissions(rootPermission, dbPermissionMap, adds, deletes, intersections);

		logger.info("menu data has synchronized to database...");
	}
 
Example 6
Source File: FlavorDomain.java    From buck with Apache License 2.0 5 votes vote down vote up
public Optional<Flavor> getFlavor(Set<Flavor> flavors) {
  Sets.SetView<Flavor> match = Sets.intersection(translation.keySet(), flavors);
  if (match.size() > 1) {
    throw new FlavorDomainException(
        String.format("multiple \"%s\" flavors: %s", name, Joiner.on(", ").join(match)));
  }

  return Optional.ofNullable(Iterables.getFirst(match, null));
}
 
Example 7
Source File: StarQuery.java    From rya with Apache License 2.0 5 votes vote down vote up
public static StarQuery getConstrainedStarQuery(final StarQuery query, final BindingSet bs) {

        if(bs.size() == 0) {
            return query;
        }

        final Set<String> bindingNames = bs.getBindingNames();
        final Set<String> unCommonVarNames = query.getUnCommonVars();
        final Set<String> intersectVar = Sets.intersection(bindingNames, unCommonVarNames);


        if (!query.commonVarConstant()) {

            final Value v = bs.getValue(query.getCommonVarName());

            if (v != null) {
                query.commonVar.setValue(v);
            }
        }

        for(final String s: intersectVar) {
            try {
                query.nodeColumnCond[query.varPos.get(s)] = query.setValue(query.nodeColumnCond[query.varPos.get(s)], bs.getValue(s));
            } catch (final RyaTypeResolverException e) {
                e.printStackTrace();
            }
        }

        return query;
    }
 
Example 8
Source File: UserServiceImpl.java    From onboard with Apache License 2.0 5 votes vote down vote up
@Override
public List<User> filterProjectMembers(List<User> users, int projectId) {
    List<User> members = getUserByProjectId(projectId);
    SetView<User> intersection = Sets.intersection(new HashSet<User>(users), new HashSet<User>(members));
    List<User> legalUsers = Lists.newArrayList();
    for (User user : intersection)
        legalUsers.add(user);
    return legalUsers;
}
 
Example 9
Source File: RepoPalTimeSimilarityCalculator.java    From scava with Eclipse Public License 2.0 5 votes vote down vote up
@Override
public double calculateSimilarity(Artifact prj1, Artifact prj2) {
	double val1 = 0;
	double val2 = 0;
	
	Set<String> usersWhoStarRepo1 = prj1.getStarred().stream().map(z->z.getLogin()).collect(Collectors.toSet());
	Set<String> usersWhoStarRepo2 = prj2.getStarred().stream().map(z->z.getLogin()).collect(Collectors.toSet());
	
	Set<String> common = Sets.intersection(usersWhoStarRepo1, usersWhoStarRepo2);
	long timeStamp1;
	long timeStamp2 = 1;
	for (String userID: common) {
		Optional<String> dateStar1 = prj1.getStarred().stream().filter(z -> z.getLogin().equals(userID)).map(z -> z.getDatestamp()).findFirst();
		Optional<String> dateStar2 = prj2.getStarred().stream().filter(z -> z.getLogin().equals(userID)).map(z -> z.getDatestamp()).findFirst();
		try {
			if(dateStar1.isPresent() && dateStar2.isPresent()) {
				DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
					timeStamp1 = df.parse(dateStar1.get()).getTime();
					timeStamp2 = df.parse(dateStar2.get()).getTime();
					if(timeStamp1 == timeStamp2)
						timeStamp2+=1;
					val1+=(double)1/Math.abs(timeStamp1-timeStamp2);
			}
		} catch (ParseException e) {
			logger.error(e.getMessage());
		}
	}
	if(common.isEmpty())
		return 0;
	val2 = (double)1/common.size();	
	return val2*val1;				

}
 
Example 10
Source File: CoordinateSharer.java    From netcdf-java with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
public boolean nearlyEquals(RuntimeSmoosher that) {
  Sets.SetView<Long> common = Sets.intersection(this.coordSet, that.coordSet);
  int total = Math.min(this.runtime.getSize(), that.runtime.getSize());

  double nptsP = common.size() / (double) total;
  return (nptsP < smooshTolerence);
}
 
Example 11
Source File: SentryStore.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
private void dropDuplicatedRoleForImport(PersistenceManager pm, Set<String> existRoleNames,
    Set<String> importedRoleNames) throws Exception {
  Set<String> duplicatedRoleNames = Sets.intersection(existRoleNames, importedRoleNames);
  for (String droppedRoleName : duplicatedRoleNames) {
    dropSentryRoleCore(pm, droppedRoleName);
  }
}
 
Example 12
Source File: ExternalValidator.java    From scava with Eclipse Public License 2.0 5 votes vote down vote up
private float computeEntropy(int clusterID) {
	float ret = 0;
	Set<String> vector1 = new HashSet();
	for (Artifact id : clusters[clusterID].getArtifacts())
		vector1.add(id.getId());
	vector1.add(clusters[clusterID].getMostRepresentative().getId());
	Set<String> vector2 = new HashSet<String>();
	Set<String> common = null;
	int m = vector1.size();
	int n = 0;
	float tmp = 0, val = 0;
	for (int i = 0; i < clusters.length; i++) {
		for (Artifact string : Classes[i].getArtifacts())
			try {
				vector2.add(string.getId());
			} catch (Exception e) {
				System.out.println("i_ " + i + " " + string);
			}
		common = Sets.intersection(vector1, vector2);
		n = common.size();
		tmp = (float) n / m;
		if (tmp == 0)
			val = 0;
		else
			val = (float) (tmp * Math.log(tmp) * (-1));
		ret += val;
	}
	return ret;
}
 
Example 13
Source File: MessageDifferencer.java    From startup-os with Apache License 2.0 5 votes vote down vote up
private boolean compareRequestedFields(
    Message message1,
    Message message2,
    Set<FieldDescriptor> message1Fields,
    Set<FieldDescriptor> message2Fields,
    @Nullable Reporter reporter,
    List<SpecificField> stack) {
  if (scope == Scope.FULL) {
    if (messageFieldComparison == MessageFieldComparison.EQUIVALENT) {
      // We need to merge the field lists of both messages (i.e.
      // we are merely checking for a difference in field values,
      // rather than the addition or deletion of fields).
      Set<FieldDescriptor> fieldsUnion = Sets.union(message1Fields, message2Fields);
      return compareWithFieldsInternal(
          message1, message2, fieldsUnion, fieldsUnion, reporter, stack);
    } else {
      // Simple equality comparison, use the unaltered field lists.
      return compareWithFieldsInternal(
          message1, message2, message1Fields, message2Fields, reporter, stack);
    }
  } else {
    if (messageFieldComparison == MessageFieldComparison.EQUIVALENT) {
      // We use the list of fields for message1 for both messages when
      // comparing.  This way, extra fields in message2 are ignored,
      // and missing fields in message2 use their default value.
      return compareWithFieldsInternal(
          message1, message2, message1Fields, message1Fields, reporter, stack);
    } else {
      // We need to consider the full list of fields for message1
      // but only the intersection for message2.  This way, any fields
      // only present in message2 will be ignored, but any fields only
      // present in message1 will be marked as a difference.
      Set<FieldDescriptor> fieldsIntersection = Sets.intersection(message1Fields, message2Fields);
      return compareWithFieldsInternal(
          message1, message2, message1Fields, fieldsIntersection, reporter, stack);
    }
  }
}
 
Example 14
Source File: TypeManager.java    From binnavi with Apache License 2.0 5 votes vote down vote up
private Set<TypeSubstitution> getAffectedTypeSubstitutions(final Set<BaseType> baseTypes) {
  final Set<BaseType> affectedTypes =
      Sets.intersection(baseTypes, substitutionsByType.keySet());
  final HashSet<TypeSubstitution> typeSubstitutions = Sets.newHashSet();
  for (final BaseType baseType : affectedTypes) {
    typeSubstitutions.addAll(substitutionsByType.get(baseType));
  }
  return typeSubstitutions;
}
 
Example 15
Source File: SparqlFluoQueryBuilder.java    From rya with Apache License 2.0 5 votes vote down vote up
/**
 * Shifts the common variables between the two children to the left so
 * that Accumulo scans when performing the join are efficient.
 *
 * @param leftVars - The left child's variables. (not null)
 * @param rightVars - The right child's variables. (not null)
 * @return An object holding the left and right children's variable orders.
 */
private JoinVarOrders getJoinArgVarOrders(final Set<String> leftVars, final Set<String> rightVars) {
    checkNotNull(leftVars);
    checkNotNull(rightVars);

    // Find the common variables between the left and right children. The common vars
    // are stored in a list to ensure iteration order is always the same.
    final List<String> commonVars = new ArrayList<>( Sets.intersection(leftVars, rightVars) );

    // Push all of the common variables to the left for each child's vars.
    final List<String> leftVarOrder = leftShiftCommonVars(commonVars, leftVars);
    final List<String> rightVarOrder = leftShiftCommonVars(commonVars, rightVars);
    return new JoinVarOrders(new VariableOrder(leftVarOrder), new VariableOrder(rightVarOrder));
}
 
Example 16
Source File: DefaultContentValidator.java    From nexus-public with Eclipse Public License 1.0 4 votes vote down vote up
@Nonnull
@Override
public String determineContentType(boolean strictContentTypeValidation,
                                   Supplier<InputStream> contentSupplier,
                                   @Nullable MimeRulesSource mimeRulesSource,
                                   @Nullable String contentName,
                                   @Nullable String declaredContentType) throws IOException
{
  checkNotNull(contentSupplier);
  final String declaredBaseContentType = mediaTypeWithoutParameters(declaredContentType);

  final LinkedHashSet<String> contentDetectedMimeTypes = new LinkedHashSet<>();
  try (InputStream is = contentSupplier.get()) {
    contentDetectedMimeTypes.addAll(mimeSupport.detectMimeTypes(is, contentName));
  }
  adjustIfHtml(contentDetectedMimeTypes);
  log.debug("Mime support detects {} as {}", contentName, contentDetectedMimeTypes);

  if (strictContentTypeValidation && isUnknown(contentDetectedMimeTypes)) {
    throw new InvalidContentException("Content type could not be determined: " + contentName);
  }

  final LinkedHashSet<String> nameAssumedMimeTypes = new LinkedHashSet<>();
  if (contentName != null) {
    nameAssumedMimeTypes.addAll(
        mimeSupport.guessMimeTypesListFromPath(
            contentName,
            mimeRulesSource != null ? mimeRulesSource : MimeRulesSource.NOOP)
    );
    adjustIfHtml(nameAssumedMimeTypes);
    log.debug("Mime support assumes {} as {}", contentName, nameAssumedMimeTypes);

    if (!isUnknown(nameAssumedMimeTypes)) {
      Set<String> intersection = Sets.intersection(contentDetectedMimeTypes, nameAssumedMimeTypes);
      log.debug("content/name types intersection {}", intersection);
      if (strictContentTypeValidation && intersection.isEmpty()) {
        throw new InvalidContentException(
            String.format("Detected content type %s, but expected %s: %s",
                contentDetectedMimeTypes, nameAssumedMimeTypes, contentName)
        );
      }
    }
  }

  String finalContentType;
  if (!isUnknown(nameAssumedMimeTypes)) {
    // format implied type or known extension
    finalContentType = nameAssumedMimeTypes.iterator().next();
  }
  else if (!isUnknown(contentDetectedMimeTypes)) {
    // use the content based one
    finalContentType = contentDetectedMimeTypes.iterator().next();
  }
  else if (!Strings.isNullOrEmpty(declaredBaseContentType)) {
    // use the declared if declared at all
    finalContentType = declaredBaseContentType;
  }
  else {
    // give up
    finalContentType = ContentTypes.APPLICATION_OCTET_STREAM;
  }

  log.debug("Content {} declared as {}, determined as {}", contentName, declaredContentType, finalContentType);

  return finalContentType;
}
 
Example 17
Source File: InvalidListPruningDebugTest.java    From phoenix-tephra with Apache License 2.0 4 votes vote down vote up
@Test
public void testIdleRegions() throws Exception {
  ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
  try (PrintWriter out = new PrintWriter(outputStream)) {
    // Get the list of regions that have the lowest prune upper bounds for the latest record time
    Long latestRecordTime = compactedRegions.asMap().lastKey();
    SortedSet<InvalidListPruningDebugTool.RegionPruneInfoPretty> latestExpected =
      ImmutableSortedSet.copyOf(pruneUpperBoundAndStringComparator(), compactedRegions.get(latestRecordTime));
    pruningDebug.execute(new String[]{"idle-regions", "-1"}, out);
    out.flush();
    assertEquals(latestExpected, readOutputStream(outputStream));

    // Same command with explicit time
    outputStream.reset();
    pruningDebug.execute(new String[]{"idle-regions", "-1", String.valueOf(latestRecordTime)}, out);
    out.flush();
    assertEquals(latestExpected, readOutputStream(outputStream));

    // Same command with relative time
    outputStream.reset();
    pruningDebug.execute(new String[]{"idle-regions", "-1", "now-2s"}, out);
    out.flush();
    assertEquals(latestExpected, readOutputStream(outputStream));

    // Same command with reduced number of regions
    outputStream.reset();
    int limit = 2;
    pruningDebug.execute(new String[]{"idle-regions", String.valueOf(limit), String.valueOf(latestRecordTime)}, out);
    out.flush();
    Assert.assertEquals(GSON.toJson(subset(latestExpected, 0, limit)), readOutputStream(outputStream));

    // For a different time, this time only live regions that are compacted are returned
    outputStream.reset();
    Long secondLastRecordTime = Iterables.get(compactedRegions.keySet(), 1);
    Set<String> compactedRegionsTime =
      Sets.newTreeSet(Iterables.transform(compactedRegions.get(secondLastRecordTime), PRUNE_INFO_TO_STRING));
    Set<String> compactedRegionsLatest =
      Sets.newTreeSet(Iterables.transform(compactedRegions.get(latestRecordTime), PRUNE_INFO_TO_STRING));
    Set<String> liveExpected = new TreeSet<>(Sets.intersection(compactedRegionsTime, compactedRegionsLatest));
    pruningDebug.execute(new String[]{"idle-regions", "-1", String.valueOf(latestRecordTime - 1)}, out);
    out.flush();
    List<RegionPruneInfo> actual = GSON.fromJson(readOutputStream(outputStream), PRUNE_INFO_LIST_TYPE);
    Assert.assertEquals(liveExpected, Sets.newTreeSet(Iterables.transform(actual, PRUNE_INFO_TO_STRING)));
  }
}
 
Example 18
Source File: TransformUncorrelatedSubqueryToJoin.java    From presto with Apache License 2.0 4 votes vote down vote up
@Override
public Result apply(CorrelatedJoinNode correlatedJoinNode, Captures captures, Context context)
{
    // handle INNER and LEFT correlated join
    if (correlatedJoinNode.getType() == INNER || correlatedJoinNode.getType() == LEFT) {
        return Result.ofPlanNode(rewriteToJoin(
                correlatedJoinNode,
                correlatedJoinNode.getType().toJoinNodeType(),
                correlatedJoinNode.getFilter()));
    }

    checkState(
            correlatedJoinNode.getType() == RIGHT || correlatedJoinNode.getType() == FULL,
            "unexpected CorrelatedJoin type: " + correlatedJoinNode.getType());

    // handle RIGHT and FULL correlated join ON TRUE
    JoinNode.Type type;
    if (correlatedJoinNode.getType() == RIGHT) {
        type = JoinNode.Type.INNER;
    }
    else {
        type = JoinNode.Type.LEFT;
    }
    JoinNode joinNode = rewriteToJoin(correlatedJoinNode, type, TRUE_LITERAL);

    if (correlatedJoinNode.getFilter().equals(TRUE_LITERAL)) {
        return Result.ofPlanNode(joinNode);
    }

    // handle RIGHT correlated join on condition other than TRUE
    if (correlatedJoinNode.getType() == RIGHT) {
        Assignments.Builder assignments = Assignments.builder();
        assignments.putIdentities(Sets.intersection(
                ImmutableSet.copyOf(correlatedJoinNode.getSubquery().getOutputSymbols()),
                ImmutableSet.copyOf(correlatedJoinNode.getOutputSymbols())));
        for (Symbol inputSymbol : Sets.intersection(
                ImmutableSet.copyOf(correlatedJoinNode.getInput().getOutputSymbols()),
                ImmutableSet.copyOf(correlatedJoinNode.getOutputSymbols()))) {
            assignments.put(inputSymbol, new IfExpression(correlatedJoinNode.getFilter(), inputSymbol.toSymbolReference(), new NullLiteral()));
        }
        ProjectNode projectNode = new ProjectNode(
                context.getIdAllocator().getNextId(),
                joinNode,
                assignments.build());

        return Result.ofPlanNode(projectNode);
    }

    // no support for FULL correlated join on condition other than TRUE
    return Result.empty();
}
 
Example 19
Source File: AbstractCSQueue.java    From hadoop with Apache License 2.0 4 votes vote down vote up
synchronized boolean canAssignToThisQueue(Resource clusterResource,
    Set<String> nodeLabels, ResourceLimits currentResourceLimits,
    Resource nowRequired, Resource resourceCouldBeUnreserved) {
  // Get label of this queue can access, it's (nodeLabel AND queueLabel)
  Set<String> labelCanAccess;
  if (null == nodeLabels || nodeLabels.isEmpty()) {
    labelCanAccess = new HashSet<String>();
    // Any queue can always access any node without label
    labelCanAccess.add(RMNodeLabelsManager.NO_LABEL);
  } else {
    labelCanAccess = new HashSet<String>(
        accessibleLabels.contains(CommonNodeLabelsManager.ANY) ? nodeLabels
            : Sets.intersection(accessibleLabels, nodeLabels));
  }
  
  for (String label : labelCanAccess) {
    // New total resource = used + required
    Resource newTotalResource =
        Resources.add(queueUsage.getUsed(label), nowRequired);

    Resource currentLimitResource =
        getCurrentLimitResource(label, clusterResource, currentResourceLimits);

    if (Resources.greaterThan(resourceCalculator, clusterResource,
        newTotalResource, currentLimitResource)) {

      // if reservation continous looking enabled, check to see if could we
      // potentially use this node instead of a reserved node if the application
      // has reserved containers.
      // TODO, now only consider reservation cases when the node has no label
      if (this.reservationsContinueLooking
          && label.equals(RMNodeLabelsManager.NO_LABEL)
          && Resources.greaterThan(resourceCalculator, clusterResource,
          resourceCouldBeUnreserved, Resources.none())) {
        // resource-without-reserved = used - reserved
        Resource newTotalWithoutReservedResource =
            Resources.subtract(newTotalResource, resourceCouldBeUnreserved);

        // when total-used-without-reserved-resource < currentLimit, we still
        // have chance to allocate on this node by unreserving some containers
        if (Resources.lessThan(resourceCalculator, clusterResource,
            newTotalWithoutReservedResource, currentLimitResource)) {
          if (LOG.isDebugEnabled()) {
            LOG.debug("try to use reserved: " + getQueueName()
                + " usedResources: " + queueUsage.getUsed()
                + ", clusterResources: " + clusterResource
                + ", reservedResources: " + resourceCouldBeUnreserved
                + ", capacity-without-reserved: "
                + newTotalWithoutReservedResource + ", maxLimitCapacity: "
                + currentLimitResource);
          }
          currentResourceLimits.setAmountNeededUnreserve(Resources.subtract(newTotalResource,
              currentLimitResource));
          return true;
        }
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug(getQueueName()
            + "Check assign to queue, label=" + label
            + " usedResources: " + queueUsage.getUsed(label)
            + " clusterResources: " + clusterResource
            + " currentUsedCapacity "
            + Resources.divide(resourceCalculator, clusterResource,
            queueUsage.getUsed(label),
            labelManager.getResourceByLabel(label, clusterResource))
            + " max-capacity: "
            + queueCapacities.getAbsoluteMaximumCapacity(label)
            + ")");
      }
      return false;
    }
    return true;
  }
  
  // Actually, this will not happen, since labelCanAccess will be always
  // non-empty
  return false;
}
 
Example 20
Source File: InvalidListPruningDebugTest.java    From phoenix-tephra with Apache License 2.0 4 votes vote down vote up
@Test
public void testIdleRegions() throws Exception {
  ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
  try (PrintWriter out = new PrintWriter(outputStream)) {
    // Get the list of regions that have the lowest prune upper bounds for the latest record time
    Long latestRecordTime = compactedRegions.asMap().lastKey();
    SortedSet<InvalidListPruningDebugTool.RegionPruneInfoPretty> latestExpected =
      ImmutableSortedSet.copyOf(pruneUpperBoundAndStringComparator(), compactedRegions.get(latestRecordTime));
    pruningDebug.execute(new String[]{"idle-regions", "-1"}, out);
    out.flush();
    assertEquals(latestExpected, readOutputStream(outputStream));

    // Same command with explicit time
    outputStream.reset();
    pruningDebug.execute(new String[]{"idle-regions", "-1", String.valueOf(latestRecordTime)}, out);
    out.flush();
    assertEquals(latestExpected, readOutputStream(outputStream));

    // Same command with relative time
    outputStream.reset();
    pruningDebug.execute(new String[]{"idle-regions", "-1", "now-2s"}, out);
    out.flush();
    assertEquals(latestExpected, readOutputStream(outputStream));

    // Same command with reduced number of regions
    outputStream.reset();
    int limit = 2;
    pruningDebug.execute(new String[]{"idle-regions", String.valueOf(limit), String.valueOf(latestRecordTime)}, out);
    out.flush();
    Assert.assertEquals(GSON.toJson(subset(latestExpected, 0, limit)), readOutputStream(outputStream));

    // For a different time, this time only live regions that are compacted are returned
    outputStream.reset();
    Long secondLastRecordTime = Iterables.get(compactedRegions.keySet(), 1);
    Set<String> compactedRegionsTime =
      Sets.newTreeSet(Iterables.transform(compactedRegions.get(secondLastRecordTime), PRUNE_INFO_TO_STRING));
    Set<String> compactedRegionsLatest =
      Sets.newTreeSet(Iterables.transform(compactedRegions.get(latestRecordTime), PRUNE_INFO_TO_STRING));
    Set<String> liveExpected = new TreeSet<>(Sets.intersection(compactedRegionsTime, compactedRegionsLatest));
    pruningDebug.execute(new String[]{"idle-regions", "-1", String.valueOf(latestRecordTime - 1)}, out);
    out.flush();
    List<RegionPruneInfo> actual = GSON.fromJson(readOutputStream(outputStream), PRUNE_INFO_LIST_TYPE);
    Assert.assertEquals(liveExpected, Sets.newTreeSet(Iterables.transform(actual, PRUNE_INFO_TO_STRING)));
  }
}