Java Code Examples for org.apache.lucene.util.OpenBitSet#clear()

protected int specializePositiveCover(FDTree posCoverTree, OpenBitSet lhs, int rhs) {
	int newFDs = 0;
	List<OpenBitSet> specLhss = posCoverTree.getFdAndGeneralizations(lhs, rhs);
	for (OpenBitSet specLhs : specLhss) {
		posCoverTree.removeFunctionalDependency(specLhs, rhs);
		
		if (specLhs.cardinality() == posCoverTree.getMaxDepth())
			continue;
		
		for (int attr = this.numAttributes - 1; attr >= 0; attr--) { // TODO: Is iterating backwards a good or bad idea?
			if (!lhs.get(attr) && (attr != rhs)) {
				specLhs.set(attr);
				if (!posCoverTree.containsFdOrGeneralization(specLhs, rhs)) {
					posCoverTree.addFunctionalDependency(specLhs, rhs);
					newFDs++;					
				}
				specLhs.clear(attr);
			}
		}
	}
	return newFDs;
}
 

/**
	 * Checks, whether the dependent attribute ends in the current tree element.
	 * 
	 * @param rhs
	 *            the i'th dependent attribute.
	 * @return true, if the tree element does not have any children with the
	 *         same dependent attribute. false, otherwise.
	 */
/*	public boolean isLastNodeOf(int rhs) {
		if (!this.hasRhsAttribute(rhs))
			return false;
		
		// Check all children for the rhs
		for (int attr = 0; attr < this.maxAttributeNumber; attr++)
			if ((this.children[attr] != null) && (this.children[attr].hasRhsAttribute(rhs)))
				return false;
		
		return true;
	}
*/
	// FUDEBS
	protected void addOneSmallerGeneralizations(OpenBitSet currentLhs, int maxCurrentLhsAttribute, int rhs, FDTree tree) {
		for (int lhsAttribute = currentLhs.nextSetBit(0); lhsAttribute != maxCurrentLhsAttribute; lhsAttribute = currentLhs.nextSetBit(lhsAttribute + 1)) {
			currentLhs.clear(lhsAttribute);
			tree.addGeneralization(currentLhs, rhs);
			currentLhs.set(lhsAttribute);
		}
	}
 

@Test
public void testDeleteGeneralizations() {
	fdtree = new FDTree(4, -1);
	OpenBitSet lhs = new OpenBitSet();
	lhs.set(0);
	lhs.set(1);
	
	this.fdtree.addFunctionalDependency(lhs, 3);
	lhs.clear(1);
	lhs.set(2);
	this.fdtree.addFunctionalDependency(lhs, 3);
	
	//lhs.set(1);
	//this.fdtree.deleteGeneralizations(lhs, 3, 0);
	//assertTrue(this.fdtree.isEmpty());
}
 

public void addPrunedElements() {
	int numAttributes = this.numAttributes;
	
	OpenBitSet currentLhs = new OpenBitSet(numAttributes);
	
	if (this.getChildren() == null)
		return;
	
	for (int attr = 0; attr < this.numAttributes; attr++) {
		if (this.getChildren()[attr] != null) {
			currentLhs.set(attr);
			this.getChildren()[attr].addPrunedElements(currentLhs, attr, this);
			currentLhs.clear(attr);
		}
	}
}
 

protected void getFdAndGeneralizations(OpenBitSet lhs, int rhs, int currentLhsAttr, OpenBitSet currentLhs, List<OpenBitSet> foundLhs) {
	if (this.isFd(rhs))
		foundLhs.add(currentLhs.clone());

	if (this.children == null)
		return;
	
	while (currentLhsAttr >= 0) {
		int nextLhsAttr = lhs.nextSetBit(currentLhsAttr + 1);
		
		if ((this.children[currentLhsAttr] != null) && (this.children[currentLhsAttr].hasRhsAttribute(rhs))) {
			currentLhs.set(currentLhsAttr);
			this.children[currentLhsAttr].getFdAndGeneralizations(lhs, rhs, nextLhsAttr, currentLhs, foundLhs);
			currentLhs.clear(currentLhsAttr);
		}
		
		currentLhsAttr = nextLhsAttr;
	}
}
 

protected void filterGeneralizations(OpenBitSet lhs, int rhs, int currentLhsAttr, OpenBitSet currentLhs) {
	if (currentLhs.equals(lhs))
		return;
	
	this.rhsFds.clear(rhs);
	
	// Is the dependency already read and we have not yet found a generalization?
	if (currentLhsAttr < 0)
		return;
	
	if (this.children != null) {
		for (int nextLhsAttr = lhs.nextSetBit(currentLhsAttr); nextLhsAttr >= 0; nextLhsAttr = lhs.nextSetBit(nextLhsAttr + 1)) {
			if ((this.children[nextLhsAttr] != null) && (this.children[nextLhsAttr].hasRhsAttribute(rhs))) {
				currentLhs.set(nextLhsAttr);
				this.children[nextLhsAttr].filterGeneralizations(lhs, rhs, lhs.nextSetBit(nextLhsAttr + 1), currentLhs);
				currentLhs.clear(nextLhsAttr);
			}
		}
	}
}
 

protected void filterGeneralizations(OpenBitSet currentLhs, FDTree tree) {
	if (this.children != null) {		
		for (int attr = 0; attr < this.numAttributes; attr++) {
			if (this.children[attr] != null) {
				currentLhs.set(attr);
				this.children[attr].filterGeneralizations(currentLhs, tree);
				currentLhs.clear(attr);
			}
		}
	}
	for (int rhs = this.rhsFds.nextSetBit(0); rhs >= 0; rhs = this.rhsFds.nextSetBit(rhs + 1))
		tree.filterGeneralizations(currentLhs, rhs);
}
 

public void addFunctionalDependenciesInto(List<FunctionalDependency> functionalDependencies, OpenBitSet lhs, ObjectArrayList<ColumnIdentifier> columnIdentifiers, List<PositionListIndex> plis) {
	for (int rhs = this.rhsFds.nextSetBit(0); rhs >= 0; rhs = this.rhsFds.nextSetBit(rhs + 1)) {
		ColumnIdentifier[] columns = new ColumnIdentifier[(int) lhs.cardinality()];
		int j = 0;
		for (int i = lhs.nextSetBit(0); i >= 0; i = lhs.nextSetBit(i + 1)) {
			int columnId = plis.get(i).getAttribute(); // Here we translate the column i back to the real column i before the sorting
			columns[j++] = columnIdentifiers.get(columnId); 
		}
		
		ColumnCombination colCombination = new ColumnCombination(columns);
		int rhsId = plis.get(rhs).getAttribute(); // Here we translate the column rhs back to the real column rhs before the sorting
		FunctionalDependency fdResult = new FunctionalDependency(colCombination, columnIdentifiers.get(rhsId));
		functionalDependencies.add(fdResult);
	}

	if (this.getChildren() == null)
		return;
		
	for (int childAttr = 0; childAttr < this.numAttributes; childAttr++) {
		FDTreeElement element = this.getChildren()[childAttr];
		if (element != null) {
			lhs.set(childAttr);
			element.addFunctionalDependenciesInto(functionalDependencies, lhs, columnIdentifiers, plis);
			lhs.clear(childAttr);
		}
	}
}
 

public void grow(OpenBitSet lhs, FDTree fdTree) {
	// Add specializations of all nodes an mark them as isFD, but if specialization exists, then it is invalid and should not be marked; only add specializations of nodes not marked as isFD!
	OpenBitSet rhs = this.rhsAttributes;
	
	OpenBitSet invalidRhs = rhs.clone();
	invalidRhs.remove(this.rhsFds);
	
	// Add specializations that are not invalid
	if (invalidRhs.cardinality() > 0) {
		for (int extensionAttr = 0; extensionAttr < this.numAttributes; extensionAttr++) {
			if (lhs.get(extensionAttr) || rhs.get(extensionAttr))
				continue;
			
			lhs.set(extensionAttr);
			fdTree.addFunctionalDependencyIfNotInvalid(lhs, invalidRhs);
			lhs.clear(extensionAttr);
		}
	}
	
	// Traverse children and let them add their specializations
	if (this.children != null) {
		for (int childAttr = 0; childAttr < this.numAttributes; childAttr++) {
			FDTreeElement element = this.children[childAttr];
			if (element != null) {
				lhs.set(childAttr);
				element.grow(lhs, fdTree);
				lhs.clear(childAttr);
			}
		}
	}
}
 

public int addFunctionalDependenciesInto(FunctionalDependencyResultReceiver resultReceiver, OpenBitSet lhs, ObjectArrayList<ColumnIdentifier> columnIdentifiers, List<PositionListIndex> plis) throws CouldNotReceiveResultException, ColumnNameMismatchException {
	int numFDs = 0;
	for (int rhs = this.rhsFds.nextSetBit(0); rhs >= 0; rhs = this.rhsFds.nextSetBit(rhs + 1)) {
		ColumnIdentifier[] columns = new ColumnIdentifier[(int) lhs.cardinality()];
		int j = 0;
		for (int i = lhs.nextSetBit(0); i >= 0; i = lhs.nextSetBit(i + 1)) {
			int columnId = plis.get(i).getAttribute(); // Here we translate the column i back to the real column i before the sorting
			columns[j++] = columnIdentifiers.get(columnId); 
		}
		
		ColumnCombination colCombination = new ColumnCombination(columns);
		int rhsId = plis.get(rhs).getAttribute(); // Here we translate the column rhs back to the real column rhs before the sorting
		FunctionalDependency fdResult = new FunctionalDependency(colCombination, columnIdentifiers.get(rhsId));
		resultReceiver.receiveResult(fdResult);
		numFDs++;
	}

	if (this.getChildren() == null)
		return numFDs;
		
	for (int childAttr = 0; childAttr < this.numAttributes; childAttr++) {
		FDTreeElement element = this.getChildren()[childAttr];
		if (element != null) {
			lhs.set(childAttr);
			numFDs += element.addFunctionalDependenciesInto(resultReceiver, lhs, columnIdentifiers, plis);
			lhs.clear(childAttr);
		}
	}
	return numFDs;
}
 

public void asBitSets(ArrayList<OpenBitSet> bitsets, OpenBitSet bitset, int thisAttribute) {
	bitset.set(thisAttribute);
	
	if (this.end)
		bitsets.add(bitset.clone());
	
	for (int i = thisAttribute; i < this.children.length; i++)
		if (this.children[i] != null)
			this.children[i].asBitSets(bitsets, bitset, i);
	
	bitset.clear(thisAttribute);
}
 

protected int specializePositiveCover(OpenBitSet lhs, int rhs, FDList nonFds) {
	int numAttributes = this.posCover.getChildren().length;
	int newFDs = 0;
	List<OpenBitSet> specLhss;
	
	if (!(specLhss = this.posCover.getFdAndGeneralizations(lhs, rhs)).isEmpty()) { // TODO: May be "while" instead of "if"?
		for (OpenBitSet specLhs : specLhss) {
			this.posCover.removeFunctionalDependency(specLhs, rhs);
			
			if ((this.posCover.getMaxDepth() > 0) && (specLhs.cardinality() >= this.posCover.getMaxDepth()))
				continue;
			
			for (int attr = numAttributes - 1; attr >= 0; attr--) { // TODO: Is iterating backwards a good or bad idea?
				if (!lhs.get(attr) && (attr != rhs)) {
					specLhs.set(attr);
					if (!this.posCover.containsFdOrGeneralization(specLhs, rhs)) {
						this.posCover.addFunctionalDependency(specLhs, rhs);
						newFDs++;
						
						// If dynamic memory management is enabled, frequently check the memory consumption and trim the positive cover if it does not fit anymore
						this.memoryGuardian.memoryChanged(1);
						this.memoryGuardian.match(this.negCover, this.posCover, nonFds);
					}
					specLhs.clear(attr);
				}
			}
		}
	}
	return newFDs;
}
 

protected void fetchBitSets(Set<OpenBitSet> posCoverTree, FDTreeElement negCoverSubtree, OpenBitSet activePath) {
	posCoverTree.add(activePath.clone());
	
	if (negCoverSubtree.getChildren() != null) {
		for (int attr = 0; attr < this.numAttributes; attr++) {
			if (negCoverSubtree.getChildren()[attr] != null) {
				activePath.set(attr);
				this.fetchBitSets(posCoverTree, negCoverSubtree.getChildren()[attr], activePath);
				activePath.clear(attr);
			}
		}
	}
}
 

protected void specializePositiveCover(FDTree posCoverTree, OpenBitSet lhs, int rhs) {
	List<OpenBitSet> specLhss = null;
	specLhss = posCoverTree.getFdAndGeneralizations(lhs, rhs);
	for (OpenBitSet specLhs : specLhss) {
		posCoverTree.removeFunctionalDependency(specLhs, rhs);
		for (int attr = this.numAttributes - 1; attr >= 0; attr--) {
			if (!lhs.get(attr) && (attr != rhs)) {
				specLhs.set(attr);
				if (!posCoverTree.containsFdOrGeneralization(specLhs, rhs))
					posCoverTree.addFunctionalDependency(specLhs, rhs);
				specLhs.clear(attr);
			}
		}
	}
}
 

protected void calculatePositiveCover(FDTree posCoverTree, FDTreeElement negCoverSubtree, OpenBitSet activePath) {
	OpenBitSet fds = negCoverSubtree.getFds();
	for (int rhs = fds.nextSetBit(0); rhs >= 0; rhs = fds.nextSetBit(rhs + 1))
		this.specializePositiveCover(posCoverTree, activePath, rhs);
	
	if (negCoverSubtree.getChildren() != null) {
		for (int attr = 0; attr < this.numAttributes; attr++) {
			if (negCoverSubtree.getChildren()[attr] != null) {
				activePath.set(attr);
				this.calculatePositiveCover(posCoverTree, negCoverSubtree.getChildren()[attr], activePath);
				activePath.clear(attr);
			}
		}
	}
}
 

public void maximizeNegative(List<PositionListIndex> plis, int[][] invertedPlis, int numRecords) {
	// Maximizing negative cover is better than maximizing positive cover, because we do not need to check minimality; inversion does this automatically, i.e., generating a non-FD that creates a valid, non-minimal FD is not possible
	int numAttributes = plis.size();
	OpenBitSet currentLhs = new OpenBitSet(numAttributes);
	
	// Traverse the tree depth-first, left-first
	if (this.getChildren() != null) {
		for (int attr = 0; attr < numAttributes; attr++) {
			if (this.getChildren()[attr] != null) {
				currentLhs.set(attr);
				this.getChildren()[attr].maximizeNegativeRecursive(plis.get(attr), currentLhs, numAttributes, invertedPlis, this);
				currentLhs.clear(attr);
			}
		}
	}

	// Add invalid root FDs {} -/-> rhs to negative cover, because these are seeds for not yet discovered non-FDs
	this.addInvalidRootFDs(plis, numRecords); // TODO: These FDs make the search complex again :-(

	// On the way back, check all rhs-FDs that all their possible supersets are valid FDs; check with refines or, if available, with previously calculated plis
	//     which supersets to consider: add all attributes A with A notIn lhs and A notequal rhs; 
	//         for newLhs->rhs check that no FdOrSpecialization exists, because it is invalid then; this check might be slower than the FD check on high levels but faster on low levels in particular in the root! this check is faster on the negative cover, because we look for a non-FD
	for (int rhs = this.rhsFds.nextSetBit(0); rhs >= 0; rhs = this.rhsFds.nextSetBit(rhs + 1)) {
		OpenBitSet extensions = currentLhs.clone();
		extensions.flip(0, numAttributes);
		extensions.clear(rhs);
		
		// If a superset is a non-FD, mark this as not rhsFD, add the superset as a new node, filterGeneralizations() of the new node, call maximizeNegative() on the new supersets node
		//     if the superset node is in a right node of the tree, it will be checked anyways later; hence, only check supersets that are left or in the same tree path
		for (int extensionAttr = extensions.nextSetBit(0); extensionAttr >= 0; extensionAttr = extensions.nextSetBit(extensionAttr + 1)) {
			currentLhs.set(extensionAttr);
			
			if (this.containsFdOrSpecialization(currentLhs, rhs) || !plis.get(extensionAttr).refines(invertedPlis[rhs])) { // Otherwise, it must be false and a specialization is already contained; Only needed in root node, because we already filtered generalizations of other nodes and use a depth-first search that always removes generalizations when a new node comes in!
				this.rhsFds.clear(rhs);
				
				FDTreeElement newElement = this.addFunctionalDependency(currentLhs, rhs);
				//this.filterGeneralizations(currentLhs, rhs); // TODO: test effect
				newElement.maximizeNegativeRecursive(plis.get(extensionAttr), currentLhs, numAttributes, invertedPlis, this);
			}
			currentLhs.clear(extensionAttr);
		}
	}
}
 

public void growNegative(List<PositionListIndex> plis, int[][] invertedPlis, int numRecords) {
	int numAttributes = plis.size();
	
	OpenBitSet currentLhs = new OpenBitSet(numAttributes);
	
	// As root node, we need to check each rhs
	for (int rhs = 0; rhs < numAttributes; rhs++) {
		if (this.isFd(rhs)) // Is already invalid?
			continue;
		
		if (plis.get(rhs).isConstant(numRecords)) // Is {} -> rhs a valid FD
			continue;
		this.markFd(rhs);
		
		for (int attr = 0; attr < numAttributes; attr++) {
			if (attr == rhs)
				continue;
			
			if ((this.getChildren() != null) && (this.getChildren()[attr] != null) && this.getChildren()[attr].hasRhsAttribute(rhs)) // If there is a child with the current rhs, then currentLhs+attr->rhs must already be a known invalid fd
				continue;
				
			if (!plis.get(attr).refines(invertedPlis[rhs])) {
				// Add a new child representing the newly discovered invalid FD
				if (this.getChildren() == null)
					this.setChildren(new FDTreeElement[this.numAttributes]);
				if (this.getChildren()[attr] == null)
					this.getChildren()[attr] = new FDTreeElement(this.numAttributes);
				this.getChildren()[attr].addRhsAttribute(rhs);
				this.getChildren()[attr].markFd(rhs);
			}
		}
	}
	
	// Recursively call children
	if (this.getChildren() == null)
		return;
	for (int attr = 0; attr < numAttributes; attr++) {
		if (this.getChildren()[attr] != null) {
			currentLhs.set(attr);
			this.getChildren()[attr].growNegative(plis.get(attr), currentLhs, attr, plis, invertedPlis, this);
			currentLhs.clear(attr);
		}
	}
}
 

private void match(OpenBitSet equalAttrs, int[] t1, int[] t2) {
	equalAttrs.clear(0, t1.length);
	for (int i = 0; i < t1.length; i++)
		if (this.valueComparator.isEqual(t1[i], t2[i]))
			equalAttrs.set(i);
}
 

protected void maximizeNegativeRecursive(PositionListIndex currentPli, OpenBitSet currentLhs, int numAttributes, int[][] rhsPlis, FDTree invalidFds) {
	PositionListIndex[] childPlis = new PositionListIndex[numAttributes];
	
	// Traverse the tree depth-first, left-first; generate plis for children and pass them over; store the child plis locally to reuse them for the checking afterwards
	if (this.children != null) {
		for (int attr = 0; attr < numAttributes; attr++) {
			if (this.children[attr] != null) {
				childPlis[attr] = currentPli.intersect(rhsPlis[attr]);
				
				currentLhs.set(attr);
				this.children[attr].maximizeNegativeRecursive(childPlis[attr], currentLhs, numAttributes, rhsPlis, invalidFds);
				currentLhs.clear(attr);
			}
		}
	}
	
	// On the way back, check all rhs-FDs that all their possible supersets are valid FDs; check with refines or, if available, with previously calculated plis
	//     which supersets to consider: add all attributes A with A notIn lhs and A notequal rhs; 
	//         for newLhs->rhs check that no FdOrSpecialization exists, because it is invalid then; this check might be slower than the FD check on high levels but faster on low levels in particular in the root! this check is faster on the negative cover, because we look for a non-FD
	for (int rhs = this.rhsFds.nextSetBit(0); rhs >= 0; rhs = this.rhsFds.nextSetBit(rhs + 1)) {
		OpenBitSet extensions = currentLhs.clone();
		extensions.flip(0, numAttributes);
		extensions.clear(rhs);
		
		for (int extensionAttr = extensions.nextSetBit(0); extensionAttr >= 0; extensionAttr = extensions.nextSetBit(extensionAttr + 1)) {
			currentLhs.set(extensionAttr);
			if (childPlis[extensionAttr] == null)
				childPlis[extensionAttr] = currentPli.intersect(rhsPlis[extensionAttr]);
			
			// If a superset is a non-FD, mark this as not rhsFD, add the superset as a new node, filterGeneralizations() of the new node, call maximizeNegative() on the new supersets node
			//     if the superset node is in a right node of the tree, it will be checked anyways later; hence, only check supersets that are left or in the same tree path
			if (!childPlis[extensionAttr].refines(rhsPlis[rhs])) {
				this.rhsFds.clear(rhs);

				FDTreeElement newElement = invalidFds.addFunctionalDependency(currentLhs, rhs);
				//invalidFds.filterGeneralizations(currentLhs, rhs); // TODO: test effect
				newElement.maximizeNegativeRecursive(childPlis[extensionAttr], currentLhs, numAttributes, rhsPlis, invalidFds);
			}
			currentLhs.clear(extensionAttr);
		}
	}
}
 

public ValidationResult call() throws Exception {
	ValidationResult result = new ValidationResult();
	
	FDTreeElement element = this.elementLhsPair.getElement();
	OpenBitSet lhs = this.elementLhsPair.getLhs();
	OpenBitSet rhs = element.getFds();
	
	int rhsSize = (int) rhs.cardinality();
	if (rhsSize == 0)
		return result;
	result.validations = result.validations + rhsSize;
	
	if (Validator.this.level == 0) {
		// Check if rhs is unique
		for (int rhsAttr = rhs.nextSetBit(0); rhsAttr >= 0; rhsAttr = rhs.nextSetBit(rhsAttr + 1)) {
			if (!Validator.this.plis.get(rhsAttr).isConstant(Validator.this.numRecords)) {
				element.removeFd(rhsAttr);
				result.invalidFDs.add(new FD(lhs, rhsAttr));
			}
			result.intersections++;
		}
	}
	else if (Validator.this.level == 1) {
		// Check if lhs from plis refines rhs
		int lhsAttribute = lhs.nextSetBit(0);
		for (int rhsAttr = rhs.nextSetBit(0); rhsAttr >= 0; rhsAttr = rhs.nextSetBit(rhsAttr + 1)) {
			if (!Validator.this.plis.get(lhsAttribute).refines(Validator.this.compressedRecords, rhsAttr)) {
				element.removeFd(rhsAttr);
				result.invalidFDs.add(new FD(lhs, rhsAttr));
			}
			result.intersections++;
		}
	}
	else {
		// Check if lhs from plis plus remaining inverted plis refines rhs
		int firstLhsAttr = lhs.nextSetBit(0);
		
		lhs.clear(firstLhsAttr);
		OpenBitSet validRhs = Validator.this.plis.get(firstLhsAttr).refines(Validator.this.compressedRecords, lhs, rhs, result.comparisonSuggestions);
		lhs.set(firstLhsAttr);
		
		result.intersections++;
		
		rhs.andNot(validRhs); // Now contains all invalid FDs
		element.setFds(validRhs); // Sets the valid FDs in the FD tree
		
		for (int rhsAttr = rhs.nextSetBit(0); rhsAttr >= 0; rhsAttr = rhs.nextSetBit(rhsAttr + 1))
			result.invalidFDs.add(new FD(lhs, rhsAttr));
	}
	return result;
}