org.apache.lucene.util.OpenBitSet Java Examples

private void fetchNonFdsWindowingOverRecordsProgressive(Set<OpenBitSet> negCover, int[][] compressedRecords) {
	System.out.println("\tMoving window over records ...");
	int numRecords = compressedRecords.length;
	int currentWindowDistance = 1;
	int newNonFDs = 0;
	do {
		newNonFDs = 0;
		
		for (int recordId = 0; recordId < numRecords; recordId++) {
			int partnerRecordId = recordId + currentWindowDistance; 
			
			if (partnerRecordId >= numRecords)
				continue;
				
			if (negCover.add(this.getViolatedFds(compressedRecords[recordId], compressedRecords[partnerRecordId])))
				newNonFDs++;
		}
		
		currentWindowDistance++;
	}
	while (newNonFDs > this.progressiveThreshold);
}
 

/**
	 * Checks, whether the dependent attribute ends in the current tree element.
	 * 
	 * @param rhs
	 *            the i'th dependent attribute.
	 * @return true, if the tree element does not have any children with the
	 *         same dependent attribute. false, otherwise.
	 */
/*	public boolean isLastNodeOf(int rhs) {
		if (!this.hasRhsAttribute(rhs))
			return false;
		
		// Check all children for the rhs
		for (int attr = 0; attr < this.maxAttributeNumber; attr++)
			if ((this.children[attr] != null) && (this.children[attr].hasRhsAttribute(rhs)))
				return false;
		
		return true;
	}
*/
	// FUDEBS
	protected void addOneSmallerGeneralizations(OpenBitSet currentLhs, int maxCurrentLhsAttribute, int rhs, FDTree tree) {
		for (int lhsAttribute = currentLhs.nextSetBit(0); lhsAttribute != maxCurrentLhsAttribute; lhsAttribute = currentLhs.nextSetBit(lhsAttribute + 1)) {
			currentLhs.clear(lhsAttribute);
			tree.addGeneralization(currentLhs, rhs);
			currentLhs.set(lhsAttribute);
		}
	}
 

protected boolean containsFdOrGeneralization(OpenBitSet lhs, int rhs, int currentLhsAttr) {
	if (this.isFd(rhs))
		return true;

	// Is the dependency already read and we have not yet found a generalization?
	if (currentLhsAttr < 0)
		return false;
	
	int nextLhsAttr = lhs.nextSetBit(currentLhsAttr + 1);
	
	if ((this.children != null) && (this.children[currentLhsAttr] != null) && (this.children[currentLhsAttr].hasRhsAttribute(rhs)))
		if (this.children[currentLhsAttr].containsFdOrGeneralization(lhs, rhs, nextLhsAttr))
			return true;
	
	return this.containsFdOrGeneralization(lhs, rhs, nextLhsAttr);
}
 

public void getLevel(int level, int currentLevel, OpenBitSet currentLhs, List<FDTreeElementLhsPair> result) {
	if (level == currentLevel) {
		result.add(new FDTreeElementLhsPair(this, currentLhs.clone()));
	}
	else {
		currentLevel++;
		if (this.children == null)
			return;
		
		for (int child = 0; child < this.numAttributes; child++) {
			if (this.children[child] == null)
				continue;
			
			currentLhs.set(child);
			this.children[child].getLevel(level, currentLevel, currentLhs, result);
			currentLhs.clear(child);
		}
	}
}
 

public boolean skipTo(int i) throws IOException
{
    if (!in.skipTo(i))
    {
        return false;
    }

    OpenBitSet deletedDocuments = getDeletedDocuments();
    while (deletedDocuments.get(in.doc()))
    {
        if (!in.next())
        {
            return false;
        }
    }
    return true;
}
 

public FDTreeElement addGeneralization(OpenBitSet lhs, OpenBitSet rhs) {
	FDTreeElement currentNode = this;
	currentNode.addRhsAttributes(rhs);

	boolean newElement = false;
	for (int i = lhs.nextSetBit(0); i >= 0; i = lhs.nextSetBit(i + 1)) {
		if (currentNode.getChildren() == null) {
			currentNode.setChildren(new FDTreeElement[this.numAttributes]);
			currentNode.getChildren()[i] = new FDTreeElement(this.numAttributes);
			newElement = true;
		}
		else if (currentNode.getChildren()[i] == null) {
			currentNode.getChildren()[i] = new FDTreeElement(this.numAttributes);
			newElement = true;
		}
			
		currentNode = currentNode.getChildren()[i];
		currentNode.addRhsAttributes(rhs);
	}
	
	if (newElement)
		return currentNode;
	return null;
}
 

protected void filterGeneralizations(OpenBitSet lhs, int rhs, int currentLhsAttr, OpenBitSet currentLhs) {
	if (currentLhs.equals(lhs))
		return;
	
	this.rhsFds.clear(rhs);
	
	// Is the dependency already read and we have not yet found a generalization?
	if (currentLhsAttr < 0)
		return;
	
	if (this.children != null) {
		for (int nextLhsAttr = lhs.nextSetBit(currentLhsAttr); nextLhsAttr >= 0; nextLhsAttr = lhs.nextSetBit(nextLhsAttr + 1)) {
			if ((this.children[nextLhsAttr] != null) && (this.children[nextLhsAttr].hasRhsAttribute(rhs))) {
				currentLhs.set(nextLhsAttr);
				this.children[nextLhsAttr].filterGeneralizations(lhs, rhs, lhs.nextSetBit(nextLhsAttr + 1), currentLhs);
				currentLhs.clear(nextLhsAttr);
			}
		}
	}
}
 

public void generalize() {
	int maxLevel = this.numAttributes;
	
	// Build an index level->nodes for the top-down, level-wise traversal
	Int2ObjectOpenHashMap<ArrayList<ElementLhsPair>> level2elements = new Int2ObjectOpenHashMap<>(maxLevel);
	for (int level = 0; level < maxLevel; level++)
		level2elements.put(level, new ArrayList<ElementLhsPair>());
	this.addToIndex(level2elements, 0, new OpenBitSet(this.numAttributes));
	
	// Traverse the levels top-down and add all direct generalizations
	for (int level = maxLevel - 1; level >= 0; level--) {
		for (ElementLhsPair pair : level2elements.get(level)) {
			// Remove isFDs, because we will mark valid FDs later on
			pair.element.removeAllFds();
			
			// Generate and add generalizations
			for (int lhsAttr = pair.lhs.nextSetBit(0); lhsAttr >= 0; lhsAttr = pair.lhs.nextSetBit(lhsAttr + 1)) {
				pair.lhs.clear(lhsAttr);
				FDTreeElement generalization = this.addGeneralization(pair.lhs, pair.element.getRhsAttributes());
				if (generalization != null)
					level2elements.get(level - 1).add(new ElementLhsPair(generalization, pair.lhs.clone()));
				pair.lhs.set(lhsAttr);
			}
		}
	}
}
 

private void fetchNonFdsWindowingOverClusters(Set<OpenBitSet> negCover, int[][] compressedRecords, List<PositionListIndex> plis) {
	System.out.println("\tMoving window over small clusters ...");
	for (PositionListIndex pli : plis) {
		boolean selectSmallClustersOnly = pli.getClusters().size() < this.attributeThreshold; 	// If there are too few clusters, then the clusters are large and we have already executed sufficient comparisons between the records of these clusters
		
		for (IntArrayList cluster : pli.getClusters()) {
			if (selectSmallClustersOnly && (cluster.size() > this.windowSize))					// But if the current cluster is very small, we should still use it for comparisons (the other cluster(s) must be very large)
				continue;
			
			for (int recordIndex = 0; recordIndex < cluster.size(); recordIndex++) {
				int recordId = cluster.getInt(recordIndex);
				
				for (int partnerRecordIndex = recordIndex + 1; partnerRecordIndex < Math.min(recordIndex + this.windowSize, cluster.size()); partnerRecordIndex++) {
					int partnerRecordId = cluster.getInt(partnerRecordIndex);
					
					negCover.add(this.getViolatedFds(compressedRecords[recordId], compressedRecords[partnerRecordId]));
				}
			}
		}
	}
}
 

private void fetchNonFdsFromClustersTopsAndBottoms(Set<OpenBitSet> negCover, int[][] compressedRecords, List<PositionListIndex> plis) {
	System.out.println("\tComparing window on clusters tops and bottoms ...");
	for (PositionListIndex pli : plis) {
		for (IntArrayList cluster : pli.getClusters()) {
			if (cluster.size() < this.windowSize)
				continue;
			
			for (int recordIndex = 0; recordIndex < this.windowSize; recordIndex++) {
				int recordId = cluster.getInt(recordIndex);
				
				for (int partnerRecordIndex = cluster.size() - 1; partnerRecordIndex > cluster.size() - this.windowSize; partnerRecordIndex--) {
					int partnerRecordId = cluster.getInt(partnerRecordIndex);
					
					if (recordId == partnerRecordId)
						continue;
					
					negCover.add(this.getViolatedFds(compressedRecords[recordId], compressedRecords[partnerRecordId]));
				}
			}
		}
	}
}
 

public FDTreeElement addGeneralization(OpenBitSet lhs, int rhs) {
	FDTreeElement currentNode = this;
	currentNode.addRhsAttribute(rhs);

	boolean newElement = false;
	for (int i = lhs.nextSetBit(0); i >= 0; i = lhs.nextSetBit(i + 1)) {
		if (currentNode.getChildren() == null) {
			currentNode.setChildren(new FDTreeElement[this.numAttributes]);
			currentNode.getChildren()[i] = new FDTreeElement(this.numAttributes);
			newElement = true;
		}
		else if (currentNode.getChildren()[i] == null) {
			currentNode.getChildren()[i] = new FDTreeElement(this.numAttributes);
			newElement = true;
		}
		
		currentNode = currentNode.getChildren()[i];
		currentNode.addRhsAttribute(rhs);
	}
	
	if (newElement)
		return currentNode;
	return null;
}
 

protected int specializePositiveCover(FDTree posCoverTree, OpenBitSet lhs, int rhs) {
	int newFDs = 0;
	List<OpenBitSet> specLhss = posCoverTree.getFdAndGeneralizations(lhs, rhs);
	for (OpenBitSet specLhs : specLhss) {
		posCoverTree.removeFunctionalDependency(specLhs, rhs);
		
		if (specLhs.cardinality() == posCoverTree.getMaxDepth())
			continue;
		
		for (int attr = this.numAttributes - 1; attr >= 0; attr--) { // TODO: Is iterating backwards a good or bad idea?
			if (!lhs.get(attr) && (attr != rhs)) {
				specLhs.set(attr);
				if (!posCoverTree.containsFdOrGeneralization(specLhs, rhs)) {
					posCoverTree.addFunctionalDependency(specLhs, rhs);
					newFDs++;					
				}
				specLhs.clear(attr);
			}
		}
	}
	return newFDs;
}
 

protected void getLhsAndGeneralizations(OpenBitSet lhs, int currentLhsAttr, OpenBitSet currentLhs, List<OpenBitSet> foundLhs) {
	if (this.children == null) {
		foundLhs.add(currentLhs.clone());
		return;
	}
	
	while (currentLhsAttr >= 0) {
		int nextLhsAttr = lhs.nextSetBit(currentLhsAttr + 1);
		
		if (this.children[currentLhsAttr] != null) {
			currentLhs.set(currentLhsAttr);
			this.children[currentLhsAttr].getLhsAndGeneralizations(lhs, nextLhsAttr, currentLhs, foundLhs);
			currentLhs.clear(currentLhsAttr);
		}
		
		currentLhsAttr = nextLhsAttr;
	}
}
 

protected ClusterIdentifier buildClusterIdentifier(int recordId, int[][] invertedPlis, OpenBitSet lhs, int lhsSize) { 
	int[] cluster = new int[lhsSize];
	
	int index = 0;
	for (int lhsAttr = lhs.nextSetBit(0); lhsAttr >= 0; lhsAttr = lhs.nextSetBit(lhsAttr + 1)) {
		int clusterId = invertedPlis[lhsAttr][recordId];
		
		if (clusterId < 0)
			return null;
		
		cluster[index] = clusterId;
		index++;
	}
	
	return new ClusterIdentifier(cluster);
}
 

@Test
public void testDeleteGeneralizations() {
	fdtree = new FDTree(4, -1);
	OpenBitSet lhs = new OpenBitSet();
	lhs.set(0);
	lhs.set(1);
	
	this.fdtree.addFunctionalDependency(lhs, 3);
	lhs.clear(1);
	lhs.set(2);
	this.fdtree.addFunctionalDependency(lhs, 3);
	
	//lhs.set(1);
	//this.fdtree.deleteGeneralizations(lhs, 3, 0);
	//assertTrue(this.fdtree.isEmpty());
}
 

public void removeLhs(OpenBitSet lhs) {
	LhsTrieElement[] path = new LhsTrieElement[(int)lhs.cardinality()];
	int currentPathIndex = 0;
	
	LhsTrieElement currentNode = this;
	path[currentPathIndex] = currentNode;
	currentPathIndex++;
	
	for (int i = lhs.nextSetBit(0); i >= 0; i = lhs.nextSetBit(i + 1)) {
		currentNode = currentNode.getChildren()[i];
		path[currentPathIndex] = currentNode;
		currentPathIndex++;
	}
	
	for (int i = path.length - 1; i >= 0; i --) {
		path[i].removeChild(i);
		if (path[i].getChildren() != null)
			break;
	}
}
 

public FDTreeElement addFunctionalDependency(OpenBitSet lhs, int rhs) {
	FDTreeElement currentNode = this;
	currentNode.addRhsAttribute(rhs);

	int lhsLength = 0;
	for (int i = lhs.nextSetBit(0); i >= 0; i = lhs.nextSetBit(i + 1)) {
		lhsLength++;
		
		if (currentNode.getChildren() == null) {
			currentNode.setChildren(new FDTreeElement[this.numAttributes]);
			currentNode.getChildren()[i] = new FDTreeElement(this.numAttributes);
		}
		else if (currentNode.getChildren()[i] == null) {
			currentNode.getChildren()[i] = new FDTreeElement(this.numAttributes);
		}
			
		currentNode = currentNode.getChildren()[i];
		currentNode.addRhsAttribute(rhs);
	}
	currentNode.markFd(rhs);
	
	this.depth = Math.max(this.depth, lhsLength);
	return currentNode;
}
 

protected void addToIndex(Int2ObjectOpenHashMap<ArrayList<ElementLhsPair>> level2elements, int level, OpenBitSet lhs) {
	level2elements.get(level).add(new ElementLhsPair(this, lhs.clone()));
	if (this.children != null) {
		for (int childAttr = 0; childAttr < this.numAttributes; childAttr++) {
			FDTreeElement element = this.children[childAttr];
			if (element != null) {
				lhs.set(childAttr);
				element.addToIndex(level2elements, level + 1, lhs);
				lhs.clear(childAttr);
			}
		}
	}
}
 

@Test
public void testContainsGeneralization() {
	OpenBitSet lhs = new OpenBitSet();
	lhs.set(0);
	lhs.set(1);
	assertFalse(this.fdtree.containsFdOrGeneralization(lhs, 2));
	lhs.set(3);
	lhs.set(4);
	assertTrue(this.fdtree.containsFdOrGeneralization(lhs, 2));
}
 

private static void applyColumns(Set<String> alreadyProcessed, OpenBitSet bits,
    Map<String, Set<String>> columnsToFetch, AtomicReader atomicReader, int primeDocRowId, int numberOfDocsInRow,
    Bits liveDocs) throws IOException {
  for (String family : columnsToFetch.keySet()) {
    if (!alreadyProcessed.contains(family)) {
      applyFamily(bits, family, atomicReader, primeDocRowId, numberOfDocsInRow, liveDocs);
      alreadyProcessed.add(family);
    }
  }
}
 

public boolean refines(int[][] compressedRecords, OpenBitSet lhs, int[] rhs) {
	for (IntArrayList cluster : this.clusters) {
		ClusterTree clusterTree = new ClusterTree();
		
		// Check if all subclusters of this cluster point into the same other clusters
		for (int recordId : cluster)
			if (!clusterTree.add(compressedRecords, lhs, recordId, rhs[recordId]))
				return false;
	}
	return true;
}
 

public List<OpenBitSet> asBitSetList() {
	List<OpenBitSet> foundLhs = new ArrayList<>();
	OpenBitSet currentLhs = new OpenBitSet();
	int nextLhsAttr = 0;
	this.asBitSetList(currentLhs, nextLhsAttr, foundLhs);
	return foundLhs;
}
 

public List<OpenBitSet> getLhsAndGeneralizations(OpenBitSet lhs) {
	List<OpenBitSet> foundLhs = new ArrayList<>();
	OpenBitSet currentLhs = new OpenBitSet();
	int nextLhsAttr = lhs.nextSetBit(0);
	this.getLhsAndGeneralizations(lhs, nextLhsAttr, currentLhs, foundLhs);
	return foundLhs;
}
 

public int addFunctionalDependenciesInto(FunctionalDependencyResultReceiver resultReceiver, OpenBitSet lhs, ObjectArrayList<ColumnIdentifier> columnIdentifiers, List<PositionListIndex> plis) throws CouldNotReceiveResultException, ColumnNameMismatchException {
	int numFDs = 0;
	for (int rhs = this.rhsFds.nextSetBit(0); rhs >= 0; rhs = this.rhsFds.nextSetBit(rhs + 1)) {
		ColumnIdentifier[] columns = new ColumnIdentifier[(int) lhs.cardinality()];
		int j = 0;
		for (int i = lhs.nextSetBit(0); i >= 0; i = lhs.nextSetBit(i + 1)) {
			int columnId = plis.get(i).getAttribute(); // Here we translate the column i back to the real column i before the sorting
			columns[j++] = columnIdentifiers.get(columnId); 
		}
		
		ColumnCombination colCombination = new ColumnCombination(columns);
		int rhsId = plis.get(rhs).getAttribute(); // Here we translate the column rhs back to the real column rhs before the sorting
		FunctionalDependency fdResult = new FunctionalDependency(colCombination, columnIdentifiers.get(rhsId));
		resultReceiver.receiveResult(fdResult);
		numFDs++;
	}

	if (this.getChildren() == null)
		return numFDs;
		
	for (int childAttr = 0; childAttr < this.numAttributes; childAttr++) {
		FDTreeElement element = this.getChildren()[childAttr];
		if (element != null) {
			lhs.set(childAttr);
			numFDs += element.addFunctionalDependenciesInto(resultReceiver, lhs, columnIdentifiers, plis);
			lhs.clear(childAttr);
		}
	}
	return numFDs;
}
 

public LhsTrieElement addLhs(OpenBitSet lhs) {
	LhsTrieElement currentNode = this;
	for (int i = lhs.nextSetBit(0); i >= 0; i = lhs.nextSetBit(i + 1)) {
		if (currentNode.getChildren()[i] != null)
			currentNode.setChild(this.numAttributes, i, new LhsTrieElement());
		currentNode = currentNode.getChildren()[i];
	}
	return currentNode;
}
 

public boolean contains(OpenBitSet fd) {
	int length = (int) fd.cardinality();
	
	if ((this.maxDepth > 0) && (length > this.maxDepth))
		return false;
	
	return this.fdLevels.get(length).contains(fd);
}
 

public int read(int[] docs, int[] freqs) throws IOException
{
    int[] innerDocs = new int[docs.length];
    int[] innerFreq = new int[docs.length];
    int count = in.read(innerDocs, innerFreq);

    // Is the stream exhausted
    if (count == 0)
    {
        return 0;
    }

    OpenBitSet deletedDocuments = getDeletedDocuments();
    while (allDeleted(innerDocs, count, deletedDocuments))
    {

        count = in.read(innerDocs, innerFreq);

        // Is the stream exhausted
        if (count == 0)
        {
            return 0;
        }
    }

    // Add non deleted

    int insertPosition = 0;
    for (int i = 0; i < count; i++)
    {
        if (!deletedDocuments.get(innerDocs[i]))
        {
            docs[insertPosition] = innerDocs[i];
            freqs[insertPosition] = innerFreq[i];
            insertPosition++;
        }
    }

    return insertPosition;
}
 

public boolean add(OpenBitSet fd) {
	int length = (int) fd.cardinality();
	
	if ((this.maxDepth > 0) && (length > this.maxDepth))
		return false;
	
	this.depth = Math.max(this.depth, length);
	return this.fdLevels.get(length).add(fd);
}
 

public FDTreeElement addFunctionalDependencyIfNotInvalid(OpenBitSet lhs, OpenBitSet rhs) {
	FDTreeElement currentNode = this;
	currentNode.addRhsAttributes(rhs);

	OpenBitSet invalidFds = currentNode.rhsAttributes.clone();
	int lhsLength = 0;
	for (int i = lhs.nextSetBit(0); i >= 0; i = lhs.nextSetBit(i + 1)) {
		lhsLength++;
		
		if (currentNode.getChildren() == null) {
			currentNode.setChildren(new FDTreeElement[this.numAttributes]);
			currentNode.getChildren()[i] = new FDTreeElement(this.numAttributes);
		}
		else if (currentNode.getChildren()[i] == null) {
			currentNode.getChildren()[i] = new FDTreeElement(this.numAttributes);
		}
			
		currentNode = currentNode.getChildren()[i];
		invalidFds.and(currentNode.rhsFds);
		currentNode.addRhsAttributes(rhs);
	}
	
	rhs.andNot(invalidFds);
	currentNode.markFds(rhs);
	rhs.or(invalidFds);

	this.depth = Math.max(this.depth, lhsLength);
	return currentNode;
}
 

public void addPrunedElements(OpenBitSet currentLhs, int maxCurrentLhsAttribute, FDTree tree) {
	this.addOneSmallerGeneralizations(currentLhs, maxCurrentLhsAttribute, this.rhsAttributes, tree);
	
	if (this.children == null)
		return;
	
	for (int attr = 0; attr < this.numAttributes; attr++) {
		if (this.children[attr] != null) {
			currentLhs.set(attr);
			this.children[attr].addPrunedElements(currentLhs, attr, tree);
			currentLhs.clear(attr);
		}
	}
}