it.unimi.dsi.fastutil.bytes.ByteArrays Java Examples

private Set<byte[]> extend(final byte[] lhs, final byte[] rhs) {
  final Set<byte[]> extendedRhsCandidates =
      new ObjectOpenCustomHashSet<>(ByteArrays.HASH_STRATEGY);
  if (lhs.length + rhs.length > this.level - 1) {
    // extended candidates would have more columns than this level allows,
    // return empty extended candidates
    return extendedRhsCandidates;
  }
  for (final byte[] singleColumn : this.singleColumnPermutations) {
    if (ByteArrayPermutations.disjoint(singleColumn, lhs)
        && ByteArrayPermutations.disjoint(singleColumn, rhs)) {
      extendedRhsCandidates.add(ByteArrayPermutations.join(rhs, singleColumn));
    }
  }
  return extendedRhsCandidates;
}
 

private void buildPrefixBlocks() {
  this.prefixBlocks = new Object2ObjectOpenCustomHashMap<>(ByteArrays.HASH_STRATEGY);

  for (final byte[] permutation : this.previousLevelCandidates) {
    // prefix is the emtpy array for permutations of size 1 (level 1)
    final byte[] prefix = ByteArrayPermutations.prefix(permutation);

    if (this.prefixBlocks.get(prefix) == null) {
      this.prefixBlocks.put(prefix, new ArrayList<byte[]>());
    }
    this.prefixBlocks.get(prefix).add(permutation);
  }
}
 

protected void initializePartitions() throws AlgorithmExecutionException {
  this.permutationToPartition = new Object2ObjectOpenCustomHashMap<>(ByteArrays.HASH_STRATEGY);

  // create partitions for level 1
  for (final int columnIndex : this.columnIndices) {
    final byte[] singleColumnPermutation = {(byte) columnIndex};

    final SortedPartition partition =
        SortedPartitionCreator.createPartition(this.dataByColumns.get(columnIndex),
            this.types.get(columnIndex));

    this.permutationToPartition.put(singleColumnPermutation, partition);
  }

}
 

/** Wraps a byte-array fragment into this byte-array character sequence.
 *
 * @param b a byte array.
 * @param offset the first valid byte in <code>b</code>.
 * @param length the number of valid bytes in <code>b</code>, starting at <code>offset</code>.
 * @return this byte-array character sequence.
 */
public ByteArrayCharSequence wrap(final byte[] b, int offset, int length) {
	ByteArrays.ensureOffsetLength(b, offset, length);
	this.b = b;
	this.offset = offset;
	this.length = length;
	return this;
}
 

/** Dequeues an object from the queue in FIFO fashion. */
@SuppressWarnings("unchecked")
public synchronized T dequeue() throws IOException {
	final int length = byteDiskQueue.dequeueInt();
	fbaos.array = ByteArrays.grow(fbaos.array, length);
	byteDiskQueue.dequeue(fbaos.array, 0, length);
	size--;
	try {
		return (T)BinIO.loadObject(new FastByteArrayInputStream(fbaos.array, 0, length));
	}
	catch (final ClassNotFoundException e) {
		throw new RuntimeException(e.getMessage(), e);
	}
}
 

void ensure(int numBytes) {
    int newCapacity = length + numBytes;
    if (newCapacity <= data.length) {
        return;
    }

    newCapacity =
            (int) Math.max(Math.min((long) data.length + (data.length >> 1), Arrays.MAX_ARRAY_SIZE),
                           newCapacity);

    data = ByteArrays.forceCapacity(data, newCapacity, length);
}
 

public int read( final byte b[], final int off, final int len ) throws IOException {
	ensureNotClosed();
	ByteArrays.ensureOffsetLength( b, off, len );
	if ( len == 0 ) return 0; // Requested by InputStream.
	if ( eofInBlock() ) return -1;
	int effectivelen = Math.min( segmentLen - relativePos, len );
	effectivelen = in.read( b, off, effectivelen );
	relativePos += effectivelen;
	return effectivelen;
}
 

@Override
public void sortDescending() {
  byte[] elements = values.toByteArray();
  ByteArrays.parallelQuickSort(elements, reverseDictionarySortComparator);
  this.values = new ByteArrayList(elements);
}
 

@Override
public void execute() throws AlgorithmExecutionException {
  super.execute();

  this.partitionsCache =
      CacheBuilder.newBuilder().maximumSize(this.partitionCacheSize)
          .build(new CacheLoader<ByteArray, SortedPartition>() {
            @Override
            public SortedPartition load(final ByteArray key) throws Exception {
              return ORDERLhsRhs.this.createPartitionFromSingletons(key.data);
            }
          });

  this.logger.info("Running order dependency detection algorithm (lhs and rhs) on table: "
      + this.tableName + " (" + this.numRows + " rows, " + this.columnIndices.length
      + " columns)");

  if (this.columnNames.size() > MAX_NUM_COLS) {
    throw new AlgorithmConfigurationException("You provided a table with "
        + this.columnNames.size()
        + " columns. This order dependency detection algorithm supports a maximum of "
        + MAX_NUM_COLS + " columns.");
  }

  this.levelCandidates = new ArrayList<>();
  this.currentRhsCandidateSet = new Object2ObjectOpenCustomHashMap<>(ByteArrays.HASH_STRATEGY);
  this.singleColumnPermutations = new ArrayList<>();
  this.validDependencies = new Object2ObjectOpenCustomHashMap<>(ByteArrays.HASH_STRATEGY);

  // initialize level 1
  for (final int lhsColumnIndex : this.columnIndices) {
    final byte[] singleLhsColumnPermutation = {(byte) lhsColumnIndex};

    // columns with only equal values mean that any OD with that lhs holds (under "<")
    // (but there exists no minimal n-ary OD that has lhs on its lhs or rhs,
    // i.e., they can be removed)
    if (this.permutationToPartition.get(singleLhsColumnPermutation).size() == 1) {
      for (final int rhsColumnIndex : this.columnIndices) {
        final byte[] rhs = new byte[] {(byte) rhsColumnIndex};
        if (Arrays.equals(singleLhsColumnPermutation, rhs)) {
          continue;
        }
        this.statistics.setNumFoundDependencies(this.statistics.getNumFoundDependencies() + 1);
        this.statistics.setNumFoundDependenciesInPreCheck(this.statistics
            .getNumFoundDependenciesInPreCheck() + 1);
        this.signalFoundOrderDependency(singleLhsColumnPermutation, rhs,
            ComparisonOperator.STRICTLY_SMALLER, OrderType.LEXICOGRAPHICAL);
      }

      continue;
    }

    this.singleColumnPermutations.add(singleLhsColumnPermutation);

    this.levelCandidates.add(singleLhsColumnPermutation);
    this.currentRhsCandidateSet.put(singleLhsColumnPermutation, new ObjectOpenCustomHashSet<>(
        ByteArrays.HASH_STRATEGY));
  }

  for (final byte[] levelCandidate : this.levelCandidates) {

    for (final byte[] singleColumnRhsCandidate : this.singleColumnPermutations) {
      if (!ByteArrayPermutations.disjoint(levelCandidate, singleColumnRhsCandidate)) {
        continue;
      }
      this.currentRhsCandidateSet.get(levelCandidate).add(singleColumnRhsCandidate);
    }
  }

  this.level = 1;
  while (!this.levelCandidates.isEmpty()) {
    this.computeDependencies();
    this.prune();
    this.logger.debug("Candidate set: {}", this.prettyPrintCurrentRhsCandidates());
    this.generateNextLevel();
    this.logger.info("Statistics after generating level " + (this.level + 1) + " for table #"
        + this.tableName + "#: " + this.statistics.toString());
    this.level++;
  }

  this.logger.info("Statistics (FINAL) for table #" + this.tableName + "#: "
      + this.statistics.toString());

}
 

@Override
public void sortAscending() {
  byte[] elements = values.toByteArray();
  ByteArrays.parallelQuickSort(elements, dictionarySortComparator);
  this.values = new ByteArrayList(elements);
}
 

public ObjectListIterator<byte[]> listIterator(final int start) {
        ensureIndex(start);

        return new AbstractObjectListIterator<byte[]>() {
            byte a[] = ByteArrays.EMPTY_ARRAY;

            int i = 0, pos = 0;

            boolean inSync; // Whether the current value in a is the string just
                            // before the next to be produced.

            {
                if (start != 0) {
                    if (start == n)
                        i = start; // If we start at the end, we do nothing.
                    else {
                        pos = p[start / ratio];
                        int j = start % ratio;
                        i = start - j;
                        while (j-- != 0)
                            next();
                    }
                }
            }

            public boolean hasNext() {
                return i < n;
            }

            public boolean hasPrevious() {
                return i > 0;
            }

            public int previousIndex() {
                return i - 1;
            }

            public int nextIndex() {
                return i;
            }

            public byte[] next() {
                int length, common;

                if (!hasNext())
                    throw new NoSuchElementException();

                final BackingBuffer bb = CustomByteArrayFrontCodedList.this.bb;
                if (i % ratio == 0) {
                    pos = p[i / ratio];
//                    length = readInt(array, pos);
                    length = bb.readInt(pos);
                    a = ByteArrays.ensureCapacity(a, length, 0);
//                    System.arraycopy(array, pos + count(length), a, 0, length);
                    bb.arraycopy(pos + count(length), a, 0, length);
                    pos += length + count(length);
                    inSync = true;
                } else {
                    if (inSync) {
//                        length = readInt(array, pos);
//                        common = readInt(array, pos + count(length));
                        length = bb.readInt(pos);
                        common = bb.readInt(pos + count(length));
                        a = ByteArrays.ensureCapacity(a, length + common,
                                common);
//                        System.arraycopy(array, pos + count(length)
//                                + count(common), a, common, length);
                        bb.arraycopy(pos + count(length)
                                + count(common), a, common, length);
                        pos += count(length) + count(common) + length;
                        length += common;
                    } else {
                        a = ByteArrays.ensureCapacity(a, length = length(i), 0);
                        extract(i, a, 0, length);
                    }
                }
                i++;
                return ByteArrays.copy(a, 0, length);
            }

            public byte[] previous() {
                if (!hasPrevious())
                    throw new NoSuchElementException();
                inSync = false;
                return getArray(--i);
            }
        };
    }
 

public CustomByteArrayFrontCodedList(final Iterator<byte[]> arrays,
            final int ratio, final boolean hasDups) {

        assertRatio(ratio);
//        if (ratio < 1)
//            throw new IllegalArgumentException("Illegal ratio (" + ratio + ")");

        byte[] array = ByteArrays.EMPTY_ARRAY;
        int[] p = IntArrays.EMPTY_ARRAY;

        byte[][] a = new byte[2][];
        int curSize = 0, b = 0, common, length, minLength;

        while (arrays.hasNext()) {
            a[b] = (byte[]) arrays.next();
            length = a[b].length;

            if (n % ratio == 0) {
                p = IntArrays.grow(p, n / ratio + 1);
                p[n / ratio] = curSize;

                array = ByteArrays.grow(array,
                        curSize + count(length) + length, curSize);
                curSize += writeInt(array, length, curSize);
                System.arraycopy(a[b], 0, array, curSize, length);
                curSize += length;
            } else {
                minLength = a[1 - b].length;
                if (length < minLength)
                    minLength = length;
                for (common = 0; common < minLength; common++)
                    if (a[0][common] != a[1][common])
                        break;
                length -= common;

                array = ByteArrays.grow(array, curSize + count(length)
                        + count(common) + length, curSize);
                curSize += writeInt(array, length, curSize);
                curSize += writeInt(array, common, curSize);
                System.arraycopy(a[b], common, array, curSize, length);
                curSize += length;
            }

            b = 1 - b;
            n++;
        }

        this.ratio = ratio;
//      this.array = ByteArrays.trim(array, curSize);
        this.bb = new BackingByteArray( ByteArrays.trim(array, curSize) );
//        this.bb = new BackingByteBuffer( ByteBuffer.wrap(ByteArrays.trim(array, curSize) ));
        this.p = IntArrays.trim(p, (n + ratio - 1) / ratio);

        this.hasDups = hasDups;
    }
 

@Override
public void sortDescending() {
  byte[] elements = values.toByteArray();
  ByteArrays.parallelQuickSort(elements, reverseDictionarySortComparator);
  this.values = new ByteArrayList(elements);
}
 

@Override
public void sortAscending() {
  byte[] elements = values.toByteArray();
  ByteArrays.parallelQuickSort(elements, dictionarySortComparator);
  this.values = new ByteArrayList(elements);
}
 

/** Creates a new empty byte-array character sequence.
 */
public ByteArrayCharSequence() {
	this(ByteArrays.EMPTY_ARRAY);
}
 

private void generateNextLevel() {

    long time = System.currentTimeMillis();

    this.previousLevelCandidates = this.levelCandidates;
    this.levelCandidates = new ArrayList<>();

    this.buildPrefixBlocks();

    System.gc();

    this.logger.debug("PREFIX_BLOCKS in level {}: {} ", Integer.valueOf(this.level), this.prettyPrintPrefixBlocks());

    for (final List<byte[]> candidatesWithSamePrefix : this.prefixBlocks.values()) {
      if (candidatesWithSamePrefix.size() < 2) {
        break;
      }
      for (final byte[] candidate : candidatesWithSamePrefix) {
        for (final byte[] candidateForJoin : candidatesWithSamePrefix) {
          if (Arrays.equals(candidate, candidateForJoin)) {
            continue;
          }

          final byte[] joinedCandidate = ByteArrayPermutations.join(candidate, candidateForJoin);
          this.levelCandidates.add(joinedCandidate);
        }
      }
    }
    this.logger.debug("Generated level {}", Integer.valueOf(this.level + 1));
    this.logger.debug("Level {} candidates: {}", Integer.valueOf(this.level + 1),
        ByteArrayPermutations.permutationListToIntegerString(this.levelCandidates));
    if (this.level > 1 && !this.levelCandidates.isEmpty()) {
      for (final byte[] newLhs : this.previousLevelCandidates) {
        this.logger
            .debug(
                "After generating level {}. Adding {} as an lhs to the candidate set for the next level.",
                Integer.valueOf(this.level + 1), ByteArrayPermutations.permutationToIntegerString(newLhs));
        this.currentRhsCandidateSet.put(newLhs, new ObjectOpenCustomHashSet<byte[]>(
            ByteArrays.HASH_STRATEGY));
      }
    }

    time = System.currentTimeMillis() - time;
    this.statistics.setGenNextTime(this.statistics.getGenNextTime() + time);

  }
 

/**
 * Stores in the given array elements from an array stored in this
 * front-coded list.
 * 
 * @param index
 *            an index.
 * @param a
 *            the array that will store the result.
 * @param offset
 *            an offset into <code>a</code> where elements will be store.
 * @param length
 *            a maximum number of elements to store in <code>a</code>.
 * @return if <code>a</code> can hold the extracted elements, the number of
 *         extracted elements; otherwise, the number of remaining elements
 *         with the sign changed.
 */
public int get(final int index, final byte[] a, final int offset,
        final int length) {
    ensureRestrictedIndex(index);
    ByteArrays.ensureOffsetLength(a, offset, length);

    final int arrayLength = extract(index, a, offset, length);
    if (length >= arrayLength)
        return arrayLength;
    return length - arrayLength;
}