it.unimi.dsi.fastutil.bytes.ByteArrayList Java Examples

@Override
public void end() {
  if (elementClass == boolean.class) {
    parent.add(((BooleanArrayList) container).toBooleanArray());
  } else if (elementClass == byte.class) {
    parent.add(((ByteArrayList) container).toByteArray());
  } else if (elementClass == char.class) {
    parent.add(((CharArrayList) container).toCharArray());
  } else if (elementClass == short.class) {
    parent.add(((ShortArrayList) container).toShortArray());
  } else if (elementClass == int.class) {
    parent.add(((IntArrayList) container).toIntArray());
  } else if (elementClass == long.class) {
    parent.add(((LongArrayList) container).toLongArray());
  } else if (elementClass == float.class) {
    parent.add(((FloatArrayList) container).toFloatArray());
  } else if (elementClass == double.class) {
    parent.add(((DoubleArrayList) container).toDoubleArray());
  } else {
    parent.add(((ArrayList) container).toArray());
  }
}
 

@Override
public BooleanColumn lag(int n) {
  int srcPos = n >= 0 ? 0 : 0 - n;
  byte[] dest = new byte[size()];
  int destPos = n <= 0 ? 0 : n;
  int length = n >= 0 ? size() - n : size() + n;

  for (int i = 0; i < size(); i++) {
    dest[i] = BooleanColumnType.MISSING_VALUE;
  }

  System.arraycopy(data.toByteArray(), srcPos, dest, destPos, length);

  BooleanColumn copy = emptyCopy(size());
  copy.data = new ByteArrayList(dest);
  copy.setName(name() + " lag(" + n + ")");
  return copy;
}
 

@Override
public void run() {
	try {
		final ByteArrayDiskQueue receivedURLs = frontier.receivedURLs;
		final ArrayBlockingQueue<ByteArrayList> quickReceivedURLs = frontier.quickReceivedURLs;
		while(! stop) {
			final ByteArrayList list = quickReceivedURLs.poll(1, TimeUnit.SECONDS);
			if (list != null) receivedURLs.enqueue(list.elements(), 0, list.size());
		}
	}
	catch (Throwable t) {
		LOGGER.error("Unexpected exception ", t);
	}

	LOGGER.info("Completed");
}
 

@Override
public BooleanColumn lag(int n) {
  int srcPos = n >= 0 ? 0 : 0 - n;
  byte[] dest = new byte[size()];
  int destPos = n <= 0 ? 0 : n;
  int length = n >= 0 ? size() - n : size() + n;

  for (int i = 0; i < size(); i++) {
    dest[i] = BooleanColumnType.MISSING_VALUE;
  }

  System.arraycopy(data.toByteArray(), srcPos, dest, destPos, length);

  BooleanColumn copy = emptyCopy(size());
  copy.data = new ByteArrayList(dest);
  copy.setName(name() + " lag(" + n + ")");
  return copy;
}
 

public DoubleColumn asDoubleColumn() {
  DoubleColumn numberColumn = DoubleColumn.create(this.name(), size());
  ByteArrayList data = data();
  for (int i = 0; i < size(); i++) {
    numberColumn.append(data.getByte(i));
  }
  return numberColumn;
}
 

@Override
public BooleanColumn unique() {
  ByteSet count = new ByteOpenHashSet(3);
  for (byte next : data) {
    count.add(next);
  }
  ByteArrayList list = new ByteArrayList(count);
  return new BooleanColumn(name() + " Unique values", list);
}
 

public static BooleanColumn create(String name, int initialSize) {
  BooleanColumn column = new BooleanColumn(name, new ByteArrayList(initialSize));
  for (int i = 0; i < initialSize; i++) {
    column.appendMissing();
  }
  return column;
}
 

public DoubleColumn asDoubleColumn() {
  DoubleColumn numberColumn = DoubleColumn.create(this.name(), size());
  ByteArrayList data = data();
  for (int i = 0; i < size(); i++) {
    numberColumn.append(data.getByte(i));
  }
  return numberColumn;
}
 

@Test
public void testUniMi() throws Exception {
    Byte2BooleanOpenHashMap map = new Byte2BooleanOpenHashMap();
    map.put((byte) 1, true);
    map.put((byte) -1, false);
    testConvert(Byte2BooleanOpenHashMap.class, map);

    ByteArrayList list = new ByteArrayList();
    list.add((byte) 1);
    list.add((byte) -1);
    testConvert(ByteArrayList.class, list);
}
 

@Override

	public void toStream(final ByteArrayList list, final OutputStream os) throws IOException {
		final int length = list.size();
		Util.writeVByte(length, os);
		os.write(list.elements(), 0, length);
	}
 

/** A serializer-deserializer for byte arrays that write the array length using variable-length byte encoding,
 * and the writes the content of the array. */
@Override
public ByteArrayList fromStream(final InputStream is) throws IOException {
	final int length = Util.readVByte(is);
	buffer.size(length);
	final int actual = is.read(buffer.elements(), 0, length);
	if (actual != length) throw new IOException("Asked for " + length + " but got " + actual);
	return buffer;
}
 

/** Extracts the path and query of an absolute BUbiNG URL in its byte-array representation.
 *
 * @param url BUbiNG URL.
 * @return the path and query in byte-array representation.
 */
public static byte[] pathAndQueryAsByteArray(ByteArrayList url){
	final byte[] array = url.elements();
	final int startOfAuthority = Bytes.indexOf(array, (byte)':') + 3;
	assert array[startOfAuthority - 1] == '/' : array[startOfAuthority - 1];
	assert array[startOfAuthority - 2] == '/' : array[startOfAuthority - 2];
	return Arrays.copyOfRange(array, ArrayUtils.indexOf(array, (byte)'/', startOfAuthority), url.size());
}
 

/** Returns a string representation of an ASCII byte array.
 *
 * <p>This method is significantly faster than those relying on character encoders, and it allocates just
 * one object&mdash;the resulting string.
 *
 * @param byteArrayList an ASCII byte-array list.
 * @return a string representation of {@code byteArrayList}.
 * @throws AssertionError if assertions are enabled and some character of {@code byteArrayList} is not ASCII.
 */
public static String toString(final ByteArrayList byteArrayList) {
	final char[] charArray = new char[byteArrayList.size()];
	final byte[] byteArray = byteArrayList.elements();
	// This needs to be fast.
	for(int i = byteArrayList.size(); i-- != 0;) {
		assert byteArray[i] < (char)0x80 : byteArray[i];
		charArray[i] = (char)byteArray[i];
	}
	return new String(charArray);
}
 

/** Writes an ASCII string in a {@link ByteArrayList}.
 *
 * @param s an ASCII string.
 * @param list a byte list that will contain the byte representation of {@code s}.
 * @return {@code list}.
 * @throws AssertionError if assertions are enabled and some character of {@code s} is not ASCII.
 */
public static ByteArrayList toByteArrayList(final String s, final ByteArrayList list) {
	list.clear();
	list.size(s.length());
	final byte[] array = list.elements();
	for (int i = list.size(); i-- != 0;) {
		assert s.charAt(i) < (char)0x80 : s.charAt(i);
		array[i] = (byte)(s.charAt(i) & 0x7F);
	}
	return list;
}
 

public static BooleanColumn create(String name, int initialSize) {
  BooleanColumn column = new BooleanColumn(name, new ByteArrayList(initialSize));
  for (int i = 0; i < initialSize; i++) {
    column.appendMissing();
  }
  return column;
}
 

@Override
public BooleanColumn unique() {
  ByteSet count = new ByteOpenHashSet(3);
  for (byte next : data) {
    count.add(next);
  }
  ByteArrayList list = new ByteArrayList(count);
  return new BooleanColumn(name() + " Unique values", list);
}
 

private BooleanColumn(String name, ByteArrayList values) {
  super(BooleanColumnType.instance(), name);
  data = values;
}
 

public static BooleanColumn create(String name) {
  return new BooleanColumn(name, new ByteArrayList(DEFAULT_ARRAY_SIZE));
}
 

ByteAttributeStore(AttributeInfo attrInfo) {
    super(attrInfo);
    this.list = new ByteArrayList();
}
 

@Override
public void sortDescending() {
  byte[] elements = values.toByteArray();
  ByteArrays.parallelQuickSort(elements, reverseDictionarySortComparator);
  this.values = new ByteArrayList(elements);
}
 

@Override
public void sortAscending() {
  byte[] elements = values.toByteArray();
  ByteArrays.parallelQuickSort(elements, dictionarySortComparator);
  this.values = new ByteArrayList(elements);
}
 

ByteArrayList values() {
  return values;
}
 

/** Returns a ByteArrayList containing 0 (false), 1 (true) or Byte.MIN_VALUE (missing) */
public ByteArrayList data() {
  return data;
}
 

@Override
public void sortDescending() {
  byte[] elements = values.toByteArray();
  ByteArrays.parallelQuickSort(elements, reverseDictionarySortComparator);
  this.values = new ByteArrayList(elements);
}
 

@Override
public void sortAscending() {
  byte[] elements = values.toByteArray();
  ByteArrays.parallelQuickSort(elements, dictionarySortComparator);
  this.values = new ByteArrayList(elements);
}
 

ByteArrayList values() {
  return values;
}
 

private BooleanColumn(String name, ByteArrayList values) {
  super(BooleanColumnType.instance(), name);
  data = values;
}
 

/** Hashes a {@link ByteArrayList} using MurmurHash3 with seed zero.
 *
 * @param list a byte-array list
 * @return a 64-bit MurmurHash3 zero-seed hash for the specified fragment. */

public final static long hash(final ByteArrayList list) {
	return hash(list.elements(), 0, list.size());
}
 

public Agent(final String hostname, final int jmxPort, final RuntimeConfiguration rc) throws Exception {
	// TODO: configure strategies

	super(rc.name, rc.weight, new InetSocketAddress(hostname,  jmxPort),
			new JChannel(System.getProperty(JGROUPS_CONFIGURATION_PROPERTY_NAME) != null ? (InputStream)new FileInputStream(System.getProperty(JGROUPS_CONFIGURATION_PROPERTY_NAME)) : JGroupsJobManager.class.getResourceAsStream('/' + JGroupsJobManager.class.getPackage().getName().replace('.',  '/') + "/jgroups.xml")),
			 rc.group, new ConsistentHashAssignmentStrategy<BubingJob>(), new LinkedBlockingQueue<BubingJob>(),
			 new TimedDroppingThreadFactory<BubingJob>(1800000), new DiscardMessagesStrategy<BubingJob>());

	LOGGER.info("Creating Agent instance with properties {}", rc);

	// TODO: check crawlIsNew for all components.
	this.rc = rc;

	store = rc.storeClass.getConstructor(RuntimeConfiguration.class).newInstance(rc);

	register();

	frontier = new Frontier(rc, store, this);
	setListener(frontier);

	(messageThread = new MessageThread(frontier)).start();
	(quickMessageThread = new QuickMessageThread(frontier)).start();

	connect();

	// It is important that threads are allocated here, that is, after the agent has been connected.
	frontier.dnsThreads(rc.dnsThreads);
	frontier.parsingThreads(rc.parsingThreads);
	frontier.fetchingThreads(rc.fetchingThreads);

	frontier.rc.ensureNotPaused();
	final ByteArrayList list = new ByteArrayList();
	while(rc.seed.hasNext()) {
		final URI nextSeed = rc.seed.next();
		if (nextSeed != null) frontier.enqueue(BURL.toByteArrayList(nextSeed, list));
	}

	// We wait for the notification of a stop event, usually caused by a call to stop().
	synchronized(this) {
		if (! rc.stopping) wait();
	}

	// Stuff to be done at stopping time

	// The message thread uses the sieve, which will be closed by the frontier.
	messageThread.stop = true;
	messageThread.join();
	LOGGER.info("Joined message thread");

	frontier.close();

	LOGGER.info("Going to close job manager " + this);
	close();
	LOGGER.info("Job manager closed");

	// We stop here the quick message thread. Messages in the receivedURLs queue will be snapped.
	quickMessageThread.stop = true;
	quickMessageThread.join();
	LOGGER.info("Joined quick message thread");

	frontier.snap();

	LOGGER.info("Agent " + this + " exits");
}
 

@Override
public BubingJob fromString(final String s) {
	final URI url = BURL.parse(s);
	if (url != null && url.isAbsolute()) return new BubingJob(ByteArrayList.wrap(BURL.toByteArray(url)));
	throw new IllegalArgumentException();
}