Java Code Examples for gnu.trove.list.array.TLongArrayList#add()

public static IrregularTimeSeriesIndex parseJson(JsonParser parser) {
    Objects.requireNonNull(parser);
    JsonToken token;
    try {
        TLongArrayList times = new TLongArrayList();
        while ((token = parser.nextToken()) != null) {
            if (token == JsonToken.VALUE_NUMBER_INT) {
                times.add(parser.getLongValue());
            } else if (token == JsonToken.END_ARRAY) {
                return new IrregularTimeSeriesIndex(times.toArray());
            }
        }
        throw new IllegalStateException("Should not happen");
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}
 

private void addIfInside(
		final TLongArrayList labelCoordinates,
		final TDoubleArrayList worldCoordinates,
		final long lx,
		final long ly,
		final long lz,
		final double wx,
		final double wy,
		final double wz,
		final double cx,
		final double cy,
		final double zMinInclusive,
		final double zMaxInclusive)
{
	if (wz >= zMinInclusive && wz <= zMaxInclusive)
	{
		final double dx = wx - cx;
		final double dy = wy - cy;
		if (dx * dx * radiusYSquared + dy * dy * radiusXSquared <= radiusXSquaredRadiusYSquared)
		{
			labelCoordinates.add(lx);
			labelCoordinates.add(ly);
			labelCoordinates.add(lz);

			worldCoordinates.add(wx);
			worldCoordinates.add(wy);
			worldCoordinates.add(wz);
		}
	}
}
 

private void addIfInside(
		final boolean isRelevant,
		final long fillLabel,
		final long pixelLabel,
		final TLongArrayList labelCoordinates,
		final TDoubleArrayList worldCoordinates,
		final long lx,
		final long ly,
		final long lz,
		final double wx,
		final double wy,
		final double wz,
		final double zMinInclusive,
		final double zMaxInclusive)
{
	if (isRelevant && fillLabel != pixelLabel && wz >= zMinInclusive && wz <= zMaxInclusive)
	{
		if (wx > minX && wx < maxX && wy > minY && wy < maxY)
		{
			labelCoordinates.add(lx);
			labelCoordinates.add(ly);
			labelCoordinates.add(lz);

			worldCoordinates.add(wx);
			worldCoordinates.add(wy);
			worldCoordinates.add(wz);
		}
	}
}
 

private TLongArrayList feedBatch() {
  TLongArrayList handles = new TLongArrayList(512);
  while (handleIterator.hasNext()) {
    handles.add(handleIterator.next());
    if (batchSize <= handles.size()) {
      break;
    }
  }
  return handles;
}
 

@Test
public void splitHandlesByRegionTest() throws Exception {
  final long tableId = 1;
  TLongArrayList handles = new TLongArrayList();
  handles.add(new long[] {
      1, 5, 4, 3, 10, 2, 100, 101, 99, 88, -1, -255, -100, -99, -98, Long.MIN_VALUE, 8960, 8959, 19999, 15001
  });

  MockRegionManager mgr = new MockRegionManager(ImmutableList.of(
      keyRangeByHandle(tableId, null, Status.EQUAL, -100L, Status.EQUAL),
      keyRangeByHandle(tableId, -100L, Status.EQUAL, 10L, Status.GREATER),
      keyRangeByHandle(tableId, 10L, Status.GREATER, 50L, Status.EQUAL),
      keyRangeByHandle(tableId, 50L, Status.EQUAL, 100L, Status.GREATER),
      keyRangeByHandle(tableId, 100L, Status.GREATER, 9000L, Status.LESS),
      keyRangeByHandle(tableId, 0x2300L /*8960*/, Status.LESS, 16000L, Status.EQUAL),
      keyRangeByHandle(tableId, 16000L, Status.EQUAL, null, Status.EQUAL)
  ));

  RangeSplitter s = RangeSplitter.newSplitter(mgr);
  List<RangeSplitter.RegionTask> tasks = s.splitHandlesByRegion(tableId, handles);

  // [-INF, -100): [Long.MIN_VALUE, Long.MIN_VALUE + 1), [-255, -254)
  assertEquals(tasks.get(0).getRegion().getId(), 0);
  assertEquals(tasks.get(0).getRanges().size(), 2);
  assertEquals(tasks.get(0).getRanges().get(0), keyRangeByHandle(tableId, Long.MIN_VALUE, Long.MIN_VALUE + 1));
  assertEquals(tasks.get(0).getRanges().get(1), keyRangeByHandle(tableId, -255L, -254L));

  // [-100, 10.x): [-100, -97), [-1, 0), [1, 6), [10, 11)
  assertEquals(tasks.get(1).getRegion().getId(), 1);
  assertEquals(tasks.get(1).getRanges().size(), 4);
  assertEquals(tasks.get(1).getRanges().get(0), keyRangeByHandle(tableId, -100L, -97L));
  assertEquals(tasks.get(1).getRanges().get(1), keyRangeByHandle(tableId, -1L, 0L));
  assertEquals(tasks.get(1).getRanges().get(2), keyRangeByHandle(tableId, 1L, 6L));
  assertEquals(tasks.get(1).getRanges().get(3), keyRangeByHandle(tableId, 10L, 11L));

  // [10.x, 50): empty
  // [50, 100.x): [88, 89) [99, 101)
  assertEquals(tasks.get(2).getRegion().getId(), 3);
  assertEquals(tasks.get(2).getRanges().size(), 2);
  assertEquals(tasks.get(2).getRanges().get(0), keyRangeByHandle(tableId, 88L, 89L));
  assertEquals(tasks.get(2).getRanges().get(1), keyRangeByHandle(tableId, 99L, 101L));

  // [100.x, less than 8960): [101, 102) [8959, 8960)
  assertEquals(tasks.get(3).getRegion().getId(), 4);
  assertEquals(tasks.get(3).getRanges().size(), 2);
  assertEquals(tasks.get(3).getRanges().get(0), keyRangeByHandle(tableId, 101L, 102L));
  assertEquals(tasks.get(3).getRanges().get(1), keyRangeByHandle(tableId, 8959L, 8960L));

  // [less than 8960, 16000): [9000, 9001), [15001, 15002)
  assertEquals(tasks.get(4).getRegion().getId(), 5);
  assertEquals(tasks.get(4).getRanges().size(), 2);
  assertEquals(tasks.get(4).getRanges().get(0), keyRangeByHandle(tableId, 8960L, 8961L));
  assertEquals(tasks.get(4).getRanges().get(1), keyRangeByHandle(tableId, 15001L, 15002L));

  // [16000, INF): [19999, 20000)
  assertEquals(tasks.get(5).getRegion().getId(), 6);
  assertEquals(tasks.get(5).getRanges().size(), 1);
  assertEquals(tasks.get(5).getRanges().get(0), keyRangeByHandle(tableId, 19999L, 20000L));
}
 

/**
 * Create and write an index index based on the input Cassandra Index.db file. Read the Index.db and generate chunks
 * (splits) based on the configured chunk size.
 *
 * @param fileSystem Hadoop file system.
 * @param sstablePath SSTable Index.db.
 * @throws IOException
 */
public static void writeIndex(final FileSystem fileSystem, final Path sstablePath) throws IOException {

    final Configuration configuration = fileSystem.getConf();

    final long splitSize = configuration.getLong(HadoopSSTableConstants.HADOOP_SSTABLE_SPLIT_MB,
            HadoopSSTableConstants.DEFAULT_SPLIT_MB) * 1024 * 1024;

    final Closer closer = Closer.create();

    final Path outputPath = sstablePath.suffix(SSTABLE_INDEX_SUFFIX);
    final Path inProgressOutputPath = sstablePath.suffix(SSTABLE_INDEX_IN_PROGRESS_SUFFIX);

    boolean success = false;
    try {
        final FSDataOutputStream os = closer.register(fileSystem.create(inProgressOutputPath));

        final TLongArrayList splitOffsets = new TLongArrayList();
        long currentStart = 0;
        long currentEnd = 0;
        final IndexOffsetScanner index = closer.register(new IndexOffsetScanner(sstablePath, fileSystem));

        while (index.hasNext()) {
            // NOTE: This does not give an exact size of this split in bytes but a rough estimate.
            // This should be good enough since it's only used for sorting splits by size in hadoop land.
            while (currentEnd - currentStart < splitSize && index.hasNext()) {
                currentEnd = index.next();
                splitOffsets.add(currentEnd);
            }

            // Record the split
            final long[] offsets = splitOffsets.toArray();
            os.writeLong(offsets[0]); // Start
            os.writeLong(offsets[offsets.length - 1]); // End

            // Clear the offsets
            splitOffsets.clear();

            if (index.hasNext()) {
                currentStart = index.next();
                currentEnd = currentStart;
                splitOffsets.add(currentStart);
            }
        }

        success = true;
    } finally {
        closer.close();

        if (!success) {
            fileSystem.delete(inProgressOutputPath, false);
        } else {
            fileSystem.rename(inProgressOutputPath, outputPath);
        }
    }
}