Java Code Examples for org.apache.spark.sql.Row#getLong()

private Map<TopicPartition, Long> getLastOffsets() throws Exception {
  Map<TopicPartition, Long> offsetRanges = Maps.newHashMap();
  // Create filter for groupid/topic
  for (String topic : topics) {
    StructType filterSchema = DataTypes.createStructType(Lists.newArrayList(
        DataTypes.createStructField("group_id", DataTypes.StringType, false),
        DataTypes.createStructField("topic", DataTypes.StringType, false)));
    Row groupIDTopicFilter = new RowWithSchema(filterSchema, groupID, topic);
    Iterable<Row> filters = Collections.singleton(groupIDTopicFilter);

    // Get results
    RandomOutput output = getOffsetsOutput(config, true);
    Iterable<Row> results = output.getExistingForFilters(filters);

    // Transform results into map
    for (Row result : results) {
      Integer partition = result.getInt(result.fieldIndex("partition"));
      Long offset = result.getLong(result.fieldIndex("offset"));
      TopicPartition topicPartition = new TopicPartition(topic, partition);
      offsetRanges.put(topicPartition, offset);
    }
  }
  
  return offsetRanges;
}
 

/**
 * Updates the given aggregation buffer `buffer` with new input data from `input`.
 *
 * @param buffer buffer to update.
 * @param input  input to update with.
 */
public void update(final MutableAggregationBuffer buffer, final Row input) {
  if (!input.isNullAt(0)) {
    long updatedSum = buffer.getLong(0) + input.getLong(0);
    long updatedCount = buffer.getLong(1) + 1;
    buffer.update(0, updatedSum);
    buffer.update(1, updatedCount);
  }
}
 

/**
 * Merges two aggregation buffers and stores the updated buffer values back to `buffer1`.
 *
 * @param buffer1 first buffer.
 * @param buffer2 second buffer.
 */
public void merge(final MutableAggregationBuffer buffer1, final Row buffer2) {
  long mergedSum = buffer1.getLong(0) + buffer2.getLong(0);
  long mergedCount = buffer1.getLong(1) + buffer2.getLong(1);
  buffer1.update(0, mergedSum);
  buffer1.update(1, mergedCount);
}
 

private static ManifestFile writeManifest(
    List<Row> rows, int startIndex, int endIndex, Broadcast<FileIO> io,
    String location, int format, PartitionSpec spec, StructType sparkType) throws IOException {

  String manifestName = "optimized-m-" + UUID.randomUUID();
  Path manifestPath = new Path(location, manifestName);
  OutputFile outputFile = io.value().newOutputFile(FileFormat.AVRO.addExtension(manifestPath.toString()));

  Types.StructType dataFileType = DataFile.getType(spec.partitionType());
  SparkDataFile wrapper = new SparkDataFile(dataFileType, sparkType);

  ManifestWriter writer = ManifestFiles.write(format, spec, outputFile, null);

  try {
    for (int index = startIndex; index < endIndex; index++) {
      Row row = rows.get(index);
      long snapshotId = row.getLong(0);
      long sequenceNumber = row.getLong(1);
      Row file = row.getStruct(2);
      writer.existing(wrapper.wrap(file), snapshotId, sequenceNumber);
    }
  } finally {
    writer.close();
  }

  return writer.toManifestFile();
}
 

private static Object getPrimitiveValue(Row row, int ord, Type type) {
  if (row.isNullAt(ord)) {
    return null;
  }
  switch (type.typeId()) {
    case BOOLEAN:
      return row.getBoolean(ord);
    case INTEGER:
      return row.getInt(ord);
    case LONG:
      return row.getLong(ord);
    case FLOAT:
      return row.getFloat(ord);
    case DOUBLE:
      return row.getDouble(ord);
    case STRING:
      return row.getString(ord);
    case BINARY:
    case FIXED:
    case UUID:
      return row.get(ord);
    case DATE:
      return row.getDate(ord);
    case TIMESTAMP:
      return row.getTimestamp(ord);
    case DECIMAL:
      return row.getDecimal(ord);
    default:
      throw new IllegalArgumentException("Unhandled type " + type);
  }
}
 

private static Object getPrimitiveValue(Row row, int ord, Type type) {
  if (row.isNullAt(ord)) {
    return null;
  }
  switch (type.typeId()) {
    case BOOLEAN:
      return row.getBoolean(ord);
    case INTEGER:
      return row.getInt(ord);
    case LONG:
      return row.getLong(ord);
    case FLOAT:
      return row.getFloat(ord);
    case DOUBLE:
      return row.getDouble(ord);
    case STRING:
      return row.getString(ord);
    case BINARY:
    case FIXED:
    case UUID:
      return row.get(ord);
    case DATE:
      return row.getDate(ord);
    case TIMESTAMP:
      return row.getTimestamp(ord);
    case DECIMAL:
      return row.getDecimal(ord);
    default:
      throw new IllegalArgumentException("Unhandled type " + type);
  }
}
 

/**
 * Updates the given aggregation buffer `buffer` with new input data from `input`.
 *
 * @param buffer buffer to update.
 * @param input input to update with.
 */
public void update(final MutableAggregationBuffer buffer, final Row input) {
  if (!input.isNullAt(0)) {
    long updatedSum = buffer.getLong(0) + input.getLong(0);
    long updatedCount = buffer.getLong(1) + 1;
    buffer.update(0, updatedSum);
    buffer.update(1, updatedCount);
  }
}
 

/**
 * Merges two aggregation buffers and stores the updated buffer values back to `buffer1`.
 *
 * @param buffer1 first buffer.
 * @param buffer2 second buffer.
 */
public void merge(final MutableAggregationBuffer buffer1, final Row buffer2) {
  long mergedSum = buffer1.getLong(0) + buffer2.getLong(0);
  long mergedCount = buffer1.getLong(1) + buffer2.getLong(1);
  buffer1.update(0, mergedSum);
  buffer1.update(1, mergedCount);
}
 

@Override
public Iterator<Span> next() {
  Row peeked = delegate.peek();
  currentTraceIdHi = hasTraceIdHigh ? peeked.getLong(0) : 0L;
  currentTraceIdLo = peeked.getLong(traceIdIndex);
  return new DependencyLinkSpanIterator(
      delegate, traceIdIndex, currentTraceIdHi, currentTraceIdLo);
}
 

@Override
public void read(Row row, int ordinal) throws StandardException {
	if (row.isNullAt(ordinal))
		setToNull();
	else {
		isNull = false;
		value = row.getLong(ordinal);
	}
}
 

/**
 * Calculates the final result.
 *
 * @param buffer buffer row.
 * @return the result.
 */
public Double evaluate(final Row buffer) {
  return ((double) buffer.getLong(0)) / buffer.getLong(1);
}
 

/**
 * Calculates the final result.
 *
 * @param buffer buffer row.
 * @return the result.
 */
public Double evaluate(final Row buffer) {
  return ((double) buffer.getLong(0)) / buffer.getLong(1);
}