Java Code Examples for org.apache.cassandra.io.sstable.Descriptor#Version

public RowIndexEntry deserialize(DataInput in, Descriptor.Version version) throws IOException
{
    long position = in.readLong();

    int size = in.readInt();
    if (size > 0)
    {
        DeletionTime deletionTime = DeletionTime.serializer.deserialize(in);

        int entries = in.readInt();
        ISerializer<IndexHelper.IndexInfo> idxSerializer = type.indexSerializer();
        List<IndexHelper.IndexInfo> columnsIndex = new ArrayList<IndexHelper.IndexInfo>(entries);
        for (int i = 0; i < entries; i++)
            columnsIndex.add(idxSerializer.deserialize(in));

        return new IndexedEntry(position, deletionTime, columnsIndex);
    }
    else
    {
        return new RowIndexEntry(position);
    }
}
 

public OnDiskAtom deserializeFromSSTable(DataInput in, ColumnSerializer.Flag flag, int expireBefore, Descriptor.Version version) throws IOException
{
    Composite name = type.serializer().deserialize(in);
    if (name.isEmpty())
    {
        // SSTableWriter.END_OF_ROW
        return null;
    }

    int b = in.readUnsignedByte();
    if ((b & ColumnSerializer.RANGE_TOMBSTONE_MASK) != 0)
        return type.rangeTombstoneSerializer().deserializeBody(in, name, version);
    else
        return type.columnSerializer().deserializeColumnBody(in, (CellName)name, b, flag, expireBefore);
}
 

private String getTempSSTablePath(File directory, Descriptor.Version version)
{
    Descriptor desc = new Descriptor(version,
                                     directory,
                                     keyspace.getName(),
                                     name,
                                     fileIndexGenerator.incrementAndGet(),
                                     Descriptor.Type.TEMP);
    return desc.filenameFor(Component.DATA);
}
 

public RangeTombstone deserializeFromSSTable(DataInput in, Descriptor.Version version) throws IOException
{
    Composite min = type.serializer().deserialize(in);

    int b = in.readUnsignedByte();
    assert (b & ColumnSerializer.RANGE_TOMBSTONE_MASK) != 0;
    return deserializeBody(in, min, version);
}
 

public CompactionMetadata deserialize(Descriptor.Version version, DataInput in) throws IOException
{
    int nbAncestors = in.readInt();
    Set<Integer> ancestors = new HashSet<>(nbAncestors);
    for (int i = 0; i < nbAncestors; i++)
        ancestors.add(in.readInt());
    ICardinality cardinality = HyperLogLogPlus.Builder.build(ByteBufferUtil.readBytes(in, in.readInt()));
    return new CompactionMetadata(ancestors, cardinality);
}
 

public StreamReader(FileMessageHeader header, StreamSession session)
{
    this.session = session;
    this.cfId = header.cfId;
    this.estimatedKeys = header.estimatedKeys;
    this.sections = header.sections;
    this.inputVersion = new Descriptor.Version(header.version);
    this.repairedAt = header.repairedAt;
}
 

public AtomDeserializer(CellNameType type, DataInput in, ColumnSerializer.Flag flag, int expireBefore, Descriptor.Version version)
{
    this.type = type;
    this.nameDeserializer = type.newDeserializer(in);
    this.in = in;
    this.flag = flag;
    this.expireBefore = expireBefore;
    this.version = version;
}
 

public Iterator<OnDiskAtom> getOnDiskIterator(DataInput in, Descriptor.Version version)
{
    return getOnDiskIterator(in, ColumnSerializer.Flag.LOCAL, Integer.MIN_VALUE, version);
}
 

@Override
public void initialize(@Nonnull InputSplit inputSplit, @Nonnull final TaskAttemptContext ctx)
        throws IOException, InterruptedException {
    Configuration conf = ctx.getConfiguration();
    final AegSplit split = (AegSplit) inputSplit;

    long start = split.getStart();
    InputStream is = split.getInput(conf);
    long end = split.getDataEnd();
    URI fileUri = split.getPath().toUri();
    String filename = fileUri.toString();
    sourcePath = fileUri.getPath();
    traceDataFromSource = conf.getBoolean(Aegisthus.Feature.CONF_TRACE_DATA_FROM_SOURCE, false);

    LOG.info("File: {}", sourcePath);
    LOG.info("Start: {}", start);
    LOG.info("End: {}", end);

    try {
        Descriptor.Version version = Descriptor.Version.CURRENT;
        try {
            version = Descriptor.fromFilename(filename).version;
        } catch (Exception ignored) {
            // The 2.0 fromFilename parser fails on the latest Cassandra filenames, ignore this error and uses latest
        }
        scanner = new SSTableColumnScanner(is, start, end, version);
        LOG.info("Creating observable");
        rx.Observable<AtomWritable> observable = scanner.observable();
        observable = observable
                .onErrorFlatMap(new Func1<OnErrorThrowable, Observable<? extends AtomWritable>>() {
                    @Override
                    public Observable<? extends AtomWritable> call(OnErrorThrowable onErrorThrowable) {
                        LOG.error("failure deserializing file {}", split.getPath(), onErrorThrowable);
                        ctx.getCounter("aegisthus", "error_skipped_input").increment(1L);
                        return Observable.empty();
                    }
                });

        iterator = ObservableToIterator.toIterator(observable);
        LOG.info("done initializing");
    } catch (IOException e) {
        throw new IOError(e);
    }
}
 

public ValidationMetadata deserialize(Descriptor.Version version, DataInput in) throws IOException
{

    return new ValidationMetadata(in.readUTF(), in.readDouble());
}
 

public StatsMetadata deserialize(Descriptor.Version version, DataInput in) throws IOException
{
    EstimatedHistogram rowSizes = EstimatedHistogram.serializer.deserialize(in);
    EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
    ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(in);
    long minTimestamp = in.readLong();
    long maxTimestamp = in.readLong();
    int maxLocalDeletionTime = in.readInt();
    double compressionRatio = in.readDouble();
    StreamingHistogram tombstoneHistogram = StreamingHistogram.serializer.deserialize(in);
    int sstableLevel = in.readInt();
    long repairedAt = 0;
    if (version.hasRepairedAt)
        repairedAt = in.readLong();

    int colCount = in.readInt();
    List<ByteBuffer> minColumnNames = new ArrayList<>(colCount);
    for (int i = 0; i < colCount; i++)
        minColumnNames.add(ByteBufferUtil.readWithShortLength(in));

    colCount = in.readInt();
    List<ByteBuffer> maxColumnNames = new ArrayList<>(colCount);
    for (int i = 0; i < colCount; i++)
        maxColumnNames.add(ByteBufferUtil.readWithShortLength(in));

    boolean hasLegacyCounterShards = true;
    if (version.tracksLegacyCounterShards)
        hasLegacyCounterShards = in.readBoolean();

    return new StatsMetadata(rowSizes,
                             columnCounts,
                             replayPosition,
                             minTimestamp,
                             maxTimestamp,
                             maxLocalDeletionTime,
                             compressionRatio,
                             tombstoneHistogram,
                             sstableLevel,
                             minColumnNames,
                             maxColumnNames,
                             hasLegacyCounterShards,
                             repairedAt);
}
 

public AtomDeserializer getOnDiskDeserializer(DataInput in, Descriptor.Version version)
{
    return new AtomDeserializer(comparator, in, ColumnSerializer.Flag.LOCAL, Integer.MIN_VALUE, version);
}
 

public Iterator<OnDiskAtom> getOnDiskIterator(DataInput in, ColumnSerializer.Flag flag, int expireBefore, Descriptor.Version version)
{
    return AbstractCell.onDiskIterator(in, flag, expireBefore, version, comparator);
}
 

public void skipBody(DataInput in, Descriptor.Version version) throws IOException
{
    type.serializer().skip(in);
    DeletionTime.serializer.skip(in);
}
 

public RangeTombstone deserializeBody(DataInput in, Composite min, Descriptor.Version version) throws IOException
{
    Composite max = type.serializer().deserialize(in);
    DeletionTime dt = DeletionTime.serializer.deserialize(in);
    return new RangeTombstone(min, max, dt);
}
 

public OnDiskAtom deserializeFromSSTable(DataInput in, Descriptor.Version version) throws IOException
{
    return deserializeFromSSTable(in, ColumnSerializer.Flag.LOCAL, Integer.MIN_VALUE, version);
}
 

@Override
public int run(String[] args) throws Exception {
    Job job = Job.getInstance(getConf());
    Configuration configuration = job.getConfiguration();

    job.setJarByClass(Aegisthus.class);
    CommandLine cl = getOptions(args);
    if (cl == null) {
        return 1;
    }

    // Check all of the paths and load the sstable version from the input filenames
    List<Path> paths = Lists.newArrayList();
    if (cl.hasOption(Feature.CMD_ARG_INPUT_FILE)) {
        for (String input : cl.getOptionValues(Feature.CMD_ARG_INPUT_FILE)) {
            paths.add(new Path(input));
        }
    }
    if (cl.hasOption(Feature.CMD_ARG_INPUT_DIR)) {
        paths.addAll(getDataFiles(configuration, cl.getOptionValue(Feature.CMD_ARG_INPUT_DIR)));
    }
    LOG.info("Processing paths: {}", paths);

    // At this point we have the version of sstable that we can use for this run
    Descriptor.Version version = Descriptor.Version.CURRENT;
    if (cl.hasOption(Feature.CMD_ARG_SSTABLE_OUTPUT_VERSION)) {
        version = new Descriptor.Version(cl.getOptionValue(Feature.CMD_ARG_SSTABLE_OUTPUT_VERSION));
    }
    configuration.set(Feature.CONF_SSTABLE_VERSION, version.toString());

    if (configuration.get(Feature.CONF_CQL_SCHEMA) != null) {
        setConfigurationFromCql(configuration);
    }

    if(cl.hasOption(Feature.CMD_ARG_COMBINE_SPLITS)) {
        job.setInputFormatClass(AegisthusCombinedInputFormat.class);
    } else {
        job.setInputFormatClass(AegisthusInputFormat.class);
    }
    job.setMapOutputKeyClass(AegisthusKey.class);
    job.setMapOutputValueClass(AtomWritable.class);
    job.setOutputKeyClass(AegisthusKey.class);
    job.setOutputValueClass(RowWritable.class);
    job.setMapperClass(AegisthusKeyMapper.class);
    job.setReducerClass(CassSSTableReducer.class);
    job.setGroupingComparatorClass(AegisthusKeyGroupingComparator.class);
    job.setPartitionerClass(AegisthusKeyPartitioner.class);
    job.setSortComparatorClass(AegisthusKeySortingComparator.class);

    TextInputFormat.setInputPaths(job, paths.toArray(new Path[paths.size()]));

    if (cl.hasOption(Feature.CMD_ARG_PRODUCE_SSTABLE)) {
        job.setOutputFormatClass(SSTableOutputFormat.class);
    } else {
        job.setOutputFormatClass(JsonOutputFormat.class);
    }
    CustomFileNameFileOutputFormat.setOutputPath(job, new Path(cl.getOptionValue(Feature.CMD_ARG_OUTPUT_DIR)));

    job.submit();
    if (configuration.getBoolean(Feature.CONF_SHUTDOWN_HOOK, true)) {
        Runtime.getRuntime().addShutdownHook(new JobKiller(job));
    }

    System.out.println(job.getJobID());
    System.out.println(job.getTrackingURL());
    boolean success = job.waitForCompletion(true);

    if (success) {
        Counter errorCounter = job.getCounters().findCounter("aegisthus", "error_skipped_input");
        long errorCount = errorCounter != null ? errorCounter.getValue() : 0L;
        int maxAllowed = configuration.getInt(Feature.CONF_MAX_CORRUPT_FILES_TO_SKIP, 0);
        if (errorCounter != null && errorCounter.getValue() > maxAllowed) {
            LOG.error("Found {} corrupt files which is greater than the max allowed {}", errorCount, maxAllowed);
            success = false;
        } else if (errorCount > 0) {
            LOG.warn("Found {} corrupt files but not failing the job because the max allowed is {}",
                    errorCount, maxAllowed);
        }
    }

    return success ? 0 : 1;
}
 

public ColumnFamily deserializeFromSSTable(DataInput in, Descriptor.Version version)
{
    throw new UnsupportedOperationException();
}
 

/**
 * Deserialize into the specified DataInputStream instance in the format
 * suited for SSTables.
 * @param in DataInput from which deserialization needs to happen.
 * @param version the version for the sstable we're reading from
 * @throws IOException
 * @return the type that was deserialized
 */
public T deserializeFromSSTable(DataInput in, Descriptor.Version version) throws IOException;
 

/**
 * Deserialize metadata component from given input.
 *
 * @param version serialize version
 * @param in deserialize source
 * @return Deserialized component
 * @throws IOException
 */
T deserialize(Descriptor.Version version, DataInput in) throws IOException;