Java Code Examples for org.apache.cassandra.io.sstable.Descriptor#fromFilename()

private static Stream.TableProgress getTableProgressFromFile(ProgressInfo progressInfo) {
  // the value of ProgressInfo.fileName is different depending on the direction of the stream
  // when sending, the file name is an absolute path of a SSTable
  // when receiving, it's just keyspace/table (probably because C* doesn't know the final file name yet)
  String ksTable;
  if (progressInfo.direction == ProgressInfo.Direction.OUT) {
    Descriptor descriptor = Descriptor.fromFilename(progressInfo.fileName);
    ksTable = String.format("%s.%s", descriptor.ksname, descriptor.cfname);
  } else {
    ksTable = progressInfo.fileName.replace('/', '.');
  }

  long currentBytes = progressInfo.currentBytes;
  long totalBytes = progressInfo.totalBytes;

  return Stream.TableProgress.builder()
      .withTable(ksTable)
      .withCurrent(currentBytes)
      .withTotal(totalBytes)
      .build();
}
 

public void cleanup()
{
    List<String> files = readLockfile(lockfile);
    for (String file : files)
    {
        try
        {
            Descriptor desc = Descriptor.fromFilename(file, true);
            SSTable.delete(desc, SSTable.componentsFor(desc));
        }
        catch (Exception e)
        {
            JVMStabilityInspector.inspectThrowable(e);
            logger.warn("failed to delete a potentially stale sstable {}", file);
        }
    }
}
 

public Compact(String... args) {
    for(String path : args) {
        try {
            for (File f : CassandraUtils.sstablesFromPath(path)) {
                if (metadata == null) {
                    metadata = CassandraUtils.tableFromSSTable(f);
                }
                Descriptor d = Descriptor.fromFilename(f.getAbsolutePath());
                sstables.add(SSTableReader.openNoValidation(d, metadata));
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
 

protected SSTableWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt) throws IOException
{
    Directories.DataDirectory localDir = cfs.directories.getWriteableLocation(totalSize);
    if (localDir == null)
        throw new IOException("Insufficient disk space to store " + totalSize + " bytes");
    desc = Descriptor.fromFilename(cfs.getTempSSTablePath(cfs.directories.getLocationForDisk(localDir)));

    return new SSTableWriter(desc.filenameFor(Component.DATA), estimatedKeys, repairedAt);
}
 

private void checkVersionFromFilename(String filename) {
    Descriptor descriptor = Descriptor.fromFilename(filename);

    if (this.version == null) {
        this.version = descriptor.version;
    } else if (!this.version.equals(descriptor.version)) {
        throw new IllegalStateException("All files must have the same sstable version.  File '" + filename
                + "' has version '" + descriptor.version + "' and we have already seen a file with version '"
                + version + "'");
    }
}
 

@SuppressWarnings("unchecked")
public static CFMetaData tableFromSSTable(File path) throws IOException, NoSuchFieldException, IllegalAccessException {
    Preconditions.checkNotNull(path);
    Descriptor desc = Descriptor.fromFilename(path.getAbsolutePath());

    EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
    Map<MetadataType, MetadataComponent> sstableMetadata = desc.getMetadataSerializer().deserialize(desc, types);
    ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
    Preconditions.checkNotNull(validationMetadata, "Validation Metadata could not be resolved, accompanying Statistics.db file must be missing.");
    SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
    Preconditions.checkNotNull(header, "Metadata could not be resolved, accompanying Statistics.db file must be missing.");

    IPartitioner partitioner = validationMetadata.partitioner.endsWith("LocalPartitioner") ?
            new LocalPartitioner(header.getKeyType()) :
            FBUtilities.newPartitioner(validationMetadata.partitioner);

    DatabaseDescriptor.setPartitionerUnsafe(partitioner);
    AbstractType<?> keyType = header.getKeyType();
    List<AbstractType<?>> clusteringTypes = header.getClusteringTypes();
    Map<ByteBuffer, AbstractType<?>> staticColumns = header.getStaticColumns();
    Map<ByteBuffer, AbstractType<?>> regularColumns = header.getRegularColumns();
    int id = cfCounter.incrementAndGet();
    CFMetaData.Builder builder = CFMetaData.Builder.create("turtle" + id, "turtles" + id);
    staticColumns.entrySet().stream()
            .forEach(entry ->
                    builder.addStaticColumn(UTF8Type.instance.getString(entry.getKey()), entry.getValue()));
    regularColumns.entrySet().stream()
            .forEach(entry ->
                    builder.addRegularColumn(UTF8Type.instance.getString(entry.getKey()), entry.getValue()));
    List<AbstractType<?>> partTypes = keyType.getComponents();
    for(int i = 0; i < partTypes.size(); i++) {
        builder.addPartitionKey("partition" + (i > 0 ? i : ""), partTypes.get(i));
    }
    for (int i = 0; i < clusteringTypes.size(); i++) {
        builder.addClusteringColumn("row" + (i > 0 ? i : ""), clusteringTypes.get(i));
    }
    CFMetaData metaData = builder.build();
    Schema.instance.setKeyspaceMetadata(KeyspaceMetadata.create(metaData.ksName, KeyspaceParams.local(),
            Tables.of(metaData), Views.none(), getTypes(), Functions.none()));
    return metaData;
}
 

/**
 * @param args a list of sstables whose metadata we're interested in
 */
public static void main(String[] args) throws IOException
{
    PrintStream out = System.out;
    if (args.length == 0)
    {
        out.println("Usage: sstablemetadata <sstable filenames>");
        System.exit(1);
    }

    for (String fname : args)
    {
        if (new File(fname).exists())
        {
            Descriptor descriptor = Descriptor.fromFilename(fname);
            Map<MetadataType, MetadataComponent> metadata = descriptor.getMetadataSerializer().deserialize(descriptor, EnumSet.allOf(MetadataType.class));
            ValidationMetadata validation = (ValidationMetadata) metadata.get(MetadataType.VALIDATION);
            StatsMetadata stats = (StatsMetadata) metadata.get(MetadataType.STATS);
            CompactionMetadata compaction = (CompactionMetadata) metadata.get(MetadataType.COMPACTION);

            out.printf("SSTable: %s%n", descriptor);
            if (validation != null)
            {
                out.printf("Partitioner: %s%n", validation.partitioner);
                out.printf("Bloom Filter FP chance: %f%n", validation.bloomFilterFPChance);
            }
            if (stats != null)
            {
                out.printf("Minimum timestamp: %s%n", stats.minTimestamp);
                out.printf("Maximum timestamp: %s%n", stats.maxTimestamp);
                out.printf("SSTable max local deletion time: %s%n", stats.maxLocalDeletionTime);
                out.printf("Compression ratio: %s%n", stats.compressionRatio);
                out.printf("Estimated droppable tombstones: %s%n", stats.getEstimatedDroppableTombstoneRatio((int) (System.currentTimeMillis() / 1000)));
                out.printf("SSTable Level: %d%n", stats.sstableLevel);
                out.printf("Repaired at: %d%n", stats.repairedAt);
                out.println(stats.replayPosition);
                out.println("Estimated tombstone drop times:%n");
                for (Map.Entry<Double, Long> entry : stats.estimatedTombstoneDropTime.getAsMap().entrySet())
                {
                    out.printf("%-10s:%10s%n",entry.getKey().intValue(), entry.getValue());
                }
                printHistograms(stats, out);
            }
            if (compaction != null)
            {
                out.printf("Ancestors: %s%n", compaction.ancestors.toString());
                out.printf("Estimated cardinality: %s%n", compaction.cardinalityEstimator.cardinality());

            }
        }
        else
        {
            out.println("No such file: " + fname);
        }
    }
}
 

/**
 * @param args a list of sstables whose metadata we are changing
 */
public static void main(final String[] args) throws IOException
{
    PrintStream out = System.out;
    if (args.length == 0)
    {
        out.println("This command should be run with Cassandra stopped!");
        out.println("Usage: sstablerepairedset [--is-repaired | --is-unrepaired] [-f <sstable-list> | <sstables>]");
        System.exit(1);
    }

    if (args.length < 3 || !args[0].equals("--really-set") || (!args[1].equals("--is-repaired") && !args[1].equals("--is-unrepaired")))
    {
        out.println("This command should be run with Cassandra stopped, otherwise you will get very strange behavior");
        out.println("Verify that Cassandra is not running and then execute the command like this:");
        out.println("Usage: sstablerepairedset --really-set [--is-repaired | --is-unrepaired] [-f <sstable-list> | <sstables>]");
        System.exit(1);
    }

    boolean setIsRepaired = args[1].equals("--is-repaired");

    List<String> fileNames;
    if (args[2].equals("-f"))
    {
        fileNames = Files.readAllLines(Paths.get(args[3]), Charset.defaultCharset());
    }
    else
    {
        fileNames = Arrays.asList(args).subList(2, args.length);
    }

    for (String fname: fileNames)
    {
        Descriptor descriptor = Descriptor.fromFilename(fname);
        if (descriptor.version.hasRepairedAt)
        {
            if (setIsRepaired)
            {
                FileTime f = Files.getLastModifiedTime(new File(descriptor.filenameFor(Component.DATA)).toPath());
                descriptor.getMetadataSerializer().mutateRepairedAt(descriptor, f.toMillis());
            }
            else
            {
                descriptor.getMetadataSerializer().mutateRepairedAt(descriptor, ActiveRepairService.UNREPAIRED_SSTABLE);
            }
        }
        else
        {
            System.err.println("SSTable " + fname + " does not have repaired property, run upgradesstables");
        }
    }
}
 

/**
 * Counts the number of tombstones, per row, in a given SSTable
 *
 * Assumes RandomPartitioner, standard columns and UTF8 encoded row keys
 *
 * Does not require a cassandra.yaml file or system tables.
 *
 * @param args command lines arguments
 *
 * @throws java.io.IOException on failure to open/read/write files or output streams
 */
public static void main(String[] args) throws IOException, ParseException {
  String usage = String.format("Usage: %s [-l] <sstable> [<sstable> ...]%n", CountTombstones.class.getName());

  final Options options = new Options();
  options.addOption("l", "legend", false, "Include column name explanation");
  options.addOption("p", "partitioner", true, "The partitioner used by database");

  CommandLineParser parser = new BasicParser();
  CommandLine cmd = parser.parse(options, args);

  if (cmd.getArgs().length < 1)
  {
    System.err.println("You must supply at least one sstable");
    System.err.println(usage);
    System.exit(1);
  }

  // Fake DatabaseDescriptor settings so we don't have to load cassandra.yaml etc
  Config.setClientMode(true);
  String partitionerName = String.format("org.apache.cassandra.dht.%s",
                                         cmd.hasOption("p") ? cmd.getOptionValue("p") : "RandomPartitioner");
  try {
    Class<?> clazz = Class.forName(partitionerName);
    IPartitioner partitioner = (IPartitioner) clazz.newInstance();
    DatabaseDescriptor.setPartitioner(partitioner);
  } catch (Exception e) {
    throw new RuntimeException("Can't instantiate partitioner " + partitionerName);
  }

  PrintStream out = System.out;

  for (String arg : cmd.getArgs()) {
    String ssTableFileName = new File(arg).getAbsolutePath();

    Descriptor descriptor = Descriptor.fromFilename(ssTableFileName);

    run(descriptor, cmd, out);
  }

  System.exit(0);
}
 

@Override
public void initialize(@Nonnull InputSplit inputSplit, @Nonnull final TaskAttemptContext ctx)
        throws IOException, InterruptedException {
    Configuration conf = ctx.getConfiguration();
    final AegSplit split = (AegSplit) inputSplit;

    long start = split.getStart();
    InputStream is = split.getInput(conf);
    long end = split.getDataEnd();
    URI fileUri = split.getPath().toUri();
    String filename = fileUri.toString();
    sourcePath = fileUri.getPath();
    traceDataFromSource = conf.getBoolean(Aegisthus.Feature.CONF_TRACE_DATA_FROM_SOURCE, false);

    LOG.info("File: {}", sourcePath);
    LOG.info("Start: {}", start);
    LOG.info("End: {}", end);

    try {
        Descriptor.Version version = Descriptor.Version.CURRENT;
        try {
            version = Descriptor.fromFilename(filename).version;
        } catch (Exception ignored) {
            // The 2.0 fromFilename parser fails on the latest Cassandra filenames, ignore this error and uses latest
        }
        scanner = new SSTableColumnScanner(is, start, end, version);
        LOG.info("Creating observable");
        rx.Observable<AtomWritable> observable = scanner.observable();
        observable = observable
                .onErrorFlatMap(new Func1<OnErrorThrowable, Observable<? extends AtomWritable>>() {
                    @Override
                    public Observable<? extends AtomWritable> call(OnErrorThrowable onErrorThrowable) {
                        LOG.error("failure deserializing file {}", split.getPath(), onErrorThrowable);
                        ctx.getCounter("aegisthus", "error_skipped_input").increment(1L);
                        return Observable.empty();
                    }
                });

        iterator = ObservableToIterator.toIterator(observable);
        LOG.info("done initializing");
    } catch (IOException e) {
        throw new IOError(e);
    }
}
 

/**
 * Create metadata about given compressed file including uncompressed data length, chunk size
 * and list of the chunk offsets of the compressed data.
 *
 * This is an expensive operation! Don't create more than one for each
 * sstable.
 *
 * @param dataFilePath Path to the compressed file
 *
 * @return metadata about given compressed file.
 */
public static CompressionMetadata create(String dataFilePath)
{
    Descriptor desc = Descriptor.fromFilename(dataFilePath);
    return new CompressionMetadata(desc.filenameFor(Component.COMPRESSION_INFO), new File(dataFilePath).length(), desc.version.hasPostCompressionAdlerChecksums);
}