org.apache.commons.io.output.CountingOutputStream Java Examples

public S3TransportBuffer(long maxBytes, boolean useCompression, S3TransportSerializer serializer)
    throws TransportException {
  this.maxBytes = maxBytes;
  this.serializer = serializer;

  baos = new ByteArrayOutputStream();
  cos = new CountingOutputStream(baos);

  if (useCompression) {
    this.isCompressed = true;
    try {
      os = new BZip2CompressorOutputStream(cos);
    } catch (IOException e) {
      throw new TransportException("unable to create BZip2CompressorOutputStream", e);
    }
  } else {
    this.isCompressed = false;
    os = cos;
  }
}
 

VideoRecordingSession(
   VideoSessionRegistry registry,
   ChannelHandlerContext ctx,
   VideoRecordingManager dao,
   RecordingEventPublisher eventPublisher,
   VideoStorageSession storage,
   double precapture,
   boolean stream,
   CountingOutputStream mpegts,
   double flushTimeInS
) throws IOException {
   super(registry, ctx, storage, dao, eventPublisher, stream, precapture, true);

   this.adapter = new Adapter(mpegts, flushTimeInS, storage.getRecordingId());
   this.storage = storage;
   this.mpegts = mpegts;
}
 

public void open(String loc) throws IOException {
	channel = new RandomAccessFile(loc + ".dat", "rw").getChannel();
	CountingInputStream countertmp = new CountingInputStream(new BufferedInputStream(Channels.newInputStream(channel), buffersize));
	DataInputStream is = new DataInputStream(countertmp);
	nrRows = is.readInt();
	nrCols = is.readInt();
	hashRows = DoubleMatrixDataset.loadIdentifiers(loc + ".rows.txt");
	hashCols = DoubleMatrixDataset.loadIdentifiers(loc + ".cols.txt");

	headerLen = 8;
	currentPos = headerLen;
	bytesPerRow = 8 * nrCols;
	buffersize = bytesPerRow * 10;
	channel.position(currentPos);
	counter = new CountingOutputStream(new BufferedOutputStream(Channels.newOutputStream(channel), buffersize));
	os = new DataOutputStream(counter);

	System.out.println("Read header. current pos: " + channel.position());
	System.out.println("Header: " + headerLen);
}
 

DataLakeDataGenerator(
    String filePath,
    OutputStreamHelper outputStreamHelper,
    DataGeneratorFormatConfig dataFormatConfig,
    long idleTimeSecs
) throws IOException, StageException {
  this.filePath = filePath;
  this.outputStreamHelper = outputStreamHelper;
  this.cos = new CountingOutputStream(outputStreamHelper.getOutputStream(filePath));
  this.generator = dataFormatConfig.getDataGeneratorFactory().getGenerator(
      cos,
      outputStreamHelper.getStreamCloseEventHandler()
  );
  this.idleTimeSecs = idleTimeSecs;
  this.recordCount = new AtomicLong(0L);
  this.idleClosed = false;
}
 

/**
 * Custom serialization method.
 *
 * @param oos the object output stream
 */
private void writeObject(ObjectOutputStream oos) throws IOException {
  // figure out size of the written network
  CountingOutputStream cos = new CountingOutputStream(new NullOutputStream());
  if (isInitializationFinished) {
    ModelSerializer.writeModel(model, cos, false);
  }
  modelSize = cos.getByteCount();

  // default serialization
  oos.defaultWriteObject();

  // Write layer configurations
  String[] layerConfigs = new String[layers.length];
  for (int i = 0; i < layers.length; i++) {
    layerConfigs[i] =
        layers[i].getClass().getName() + "::"
            + weka.core.Utils.joinOptions(layers[i].getOptions());
  }
  oos.writeObject(layerConfigs);

  // actually write the network
  if (isInitializationFinished) {
    ModelSerializer.writeModel(model, oos, false);
  }
}
 

/**
 * Custom serialization method.
 *
 * @param oos the object output stream
 */
private void writeObject(ObjectOutputStream oos) throws IOException {
  // figure out size of the written network
  CountingOutputStream cos = new CountingOutputStream(new NullOutputStream());
  if (isInitializationFinished) {
    ModelSerializer.writeModel(model, cos, false);
  }
  modelSize = cos.getByteCount();

  // default serialization
  oos.defaultWriteObject();

  // Write layer configurations
  String[] layerConfigs = new String[layers.length];
  for (int i = 0; i < layers.length; i++) {
    layerConfigs[i] =
        layers[i].getClass().getName() + "::"
            + weka.core.Utils.joinOptions(layers[i].getOptions());
  }
  oos.writeObject(layerConfigs);

  // actually write the network
  if (isInitializationFinished) {
    ModelSerializer.writeModel(model, oos, false);
  }
}
 

public void write(int row, int col, double val) throws IOException {
		long seekLoc = ((long) row * bytesPerRow) + headerLen + (col * 8);
//        System.out.println(row + "\t" + col + "\t" + seekLoc + "\t" + currentPos + "\t" + val);
		if (seekLoc - currentPos == 0) {

			os.writeDouble(val);
			os.flush();
			currentPos = seekLoc + 8;

		} else {
			if (seekLoc > channel.size()) {
				throw new IllegalArgumentException("Seek location for row: " + row + ", " + seekLoc + " is outside file size: " + channel.size());
			}

			if (singledouble == null) {
				singledouble = ByteBuffer.allocate(8);
			}

			singledouble.putDouble(val);
			singledouble.flip();

//            System.out.println("Seeking: " + seekLoc);
			channel.position(seekLoc);

			channel.write(singledouble);
			currentPos = seekLoc + 8;
			singledouble.compact();


			// this is probably extremely slow?
			counter = new CountingOutputStream(new BufferedOutputStream(Channels.newOutputStream(channel), buffersize));
			os = new DataOutputStream(counter);
		}

	}
 

public static IDownloadIndicator downloadFile(URL url, Path out) {
    DownloadTracker tracker = new DownloadTracker();
    new Thread(() -> {
        tracker.init(getFileSize(url));
        try (InputStream input = url.openStream()) {
            Path downloads = Paths.get("downloads");
            Files.createDirectories(downloads);

            Path temp = downloads.resolve(UUID.randomUUID().toString() + ".sbxdl");
            int totalDataRead = 0;
            try (CountingOutputStream output = new CountingOutputStream(Files.newOutputStream(temp))) {
                byte[] data = new byte[1024];
                int b;
                while ((b = input.read(data, 0, 1024)) >= 0) {
                    output.write(data);
                    totalDataRead += b;
                    tracker.set(totalDataRead);
                }
            }
            if (Files.exists(out))
                Files.delete(out);
            if (Files.notExists(out.getParent()))
                Files.createDirectories(out.getParent());
            Files.move(temp, out);
            tracker.complete();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            tracker.complete();
        }
    }).start();
    return tracker;
}
 

public void writeRow(int row, double[] cols) throws IOException {
	long seekLoc = ((long) row * bytesPerRow) + headerLen;

	if (seekLoc > channel.size()) {
		throw new IllegalArgumentException("Seek location for row: " + row + ", " + seekLoc + " is outside file size: " + channel.size());
	}

	// if row is the next row, just write.
	if (seekLoc - currentPos == 0) {

		writeRow(cols);
	} else {
		// else, random access to new location

		channel.position(seekLoc);
		if (bytebuffer == null) {
			bytebuffer = ByteBuffer.wrap(new byte[bytesPerRow]);

		}
		channel.write(bytebuffer);

		// this is probably extremely slow?
		counter = new CountingOutputStream(new BufferedOutputStream(Channels.newOutputStream(channel), buffersize));
		os = new DataOutputStream(counter);
		currentPos = seekLoc + bytesPerRow;


	}
}
 

public RecordWriter(Path path, long timeToLiveMillis, OutputStream textOutputStream,
                    DataGeneratorFactory generatorFactory, StreamCloseEventHandler streamCloseEventHandler) throws StageException, IOException {
  this(path, timeToLiveMillis, generatorFactory);
  this.textOutputStream = new CountingOutputStream(textOutputStream);
  generator = generatorFactory.getGenerator(this.textOutputStream, streamCloseEventHandler);
  textFile = true;
  this.idleTimeout = -1L;
}
 

@Test
public void downloadMethodShouldCreateLocalFileStreamContainingProgressListener() throws IOException {

    FTPFile file = new FTPFile("remote.file", 0l, "path/to", 0, false);

    ftpConnection.setProgressListener(new ProgressListener());
    ftpConnection.download(file, LOCAL_DIRECTORY);

    verify(mockFileStreamFactory).createOutputStream(LOCAL_DIRECTORY + "/remote.file");
    verify(mockFtpClient).retrieveFile(eq("path/to/remote.file"), any(CountingOutputStream.class));
}
 

public FileHolder() throws IOException {
  super();
  this.plainTempFile = Files.createTempFile("", "");
  this.plainStream = new CountingOutputStream(Files.newOutputStream(plainTempFile));
  this.gzTempFile = Files.createTempFile("", "");
  this.gzStream = new CountingOutputStream(Files.newOutputStream(gzTempFile));
  this.bzTempFile = Files.createTempFile("", "");
  this.bzStream = new CountingOutputStream(Files.newOutputStream(bzTempFile));
}
 

public FileHolder() throws IOException {
  super();
  this.plainTempFile = Files.createTempFile("", "");
  this.plainStream = new CountingOutputStream(Files.newOutputStream(plainTempFile));
  this.gzTempFile = Files.createTempFile("", "");
  this.gzStream = new CountingOutputStream(Files.newOutputStream(gzTempFile));
  this.bzTempFile = Files.createTempFile("", "");
  this.bzStream = new CountingOutputStream(Files.newOutputStream(bzTempFile));
}
 

/** Initialize the OutputStream to the next file to write to.
 */
private void openNextFile() throws IOException {
  StringBuffer sb = new StringBuffer();
  Formatter fmt = new Formatter(sb);
  fmt.format("%05d", this.fileNum++);
  String filename = filePrefix + fmt.toString();
  if (codec != null) {
    filename = filename + codec.getDefaultExtension();
  }
  Path destFile = new Path(destDir, filename);
  FileSystem fs = destFile.getFileSystem(conf);
  LOG.debug("Opening next output file: " + destFile);
  if (fs.exists(destFile)) {
    Path canonicalDest = destFile.makeQualified(fs);
    throw new IOException("Destination file " + canonicalDest
        + " already exists");
  }

  OutputStream fsOut = fs.create(destFile);

  // Count how many actual bytes hit HDFS.
  this.countingFilterStream = new CountingOutputStream(fsOut);

  if (codec != null) {
    // Wrap that in a compressing stream.
    this.writeStream = codec.createOutputStream(this.countingFilterStream);
  } else {
    // Write to the counting stream directly.
    this.writeStream = this.countingFilterStream;
  }
}
 

@Override
/**
 * {@inheritDoc}
 */
public OutputStream writeBlobRecord(long claimedLen) throws IOException {
  finishRecord(); // finish any previous record.
  checkForNull(this.out);
  startRecordIndex();
  this.header.getStartMark().write(out);
  LOG.debug("Starting new record; id=" + curEntryId
      + "; claimedLen=" + claimedLen);
  WritableUtils.writeVLong(out, curEntryId);
  WritableUtils.writeVLong(out, claimedLen);
  this.curClaimedLen = claimedLen;
  this.userCountingOutputStream = new CountingOutputStream(
      new CloseShieldOutputStream(out));
  if (null == this.codec) {
    // No codec; pass thru the same OutputStream to the user.
    this.userOutputStream = this.userCountingOutputStream;
  } else {
    // Wrap our CountingOutputStream in a compressing OutputStream to
    // give to the user.
    this.compressor.reset();
    this.userOutputStream = new CompressorStream(
        this.userCountingOutputStream, compressor);
  }

  return this.userOutputStream;
}
 

/**
 * Open the file and write its header.
 */
private void init() throws IOException {
  FileSystem fs = this.path.getFileSystem(conf);
  FSDataOutputStream fsOut = fs.create(this.path);
  this.countingOut = new CountingOutputStream(
      new BufferedOutputStream(fsOut));
  this.out = new DataOutputStream(this.countingOut);

  // put any necessary config strings into the header.
  MetaBlock m = this.header.getMetaBlock();
  if (isCharData) {
    m.put(MetaBlock.ENTRY_ENCODING_KEY, MetaBlock.CLOB_ENCODING);
  } else {
    m.put(MetaBlock.ENTRY_ENCODING_KEY, MetaBlock.BLOB_ENCODING);
  }

  if (null != codec) {
    m.put(MetaBlock.COMPRESSION_CODEC_KEY, this.codecName);
  }

  // Serialize the value of maxEntriesPerSegment as a VInt in a byte array
  // and put that into the metablock as ENTRIES_PER_SEGMENT_KEY.
  int segmentBufLen = WritableUtils.getVIntSize(this.maxEntriesPerSegment);
  DataOutputBuffer entriesPerSegBuf = new DataOutputBuffer(segmentBufLen);
  WritableUtils.writeVInt(entriesPerSegBuf, this.maxEntriesPerSegment);
  byte [] entriesPerSegArray =
      Arrays.copyOf(entriesPerSegBuf.getData(), segmentBufLen);
  m.put(MetaBlock.ENTRIES_PER_SEGMENT_KEY,
      new BytesWritable(entriesPerSegArray));

  // Write the file header to the file.
  this.header.write(out);

  // Now we're ready to accept record data from the user.
}
 

@Override
protected VideoRecordingSession create(VideoSessionRegistry registry, ChannelHandlerContext ctx, VideoRecordingManager dao, RecordingEventPublisher eventPublisher, VideoStorageSession storage, double precapture, boolean stream) {
   try {
      return new VideoRecordingSession(registry, ctx, dao, eventPublisher, storage, precapture, stream, new CountingOutputStream(storage.output()), config.getVideoFlushFrequency());
   } catch(Exception ioe) {
      throw new RuntimeException(ioe);
   }
}
 

private CountingOutputStream listenOn(OutputStream outputStream) {

        LOGGER.debug("Creating wrapping output stream for progress listener");

        CountingOutputStream countingStream = new CountingOutputStream(outputStream) {

            @Override
            protected void beforeWrite(int n) {

                super.beforeWrite(n);
                progressListener.setBytesWritten(getByteCount());
            }
        };

        return countingStream;
    }
 

protected MpegTsH264Adapter(CountingOutputStream mpegts, double flushTimeInS, UUID recId) throws IOException {
   super(new DataOutputStream(mpegts), recId);
   this.mpegts = mpegts;

   this.flushFrequencyIn90KHz = (long)(flushTimeInS * 90000);
}
 

Adapter(CountingOutputStream mpegts, double flushTimeInS, UUID recId) throws IOException {
   super(mpegts, flushTimeInS, recId);
}