stream#Transform TypeScript Examples

/**
 * Transform stream that only stream the first X number of bytes.
 */
export class PreviewStream extends Transform {
  constructor({ peek, ...opts }: PreviewStreamOptions) {
    // acts like passthrough
    let transform: TransformOptions['transform'] = (chunk, _encoding, callback) =>
      callback(undefined, chunk);
    // implements preview - peek must be positive number
    if (peek && peek > 0) {
      let size = 0;
      transform = (chunk, _encoding, callback) => {
        const delta = peek - size;
        size += chunk.length;
        if (size >= peek) {
          callback(undefined, chunk.slice(0, delta));
          this.resume(); // do not handle any subsequent data
          return;
        }
        callback(undefined, chunk);
      };
    }
    super({ ...opts, transform });
  }
}

private constructor({
    collator,
    decorator,
    indexer,
  }: {
    collator?: Readable;
    decorator?: Transform;
    indexer?: Writable;
  }) {
    this.collator = collator;
    this.decorator = decorator;
    this.indexer = indexer;
  }

private initRecord = () => {
    if (!this.connection || !vp) {
      return;
    }

    this.connection.dispatcher.end();

    const recorder = pipeline(
      (this.recordStream = vp.createRecordStream()).stream, // Open a recording stream
      ((this.recordVolumeTransformer = new VolumeTransformer({
        type: 's16le',
      })) as unknown) as Transform,
      // Change the volume
      (this.recordNoiseReductor = new NoiseReductor(this.onSpeaking.bind(this))), // Audio gate
      (err) => err && debug("Couldn't finish recording pipeline.", err)
    );

    this.connection.play(recorder, { bitrate: 256, type: 'converted', highWaterMark: 0 });
  };

private initPlayback = () => {
    if (!this.connection || !this.channel || !vp) {
      return;
    }

    const { channel } = this;

    this.mixer?.close();
    this.playbackStream?.end();
    this.streams.clear();

    pipeline(
      (this.mixer = new Mixer(MIXER_OPTIONS)), // Initialize the member streams mixer
      ((this.playbackVolumeTransformer = new VolumeTransformer({
        type: 's16le',
      })) as unknown) as Transform, // Change the volume
      (this.playbackStream = vp.createPlaybackStream()).stream, // Output to a playback stream
      (err) => err && debug("Couldn't finish playback pipeline.", err)
    );

    for (const member of channel.members.filter((m) => m.user !== app.client.user).values()) {
      this.connectToMixer(member);
    }
  };

spawnPromise = (
    command: string,
    args: string[] = [],
    options: SpawnOptionsWithoutStdio = {},
    stream: ReadStream | Transform | undefined = undefined,
): Promise<string> => {
    return new Promise((resolve, reject) => {
        let commandEscaped = command;
        let argsEscaped = args;
        if (process.platform === 'win32') {
            // https://nodejs.org/api/child_process.html#child_process_spawning_bat_and_cmd_files_on_windows
            commandEscaped = `"${command}"`;
            argsEscaped = args.map((arg) => `"${arg}"`);
            options.shell = true;
        }

        const child = spawn(commandEscaped, argsEscaped, options);

        if (stream) {
            stream.pipe(child.stdin);
        }

        const data: string[] = [];
        const collect = (chunk: Buffer): void => {
            data.push(chunk.toString());
        };

        child.stderr.on('data', collect);
        child.stderr.on('error', reject);
        child.stderr.on('close', () => resolve(data.join('')));
        child.stderr.on('end', () => resolve(data.join('')));

        child.stdout.on('data', collect);
        child.stdout.on('error', reject);
        child.stdout.on('close', () => resolve(data.join('')));
        child.stdout.on('end', () => resolve(data.join('')));
    });
}

appendSemicolon = (): Transform => {
    let finalChunk: string;
    return new Transform({
        emitClose: true,
        transform: (data, _, next) => {
            finalChunk = data;
            next(null, data);
        },
        flush: (next) => {
            if (finalChunk !== undefined) {
                const finalChunkString = Buffer.isBuffer(finalChunk) ? finalChunk.toString() : finalChunk;
                if (!finalChunkString.trim().endsWith(';')) {
                    next(null, ';');
                } else {
                    next();
                }
            } else {
                next();
            }
        },
    });
}

/**
 * Returns a stream from an object in a s3 compatible object store (e.g. minio).
 * The actual content of the stream depends on the object.
 *
 * Any gzipped or deflated objects will be ungzipped or inflated. If the object
 * is a tarball, only the content of the first record in the tarball will be
 * returned. For any other objects, the raw content will be returned.
 *
 * @param param.bucket Bucket name to retrieve the object from.
 * @param param.key Key of the object to retrieve.
 * @param param.client Minio client.
 *
 */
export async function getObjectStream({
  bucket,
  key,
  client,
}: MinioRequestConfig): Promise<Transform> {
  const stream = await client.getObject(bucket, key);
  return stream.pipe(gunzip()).pipe(maybeTarball());
}

/**
 * Returns a transform stream where the first record inside a tarball will be
 * pushed - i.e. all other contents will be dropped.
 */
function extractFirstTarRecordAsStream() {
  const extract = tar.extract();
  const transformStream = new Transform({
    write: (chunk: any, encoding: string, callback: (error?: Error | null) => void) => {
      extract.write(chunk, encoding, callback);
    },
  });
  extract.once('entry', function(_header, stream, next) {
    stream.on('data', (buffer: any) => transformStream.push(buffer));
    stream.on('end', () => {
      transformStream.emit('end');
      next();
    });
    stream.resume(); // just auto drain the stream
  });
  extract.on('error', error => transformStream.emit('error', error));
  return transformStream;
}

/**
 * Returns a stream that extracts the first record of a tarball if the source
 * stream is a tarball, otherwise just pipe the content as is.
 */
export function maybeTarball(): Transform {
  return peek(
    { newline: false, maxBuffer: 264 },
    (data: Buffer, swap: (error?: Error, parser?: Transform) => void) => {
      if (isTarball(data)) swap(undefined, extractFirstTarRecordAsStream());
      else swap(undefined, new PassThrough());
    },
  );
}

// starts an infinite loop of reading and outputting messages from the queue
    // table into a destination write stream
    Consume(queueName: string, destination: Writable): void {
        // in order to use the streaming query we have to call the client specifically
        void this.pool.connect((err, client, done) => {
            // stream is already in object mode
            const query = new QueryStream(`SELECT * FROM queue WHERE queue_name = $1 AND processed_at IS NULL ORDER BY created_at ASC`, [queueName]);
            const stream = client.query(query);

            // mark the task completed in the database
            stream.on('data', (data: any) => {
                this.pool.query({text: `DELETE FROM queue WHERE id = $1`, values: [data.id]}).catch((e) => {
                    Logger.error(`unable to mark queue item as done ${e}`);
                });
            });

            stream.on('end', () => {
                done();
                setTimeout(() => {
                    this.Consume(queueName, destination);
                }, 200); // throw a slight delay so that an empty queue doesn't swamp the database
            });

            // we need a transform stream to output only the message itself
            const transform = new Transform({
                transform(data: any, en, cb) {
                    this.push(data.data);
                    cb();
                },
                objectMode: true, // must maintain object mode
            });

            // the read stream ending will end write destination stream unless
            // we explicitly tell it not to. Since we're doing crazy recursion,
            // tell it to not end when the stream does
            stream.pipe(transform).pipe(destination, {end: false});
        });
    }

class ArrayToCsvTransform extends Transform {
    constructor() {
        super({objectMode: true})
    }
    _transform(chunk: any[], _encoding: string, callback: any) {
        this.push(chunk.join(",") + '\n');
        callback();
    }
}

export class TimeStamper extends Transform {
  public constructor() {
    super({
      transform(chunk, _encoding, callback) {
        this.push(
          chunk.toString().replace(/^/gm, `[${TimeStamper.getTimestamp()}]`)
        );
        callback();
      },
    });
  }

  private static getTimestamp() {
    return new Date().toLocaleTimeString(undefined, {
      hour: '2-digit',
      minute: '2-digit',
      second: '2-digit',
      hour12: false,
    });
  }
}

class LineReader extends Transform {
  _buffer: string;

  constructor() {
    super({ readableObjectMode: true });
    this._buffer = "";
  }

  _transform(chunk: Buffer, encoding: string, callback: () => void) {
    const parts = chunk.toString("utf8").split(/\r\n|\n\r|\n|\r/);
    parts[0] = this._buffer + parts[0];
    this._buffer = parts.pop()!;
    for (const part of parts) {
      if (part) {
        this.push(part);
      }
    }
    callback();
  }
  _flush(callback: () => void) {
    const buffer = this._buffer;
    if (buffer) {
      this._buffer = "";
      this.push(buffer);
    }
    callback();
  }
}

/**
   * Execute the test pipeline so that you can make assertions about the result
   * or behavior of the given test subject.
   */
  async execute(): Promise<TestPipelineResult> {
    const documents: IndexableDocument[] = [];
    if (!this.collator) {
      throw new Error(
        'Cannot execute pipeline without a collator or documents',
      );
    }

    // If we are here and there is no indexer, we are testing a collator or a
    // decorator. Set up a naive writable that captures documents in memory.
    if (!this.indexer) {
      this.indexer = new Writable({ objectMode: true });
      this.indexer._write = (document: IndexableDocument, _, done) => {
        documents.push(document);
        done();
      };
    }

    return new Promise<TestPipelineResult>(done => {
      const pipes: (Readable | Transform | Writable)[] = [this.collator!];
      if (this.decorator) {
        pipes.push(this.decorator);
      }
      pipes.push(this.indexer!);

      pipeline(pipes, (error: NodeJS.ErrnoException | null) => {
        done({
          error,
          documents,
        });
      });
    });
  }

/**
   * Provide the collator, decorator, or indexer to be tested.
   */
  static withSubject(subject: Readable | Transform | Writable) {
    if (subject instanceof Transform) {
      return new TestPipeline({ decorator: subject });
    }

    if (subject instanceof Writable) {
      return new TestPipeline({ indexer: subject });
    }

    if (subject.readable || subject instanceof Readable) {
      return new TestPipeline({ collator: subject });
    }

    throw new Error(
      'Unknown test subject: are you passing a readable, writable, or transform stream?',
    );
  }

/** Creates a transform stream for encoding data to Quoted-Printable encoding */
export class Encoder extends Transform {
    options: EncoderOptions;

    inputBytes: number;
    outputBytes: number;

    constructor(options?: EncoderOptions);
}

/**
 * A transform stream that takes a buffer with data and converts it to MavLinkPacket object
 */
export class MavLinkPacketParser extends Transform {
  protected readonly log = Logger.getLogger(this)

  constructor(opts = {}) {
    super({ ...opts, objectMode: true })
  }

  private getProtocol(buffer): MavLinkProtocol {
    const startByte = buffer.readUInt8(0)
    switch (startByte) {
      case MavLinkProtocolV1.START_BYTE:
        return new MavLinkProtocolV1()
      case MavLinkProtocolV2.START_BYTE:
        return new MavLinkProtocolV2()
      default:
        throw new Error(`Unknown protocol '${hex(startByte)}'`)
    }
  }

  _transform(chunk: Buffer, encoding, callback: TransformCallback) {
    const protocol = this.getProtocol(chunk)
    const header = protocol.header(chunk)
    const payload = protocol.payload(chunk)
    const crc = protocol.crc(chunk)
    const signature = protocol instanceof MavLinkProtocolV2
      ? protocol.signature(chunk, header)
      : null

    const packet = new MavLinkPacket(chunk, header, payload, crc, protocol, signature)

    callback(null, packet)
  }
}

// needed if you're passing raw json objects or an object stream
    transformStreams?: Transform[];

export class Encoder extends Transform {
    options: EncoderOptions;

    inputBytes: number;
    outputBytes: number;

    constructor(options?: EncoderOptions);
}

/**
 * MessageParser instance is a transform stream that separates message headers
 * from the rest of the body. Headers are emitted with the 'headers' event. Message
 * body is passed on as the resulting stream.
 */
declare class MessageParser extends Transform {
    addListener(event: 'headers', listener: (headers: MessageParser.Header[]) => void): this;
    addListener(event: 'close', listener: () => void): this;
    addListener(event: 'data', listener: (chunk: any) => void): this;
    addListener(event: 'end', listener: () => void): this;
    addListener(event: 'readable', listener: () => void): this;
    addListener(event: 'error', listener: (err: Error) => void): this;
    addListener(event: string | symbol, listener: (...args: any[]) => void): this;

    emit(event: 'headers', headers: MessageParser.Header[]): boolean;
    emit(event: 'close'): boolean;
    emit(event: 'data', chunk: any): boolean;
    emit(event: 'end'): boolean;
    emit(event: 'readable'): boolean;
    emit(event: 'error', err: Error): boolean;
    emit(event: string | symbol, ...args: any[]): boolean;

    on(event: 'headers', listener: (headers: MessageParser.Header[]) => void): this;
    on(event: 'close', listener: () => void): this;
    on(event: 'data', listener: (chunk: any) => void): this;
    on(event: 'end', listener: () => void): this;
    on(event: 'readable', listener: () => void): this;
    on(event: 'error', listener: (err: Error) => void): this;
    on(event: string | symbol, listener: (...args: any[]) => void): this;

    once(event: 'headers', listener: (headers: MessageParser.Header[]) => void): this;
    once(event: 'close', listener: () => void): this;
    once(event: 'data', listener: (chunk: any) => void): this;
    once(event: 'end', listener: () => void): this;
    once(event: 'readable', listener: () => void): this;
    once(event: 'error', listener: (err: Error) => void): this;
    once(event: string | symbol, listener: (...args: any[]) => void): this;

    prependListener(event: 'headers', listener: (headers: MessageParser.Header[]) => void): this;
    prependListener(event: 'close', listener: () => void): this;
    prependListener(event: 'data', listener: (chunk: any) => void): this;
    prependListener(event: 'end', listener: () => void): this;
    prependListener(event: 'readable', listener: () => void): this;
    prependListener(event: 'error', listener: (err: Error) => void): this;
    prependListener(event: string | symbol, listener: (...args: any[]) => void): this;

    prependOnceListener(event: 'headers', listener: (headers: MessageParser.Header[]) => void): this;
    prependOnceListener(event: 'close', listener: () => void): this;
    prependOnceListener(event: 'data', listener: (chunk: any) => void): this;
    prependOnceListener(event: 'end', listener: () => void): this;
    prependOnceListener(event: 'readable', listener: () => void): this;
    prependOnceListener(event: 'error', listener: (err: Error) => void): this;
    prependOnceListener(event: string | symbol, listener: (...args: any[]) => void): this;

    removeListener(event: 'headers', listener: (headers: MessageParser.Header[]) => void): this;
    removeListener(event: 'close', listener: () => void): this;
    removeListener(event: 'data', listener: (chunk: any) => void): this;
    removeListener(event: 'end', listener: () => void): this;
    removeListener(event: 'readable', listener: () => void): this;
    removeListener(event: 'error', listener: (err: Error) => void): this;
    removeListener(event: string | symbol, listener: (...args: any[]) => void): this;
}

/**
 * Streams through a message body and calculates relaxed body hash
 */
declare class RelaxedBody extends Transform {
    constructor(options?: RelaxedBody.Options);

    addListener(event: 'hash', listener: (digest: Buffer, debugBody: Buffer | false) => void): this;
    addListener(event: 'close', listener: () => void): this;
    addListener(event: 'data', listener: (chunk: any) => void): this;
    addListener(event: 'end', listener: () => void): this;
    addListener(event: 'readable', listener: () => void): this;
    addListener(event: 'error', listener: (err: Error) => void): this;
    addListener(event: string | symbol, listener: (...args: any[]) => void): this;

    emit(event: 'hash', digest: Buffer, debugBody: Buffer | false): boolean;
    emit(event: 'close'): boolean;
    emit(event: 'data', chunk: any): boolean;
    emit(event: 'end'): boolean;
    emit(event: 'readable'): boolean;
    emit(event: 'error', err: Error): boolean;
    emit(event: string | symbol, ...args: any[]): boolean;

    on(event: 'hash', listener: (digest: Buffer, debugBody: Buffer | false) => void): this;
    on(event: 'close', listener: () => void): this;
    on(event: 'data', listener: (chunk: any) => void): this;
    on(event: 'end', listener: () => void): this;
    on(event: 'readable', listener: () => void): this;
    on(event: 'error', listener: (err: Error) => void): this;
    on(event: string | symbol, listener: (...args: any[]) => void): this;

    once(event: 'hash', listener: (digest: Buffer, debugBody: Buffer | false) => void): this;
    once(event: 'close', listener: () => void): this;
    once(event: 'data', listener: (chunk: any) => void): this;
    once(event: 'end', listener: () => void): this;
    once(event: 'readable', listener: () => void): this;
    once(event: 'error', listener: (err: Error) => void): this;
    once(event: string | symbol, listener: (...args: any[]) => void): this;

    prependListener(event: 'hash', listener: (digest: Buffer, debugBody: Buffer | false) => void): this;
    prependListener(event: 'close', listener: () => void): this;
    prependListener(event: 'data', listener: (chunk: any) => void): this;
    prependListener(event: 'end', listener: () => void): this;
    prependListener(event: 'readable', listener: () => void): this;
    prependListener(event: 'error', listener: (err: Error) => void): this;
    prependListener(event: string | symbol, listener: (...args: any[]) => void): this;

    prependOnceListener(event: 'hash', listener: (digest: Buffer, debugBody: Buffer | false) => void): this;
    prependOnceListener(event: 'close', listener: () => void): this;
    prependOnceListener(event: 'data', listener: (chunk: any) => void): this;
    prependOnceListener(event: 'end', listener: () => void): this;
    prependOnceListener(event: 'readable', listener: () => void): this;
    prependOnceListener(event: 'error', listener: (err: Error) => void): this;
    prependOnceListener(event: string | symbol, listener: (...args: any[]) => void): this;

    removeListener(event: 'hash', listener: (digest: Buffer, debugBody: Buffer | false) => void): this;
    removeListener(event: 'close', listener: () => void): this;
    removeListener(event: 'data', listener: (chunk: any) => void): this;
    removeListener(event: 'end', listener: () => void): this;
    removeListener(event: 'readable', listener: () => void): this;
    removeListener(event: 'error', listener: (err: Error) => void): this;
    removeListener(event: string | symbol, listener: (...args: any[]) => void): this;
}

/**
     * Appends a transform stream object to the transforms list. Final output
     * is passed through this stream before exposing
     */
    transform(transform: Transform): void;

declare class LastNewline extends Transform {
    lastByte: boolean;
}

export default class ProgressCallbackTransform extends Transform {
  private start = Date.now();
  private transferred = 0;
  private delta = 0;
  private readonly total: number;
  private readonly onProgress: ProgressCallback;

  private nextUpdate = this.start + 1000;

  constructor(total: number, onProgress: ProgressCallback) {
    super();
    this.total = total;
    this.onProgress = onProgress;
  }

  _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback) {
    const chunkLength = chunk.length;
    this.transferred += chunkLength;
    this.delta += chunkLength;

    if (this.total >= this.transferred) {
      const now = Date.now();
      if (now >= this.nextUpdate) {
        this.nextUpdate = now + 1000;
        this.onProgress({
          total: this.total,
          delta: this.delta,
          transferred: this.transferred,
          percent: (this.transferred / this.total) * 100,
          bytesPerSecond: Math.round(this.transferred / ((now - this.start) / 1000)),
        });
        this.delta = 0;
      }
    }

    callback(null, chunk);
  }

  _flush(callback: TransformCallback): void {
    const { total, transferred } = this;
    const totalChunk = transferred > total ? transferred : total;

    this.onProgress({
      total: totalChunk,
      delta: this.delta,
      transferred: totalChunk,
      percent: 100,
      bytesPerSecond: Math.round(this.transferred / ((Date.now() - this.start) / 1000)),
    });
    this.delta = 0;

    callback(null);
  }
}

class DigestTransform extends Transform {
  private readonly digester: Hash;
  private _actual: string | null = null;
  readonly expected: string;
  private readonly algorithm: string;
  private readonly encoding: BinaryToTextEncoding;
  isValidateOnEnd = true;

  // noinspection JSUnusedGlobalSymbols
  get actual() {
    return this._actual;
  }

  constructor(options: ValidateOptions) {
    super();
    const { expected, algorithm = 'md5', encoding = 'base64' } = options;
    this.expected = expected;
    this.algorithm = algorithm;
    this.encoding = encoding;
    this.digester = createHash(algorithm);
  }

  // noinspection JSUnusedGlobalSymbols
  _transform(chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback) {
    this.digester.update(chunk);
    callback(null, chunk);
  }

  // noinspection JSUnusedGlobalSymbols
  _flush(callback: TransformCallback): void {
    this._actual = this.digester.digest(this.encoding);

    if (this.isValidateOnEnd) {
      try {
        this.validate();
      } catch (e) {
        callback(e as Error);
        return;
      }
    }

    callback(null);
  }

  validate() {
    if (this._actual == null) {
      throw newError('Not finished yet', 'ERR_STREAM_NOT_FINISHED');
    }

    if (this._actual !== this.expected) {
      throw newError(
        `${this.algorithm} checksum mismatch, expected ${this.expected}, got ${this._actual}`,
        'ERR_CHECKSUM_MISMATCH',
      );
    }

    return null;
  }
}

export class NoiseReductor extends Transform {
  private sensivity = 0;

  private muted = false;

  private timer?: any;

  constructor(
    private onSetSpeaking: (isSpeaking: boolean) => void,
    private onLoudnessChanged?: (loudness: number) => void
  ) {
    super();
  }

  _transform(chunk: Buffer, _encoding: string, callback: (err: Error | null, val: any) => void) {
    const N = chunk.length;
    const sum = chunk.reduce((prev, cur) => prev + cur, 0);
    const loudness = Math.sqrt(sum / N);

    if (this.onLoudnessChanged) {
      this.onLoudnessChanged(loudness);
    }

    if (loudness < this.sensivity) {
      if (!this.timer) {
        this.timer = setTimeout(() => {
          this.muted = true;
          this.onSetSpeaking(false);
        }, 400);
      }
    } else {
      clearTimeout(this.timer);
      this.timer = undefined;
      this.muted = false;
      this.onSetSpeaking(true);
    }

    callback(null, this.muted ? Buffer.alloc(chunk.length, 0) : chunk);
  }

  setSensivity(value: number) {
    this.sensivity = value;
  }
}

declare class LeUnix extends Transform {}

declare class LeWindows extends Transform {}

/**
 * @inheritdoc
 */
export class NodeHash<Reader> extends Transform implements IHasher<Buffer> {
  private readonly hash: BaseHash<Buffer, Reader, NodeHashReader>;

  constructor(implementation: IInternalHash<Reader>, getReader: (r: Reader) => NodeHashReader) {
    super();
    this.hash = new BaseHash(implementation, l => Buffer.alloc(l), getReader);
  }

  /**
   * @reader
   */
  public reader(options?: { dispose?: boolean }) {
    const reader = this.hash.reader(options);
    return reader;
  }

  /**
   * @inheritdoc
   */
  public update(data: HashInput, encoding?: BufferEncoding): this {
    this.hash.update(normalizeInput(data, encoding));
    return this;
  }

  /**
   * @inheritdoc
   */
  public digest(encoding?: IHasherDigestOptions): Buffer;
  public digest(encoding: undefined, options: IHasherDigestOptions): Buffer;
  public digest(encoding: BufferEncoding, options?: IHasherDigestOptions): string;
  public digest(
    encoding?: IHasherDigestOptions | BufferEncoding,
    options?: IHasherDigestOptions,
  ): string | Buffer {
    let resolvedOpts: IHasherDigestOptions | undefined;
    let resolvedEnc: BufferEncoding | undefined;
    if (encoding && typeof encoding === 'object') {
      resolvedOpts = encoding;
      resolvedEnc = undefined;
    } else {
      resolvedOpts = options;
      resolvedEnc = encoding;
    }

    const result = this.hash.digest(resolvedOpts);
    return resolvedEnc ? result.toString(resolvedEnc) : result;
  }

  /**
   * @inheritdoc
   */
  public dispose() {
    this.hash.dispose();
  }

  /**
   * @inheritdoc
   * @hidden
   */
  _transform(chunk: Buffer | string, encoding: string, callback: TransformCallback): void {
    this.update(chunk, encoding as BufferEncoding);
    callback();
  }

  /**
   * @inheritdoc
   * @hidden
   */
  _flush(callback: TransformCallback): void {
    callback(null, this.digest());
  }
}