fs#createReadStream TypeScript Examples

async function main() {
  const config = await configure()

  const command = config._[0]
  if (command === 'e2e') {
    const REGISTRY: MavLinkPacketRegistry = {
      ...minimal.REGISTRY,
      ...common.REGISTRY,
      ...ardupilotmega.REGISTRY,
    }

    const input = config.input === '-' ? process.stdin : createReadStream(config.input)
    const reader = createMavLinkStream(input, dump)

    reader.on('data', (packet: MavLinkPacket) => {
      const clazz = REGISTRY[packet.header.msgid]
      if (clazz) {
        packet.protocol.data(packet.payload, clazz)
      } else {
        console.log('< (unknown)', packet.debug())
      }
    })
  }
}

generator: FontGenerator<void> = {
  generate: ({
    name: fontName,
    fontHeight,
    descent,
    normalize,
    assets,
    codepoints,
    formatOptions: { svg } = {}
  }) =>
    new Promise(resolve => {
      let font = Buffer.alloc(0);

      const fontStream = new SVGIcons2SVGFontStream({
        fontName,
        fontHeight,
        descent,
        normalize,
        log: () => null,
        ...svg
      })
        .on('data', data => (font = Buffer.concat([font, data])))
        .on('end', () => resolve(font.toString()));

      for (const { id, absolutePath } of Object.values(assets)) {
        const glyph: GglyphStream = createReadStream(absolutePath);
        const unicode = String.fromCharCode(codepoints[id]);

        glyph.metadata = { name: id, unicode: [unicode] };

        fontStream.write(glyph);
      }

      fontStream.end();
    })
}

imageDimensionsFromFile = async (file: string, noCache?: boolean) => {
  if (!file) return null

  const cacheKey = await genCacheKey(file, noCache)
  const cached = getCache<Dimensions>(cacheKey)
  if (cached) return cached

  const { width, height } = await probe(createReadStream(file))
  if (0 === width + height) return null

  setCache(cacheKey, { width, height })
  return { width, height }
}

fileReader = (path: string, targetLines: number[]) =>
  new Promise((done) => {
    const collectedLines: LineContents[] = []
    const linesOfInterest = new Set(targetLines)
    const maxLineIndex = Math.max(...targetLines)
    let currentLineIndex = 0
    let buffer = ''

    // not using NewlineSplitter here because it filters out empty lines
    // we need the empty lines, since we track the line index
    const readStream = createReadStream(path).on('data', (raw) => {
      const lines = (buffer + raw).split(/\r?\n/)
      buffer = lines.pop() || ''

      lines.forEach((line) => {
        const needThisLine = linesOfInterest.has(currentLineIndex)
        if (needThisLine) collectedLines.push({ ix: currentLineIndex, line })

        const reachedMaximumLine = currentLineIndex === maxLineIndex
        if (reachedMaximumLine) readStream.close()

        currentLineIndex++
      })
    })

    readStream.on('close', () => done(collectedLines))
  })

async readFileStream(
    filePath: string,
    options: {
      cancelToken?: CancelToken | null
    } = {}
  ): Promise<ReadStream> {
    const cancelToken = createCancelToken.race([
      options.cancelToken,
      this._destroyTokenController.token,
    ])

    const filename = this._resolvePath(filePath)
    const stream = createReadStream(filename)

    return new Promise((resolve, reject) => {
      const unregisterCanceller = cancelToken.onCancelled((reason: unknown) => {
        stream.close()
        reject(reason)
      })
      const handleError = (err: Error) => {
        unregisterCanceller?.()
        reject(err)
      }
      const handleReadble = () => {
        unregisterCanceller?.()
        resolve(stream)
      }

      stream.once('readable', handleReadble)
      stream.once('error', handleError)
    })
  }

protected getStream(file: string | Buffer) {
    if (Buffer.isBuffer(file)) {
      return new Readable({
        read() {
          this.push(file);
          this.push(null);
        }
      });
    } else {
      return createReadStream(file);
    }
  }

async upload(options: UploadOptions): Promise<UploadResponse> {
    const { file, fileName } = options;
    const fileStream = Buffer.isBuffer(file) ? file : createReadStream(file);
    const { domain, params = '' } = this.getConfig();
    try {
      const res = await this.postFile(fileName, fileStream);
      if (res.status === 200) {
        return {
          success: true,
          data: {
            url: domain + '/' + fileName + params
          }
        };
      } else {
        return {
          success: false,
          desc: '上传失败'
        };
      }
    } catch (err) {
      return {
        success: false,
        desc: err.message
      };
    }
  }

protected getStream(file: string | Buffer) {
    if (Buffer.isBuffer(file)) {
      return new Readable({
        read() {
          this.push(file);
          this.push(null);
        }
      });
    } else {
      return createReadStream(file);
    }
  }

protected getStream(file: string | Buffer) {
    if (Buffer.isBuffer(file)) {
      return new Readable({
        read() {
          this.push(file);
          this.push(null);
        }
      });
    } else {
      return createReadStream(file);
    }
  }

protected getStream(file: string | Buffer) {
    if (Buffer.isBuffer(file)) {
      return new Readable({
        read() {
          this.push(file);
          this.push(null);
        }
      });
    } else {
      return createReadStream(file);
    }
  }

async function* parseInputFile(): AsyncIterableIterator<InputKey> {
    const rl = readline.createInterface({
        input: createReadStream(IN_TXT_FILE),
        crlfDelay: Infinity,
    })

    let currentKind: InputKey['kind'] = 'old'

    let current = ''

    for await (const line of rl) {
        if (line === '### OLD ###') currentKind = 'old'
        if (line === '### NEW ###') currentKind = 'new'

        if (line[0] === '#') continue

        current += line + '\n'

        if (line === '-----END RSA PUBLIC KEY-----') {
            yield {
                kind: currentKind,
                pem: current.trim(),
            }
            current = ''
        }
    }
}

export function copyData(task: Operation, out: Writable, oldFileFd: number, reject: (error: Error) => void, resolve: () => void): void {
  const readStream = createReadStream("", {
    fd: oldFileFd,
    autoClose: false,
    start: task.start,
    // end is inclusive
    end: task.end - 1,
  })
  readStream.on("error", reject)
  readStream.once("end", resolve)
  readStream.pipe(out, {
    end: false
  })
}

function hashFile(file: string, algorithm = "sha512", encoding: "base64" | "hex" = "base64", options?: any): Promise<string> {
  return new Promise<string>((resolve, reject) => {
    const hash = createHash(algorithm)
    hash
      .on("error", reject)
      .setEncoding(encoding)

    createReadStream(file, {...options, highWaterMark: 1024 * 1024 /* better to use more memory but hash faster */})
      .on("error", reject)
      .on("end", () => {
        hash.end()
        resolve(hash.read() as string)
      })
      .pipe(hash, {end: false})
  })
}

public async up (queryRunner: QueryRunner): Promise<void> {
        // Old migration code, super slow and doesn't work for restricted users
        if (process.env.NODE_ENV === "production") {
            await queryRunner.query("ALTER TABLE `user` ADD `country` tinytext NOT NULL");

            const users = await User
                .createQueryBuilder("user")
                .getMany();

            await Promise.all(users.map(async user => {
                const apiUser = (await osuClient.user.get(user.osu.userID)) as APIUser;
                user.country = apiUser.country.toString();
                return user.save();
            }));
        } else {
            const bigSql = await streamToString(createReadStream(resolve(__dirname, "1631140092497-userCountries.sql.gz")).pipe(createGunzip()));
            const sqlInstructions = bigSql.split("\n").filter(sql => sql.trim().length !== 0);
            for(const sqlInstruction of sqlInstructions)
                if(sqlInstruction.trim().length !== 0)
                    await queryRunner.query(sqlInstruction);
        }
    }

describe('setOneAutoExtractorContent()', () => {
	const deleteOneAutoExtractor = makeDeleteOneAutoExtractor(TEST_BASE_API_CONTEXT);
	const uploadManyAutoExtractors = makeUploadManyAutoExtractors(TEST_BASE_API_CONTEXT);
	const getAllAutoExtractors = makeGetAllAutoExtractors(TEST_BASE_API_CONTEXT);

	beforeEach(async () => {
		// Delete all autoExtractors
		const currentAutoExtractors = await getAllAutoExtractors();
		const currentAutoExtractorIDs = currentAutoExtractors.map(m => m.id);
		const deletePromises = currentAutoExtractorIDs.map(autoExtractorID => deleteOneAutoExtractor(autoExtractorID));
		await Promise.all(deletePromises);
	});

	it(
		'Should upload an auto extractor file and create the auto extractors defined in the file',
		integrationTest(async () => {
			const autoExtractors1 = await getAllAutoExtractors();
			expect(autoExtractors1.length).toBe(0);

			const createFileStream = () => createReadStream(join(TEST_ASSETS_PATH!, 'auto-extractors.config'));
			const fileStream = createFileStream();

			const autoExtractors2 = await uploadManyAutoExtractors({ file: fileStream });
			expect(autoExtractors2.length).toBe(2);
			for (const ae of autoExtractors2) expect(isAutoExtractor(ae)).toBeTrue();
		}),
		10000,
	);
});

async render(model: any, { metadata }: ViewMetadata): Promise<void> {
        if (metadata?.file) {
            model = createReadStream(join(__dirname, this.baseFileDir, metadata.file));
        }
        const response = Context.getCurrent().response;
        if (isStream(model)) {
            Context.setSkipAutoEnd(true);
            this.streamDownload(response, model);
        } else {
            if (Buffer.isBuffer(model)) {
                response.setHeader(HttpHeaders.CONTENT_LENGTH, model.byteLength);
            }
            response.body = model;
        }
    }

export function copyFile(source: PathLike, dest: PathLike): Promise<void> {
  return new Promise<void>((resolve) => {
    const input = createReadStream(source);
    const output = createWriteStream(dest);

    output.once('close', resolve);
    input.pipe(output);
  });
}

unzip = (from: string, to: string) =>
  new Promise((resolve, reject) => {
    const stream = createReadStream(from);
    const extract = Extract({ path: to });

    extract.on('error', reject);
    extract.once('close', resolve);
    stream.pipe(extract);
  })

upload(): any {
        vscode.window.showOpenDialog({ canSelectFiles: true, canSelectMany: false, canSelectFolders: false, openLabel: "Select Upload Path" })
            .then(async uri => {
                if (uri) {
                    const { sftp } = await ClientManager.getSSH(this.sshConfig)
                    const targetPath = uri[0].fsPath;

                    vscode.window.withProgress({
                        location: vscode.ProgressLocation.Notification,
                        title: `Start uploading ${targetPath}`,
                        cancellable:true
                    }, (progress, token) => {
                        return new Promise((resolve) => {
                            const fileReadStream = createReadStream(targetPath)
                            var str = progressStream({
                                length: statSync(targetPath).size,
                                time: 100
                            });
                            let before=0;
                            str.on("progress", (progressData: any) => {
                                if (progressData.percentage == 100) {
                                    resolve(null)
                                    vscode.window.showInformationMessage(`Upload ${targetPath} success, cost time: ${progressData.runtime}s`)
                                    return;
                                }
                                progress.report({ increment: progressData.percentage-before,message:`remaining : ${prettyBytes(progressData.remaining)}` });
                                before=progressData.percentage
                            })
                            str.on("error",err=>{
                                vscode.window.showErrorMessage(err.message)
                            })
                            const outStream = sftp.createWriteStream(this.fullPath + "/" + path.basename(targetPath));
                            fileReadStream.pipe(str).pipe(outStream);
                            token.onCancellationRequested(() => {
                                fileReadStream.destroy()
                                outStream.destroy()
                            });
                        })
                    })

                    // const start = new Date()
                    // vscode.window.showInformationMessage(`Start uploading ${targetPath}.`)
                    // sftp.fastPut(targetPath, this.fullPath + "/" + path.basename(targetPath), err => {
                    //     if (err) {
                    //         vscode.window.showErrorMessage(err.message)
                    //     } else {
                    //         vscode.window.showInformationMessage(`Upload ${this.fullPath} success, cost time: ${new Date().getTime() - start.getTime()}`)
                    //         vscode.commands.executeCommand(Command.REFRESH)
                    //     }
                    // })
                }
            })
    }

describe('updateOneFile()', () => {
	const createOneFile = makeCreateOneFile(TEST_BASE_API_CONTEXT);
	const updateOneFile = makeUpdateOneFile(TEST_BASE_API_CONTEXT);
	const deleteOneFile = makeDeleteOneFile(TEST_BASE_API_CONTEXT);
	const getAllFiles = makeGetAllFiles(TEST_BASE_API_CONTEXT);
	const getOneFileContent = makeGetOneFileContent(TEST_BASE_API_CONTEXT);

	let createdFile: FileMetadata;

	beforeEach(async () => {
		jasmine.addMatchers(myCustomMatchers);

		// Delete all files
		const currentFiles = await getAllFiles();
		const currentFileIDs = currentFiles.map(f => f.globalID);
		const deletePromises = currentFileIDs.map(fileID => deleteOneFile(fileID));
		await Promise.all(deletePromises);

		// Create one file
		const data: CreatableFile = {
			name: 'F1',
			file: createReadStream(join(TEST_ASSETS_PATH!, 'file-a.txt')),
		};
		createdFile = await createOneFile(data);
	});

	const updateTests: Array<Omit<UpdatableFile, 'id'>> = [
		{ name: 'New Name' },

		{ description: 'New description' },
		{ description: null },

		{ groupIDs: ['1'] },
		{ groupIDs: ['1', '2'] },
		{ groupIDs: [] },

		{ labels: ['Label 1'] },
		{ labels: ['Label 1', 'Label 2'] },
		{ labels: [] },

		{ isGlobal: true },
		{ isGlobal: false },

		// TODO: Enable test below
		// { file: createReadStream(join(TEST_ASSETS_PATH!, 'auto-extractors.config')) },
	];
	updateTests.forEach((_data, testIndex) => {
		const updatedFields = Object.keys(_data);
		const formatedUpdatedFields = updatedFields.join(', ');
		const formatedTestIndex = (testIndex + 1).toString().padStart(2, '0');

		it(
			`Test ${formatedTestIndex}: Should update a file ${formatedUpdatedFields} and return itself updated`,
			integrationTest(async () => {
				const current = createdFile;
				expect(isFileMetadata(current)).toBeTrue();

				// !WARNING: Using globalID because gravwell/gravwell#2509
				const data: UpdatableFile = { ..._data, id: current.globalID };
				const updated = await updateOneFile(data);

				if (data.file) {
					const stream = createReadStream(join(TEST_ASSETS_PATH!, 'auto-extractors.config'));
					const expectedContent = await streamToString(stream);
					const actualContent = await getOneFileContent(data.id);
					expect(actualContent).toBe(expectedContent);
				}

				expect(isFileMetadata(updated)).toBeTrue();
				expect(updated).toPartiallyEqual(omit(data, ['file', 'id']));
			}),
			20000,
		);
	});
});

private async buildTransaction(filePath: string, tags: Tags): Promise<Transaction> {
    const tx = await pipeline(createReadStream(filePath), createTransactionAsync({}, this.arweave, this.wallet));
    tags.addTagsToTransaction(tx);
    await this.arweave.transactions.sign(tx, this.wallet);

    // @ts-ignore
    return tx;
  }

/**
 * Serve index.html
 */
router.get( '/', async (ctx: koa.Context, next: () => Promise<any>) => {
    ctx.type = 'text/html; charset=utf-8';
    ctx.body = createReadStream(__dirname + '/../client/index.html');
    await next();
});

file = createReadStream('./GH-5.bin')

public async up (queryRunner: QueryRunner): Promise<void> {
        const bigSql = await streamToString(createReadStream(resolve(__dirname, "1614784355565-MCA2020seeding.sql.gz")).pipe(createGunzip()));
        const sqlInstructions = bigSql.split("\n").filter(sql => sql.trim().length !== 0);
        for(const sqlInstruction of sqlInstructions)
            if(sqlInstruction.trim().length !== 0)
                await queryRunner.query(sqlInstruction);
    }

public async get(path: string): Promise<NodeJS.ReadableStream> {
		return createReadStream(join(__dirname, '../../../../../', 'uploads', path));
	}

createReadStream = (path: string) => {
        return createReadStream(path);
    }

async function main(): Promise<void> {
  const packetFile = process.argv.pop();
  if (packetFile == null || process.argv.length < 2 || !existsSync(packetFile)) {
    console.log('Usage: ./replay-packets :packetFile');
    return;
  }

  if (process.env['DIABLO2_PATH']) await Diablo2MpqLoader.load(process.env['DIABLO2_PATH'], Log, Diablo2Mpq);

  Log.info('Process Packets');

  if (Diablo2Mpq.basePath == null) throw new Error('No $DIABLO2_PATH set');
  const version = getDiabloVersion(Diablo2Mpq.basePath);
  const client = new Diablo2Client(version);
  const session = client.startSession(Log);

  SniffExample.item(session);
  SniffExample.npc(session);

  const PacketByName = new Map<string, number>();
  session.parser.all((pkt) => {
    const pktName = pkt.packet.direction + ':' + pkt.packet.name;
    PacketByName.set(pktName, (PacketByName.get(pktName) ?? 0) + 1);
  });

  const reader = readline.createInterface({ input: createReadStream(packetFile), crlfDelay: Infinity });

  for await (const line of reader) {
    const json = JSON.parse(line) as PacketLine;
    // if (json.direction ===  'out') continue;
    session.onPacket(json.direction, Buffer.from(json.bytes, 'hex'), Log);
  }

  Log.info({ packetsIn: session.parser.inPacketParsedCount, packetsOut: session.parser.outPacketParsedCount }, 'Done');

  if (process.argv.includes('--stats')) {
    const packetStats = [...PacketByName.entries()].sort((a, b) => b[1] - a[1]);
    Log.debug('PacketStats');
    for (const [pktName, count] of packetStats) {
      Log.debug({ count }, pktName);
    }
  }

  if (process.argv.includes('--kills')) {
    const kills = session.state.toJSON().kills.sort((a, b) => b.total - a.total);
    for (const kill of kills) {
      const name = kill.name;
      delete (kill as any).name;
      Log.debug(kill, `KillCount:${name}`);
    }
  }
}

async upload(options: UploadOptions): Promise<UploadResponse> {
    try {
      const { file, fileName } = options;
      const formData = new FormData();
      const uploaderOptions = this.getConfig();
      const fileStream = Buffer.isBuffer(file) ? file : createReadStream(file);
      formData.append(uploaderOptions.fileFieldName, fileStream, { filename: fileName });
      const length = await new Promise((resolve, reject) => {
        formData.getLength(async (err, length) => {
          if (err) {
            reject(err);
          } else {
            resolve(length);
          }
        });
      });
      const requestOpetion: AxiosRequestConfig = {
        url: uploaderOptions.url,
        method: uploaderOptions.method,
        headers: {
          ...formData.getHeaders(),
          'Content-Length': length,
          Authorization: uploaderOptions.token || ''
        },
        params: uploaderOptions.requestParams ? JSON.parse(uploaderOptions.requestParams) : {},
        data: uploaderOptions.contentType === 'multipart/form-data' ? formData : uploaderOptions.requestBody
      };
      // 发起请求
      const { data: res } = await axios(requestOpetion);
      let imageUrl = uploaderOptions.responseUrlFieldName?.split('.').reduce((pre, cur) => {
        try {
          return pre[cur];
        } catch (err) {
          return undefined;
        }
      }, res);
      if (imageUrl) {
        return {
          success: true,
          data: {
            url: imageUrl + uploaderOptions.params || ''
          }
        };
      } else {
        const message = uploaderOptions?.responseMessageName?.split('.').reduce((pre, cur) => {
          try {
            return pre[cur];
          } catch (err) {
            return undefined;
          }
        }, res);
        return {
          success: false,
          desc: message || '上传失败'
        };
      }
    } catch (err) {
      return {
        success: false,
        desc: err.message
      };
    }
  }

describe('createOneFile()', () => {
	const createOneFile = makeCreateOneFile(TEST_BASE_API_CONTEXT);
	const createOneGroup = makeCreateOneGroup(TEST_BASE_API_CONTEXT);
	const deleteAllGroups = makeDeleteAllGroups(TEST_BASE_API_CONTEXT);

	let groupIDs: Array<NumericID>;

	beforeEach(async () => {
		jasmine.addMatchers(myCustomMatchers);
		// avoid dup error from backend
		await deleteAllGroups();
		groupIDs = (
			await Promise.all(
				Array.from({ length: 3 })
					.map((_, i) => `G${i}`)
					.map(name => createOneGroup({ name })),
			)
		).map(g => g.id);
	});

	it(
		'Should create a file and return it',
		integrationTest(async () => {
			const createFileStream = () => createReadStream(join(TEST_ASSETS_PATH!, 'file-a.txt'));
			const fileStream = createFileStream();

			const data: CreatableFile = {
				groupIDs,
				isGlobal: true,

				name: 'name',
				description: 'description',
				labels: ['Label 1', 'Label 2'],

				file: fileStream as any,
			};

			const file = await createOneFile(data);
			expect(isFileMetadata(file)).toBeTrue();
			expect(file).toPartiallyEqual(omit(data, ['file']));
		}),
	);
});