stream#PassThrough TypeScript Examples

describe('utils', () => {
  describe('PreviewStream', () => {
    it('should stream first 5 bytes', done => {
      const peek = 5;
      const input = 'some string that will be truncated.';
      const source = new PassThrough();
      const preview = new PreviewStream({ peek });
      const dst = source.pipe(preview).on('end', done);
      source.end(input);
      dst.once('readable', () => expect(dst.read().toString()).toBe(input.slice(0, peek)));
    });

    it('should stream everything if peek==0', done => {
      const peek = 0;
      const input = 'some string that will be truncated.';
      const source = new PassThrough();
      const preview = new PreviewStream({ peek });
      const dst = source.pipe(preview).on('end', done);
      source.end(input);
      dst.once('readable', () => expect(dst.read().toString()).toBe(input));
    });
  });
});

export default class BeanConverterStream extends PassThrough {

    public type;
    public R;

    constructor(type, R) {
        super({
            readableObjectMode: true,
            writableObjectMode: true,
        });
        this.type = type;
        this.R = R;
    }

    _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback) {
        let bean = this.R.convertToBean(this.type, chunk);
        super._transform(bean, encoding, callback);
    }

    static createStream(type : string, R : RedBeanNode, queryPromise : QueryBuilder) {
        let converterStream = new BeanConverterStream(type, R);
        let stream = queryPromise.stream();
        return stream.pipe(converterStream);
        /*
        return new Promise<BeanConverterStream>((resolve, reject) => {
            let converterStream = new BeanConverterStream(type, R);

            queryPromise.stream((stream) => {
                resolve(stream.pipe(converterStream));
            }).catch((error) => { });
        });*/
    }
}

createStepLogger = ({
  task,
  step,
}: {
  task: TaskContext;
  step: TaskStep;
}) => {
  const metadata = { stepId: step.id };
  const taskLogger = winston.createLogger({
    level: process.env.LOG_LEVEL || 'info',
    format: winston.format.combine(
      winston.format.colorize(),
      winston.format.timestamp(),
      winston.format.simple(),
    ),
    defaultMeta: {},
  });

  const streamLogger = new PassThrough();
  streamLogger.on('data', async data => {
    const message = data.toString().trim();
    if (message?.length > 1) {
      await task.emitLog(message, metadata);
    }
  });

  taskLogger.add(new winston.transports.Stream({ stream: streamLogger }));

  return { taskLogger, streamLogger };
}

parseBuffers = async (buffers: Buffer[]): Promise<BackendMessage[]> => {
  const stream = new PassThrough()
  for (const buffer of buffers) {
    stream.write(buffer)
  }
  stream.end()
  const msgs: BackendMessage[] = []
  await parse(stream, (msg) => msgs.push(msg))
  return msgs
}

/**
 * Returns a stream that extracts the first record of a tarball if the source
 * stream is a tarball, otherwise just pipe the content as is.
 */
export function maybeTarball(): Transform {
  return peek(
    { newline: false, maxBuffer: 264 },
    (data: Buffer, swap: (error?: Error, parser?: Transform) => void) => {
      if (isTarball(data)) swap(undefined, extractFirstTarRecordAsStream());
      else swap(undefined, new PassThrough());
    },
  );
}

export class GetColConverterStream extends PassThrough {

    public type;
    public R;

    constructor(type, R) {
        super({
            readableObjectMode: true,
            writableObjectMode: true,
        });
        this.type = type;
        this.R = R;
    }

    _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback) {
        let bean = this.R.convertToBean(this.type, chunk);
        super._transform(bean, encoding, callback);
    }

    static createStream(type : string, R : RedBeanNode, queryPromise : QueryBuilder) {
        return new Promise<BeanConverterStream>((resolve, reject) => {
            let converterStream = new BeanConverterStream(type, R);

            queryPromise.stream((stream) => {
                resolve(stream.pipe(converterStream));
            }).catch((error) => { });
        });
    }

}

/**
 * 空のPassThroughを生成します
 * @returns PassThrough
 */
export function InitPassThrough():PassThrough{
  const stream = new PassThrough({
    highWaterMark: 1024 * 512
  });
  stream._destroy = () => { 
    stream.destroyed = true;
    stream.emit("close", []);
  };
  return stream;
}

describe('acme:example', () => {
  afterEach(() => {
    jest.resetAllMocks();
  });

  it('should call action', async () => {
    const action = createAcmeExampleAction();

    const logger = getVoidLogger();
    jest.spyOn(logger, 'info');

    await action.handler({
      input: {
        myParameter: 'test',
      },
      workspacePath: '/tmp',
      logger,
      logStream: new PassThrough(),
      output: jest.fn(),
      createTemporaryDirectory() {
        // Usage of mock-fs is recommended for testing of filesystem operations
        throw new Error('Not implemented');
      },
    });

    expect(logger.info).toHaveBeenCalledWith(
      'Running example template with parameters: test',
    );
  });
});

createS3ZipFromS3Objects = async (
  params: CreateS3ZipFromStreamsOptions,
) => {
  // read in params
  const { key, tags, objects } = params

  // create write stream
  const streamPassThrough = new PassThrough()
  const s3Upload = uploadObjectStream(streamPassThrough, key, {
    ContentType: 'application/zip',
    Tagging: tags,
  })

  // attach object streams from S3 to archive
  await new Promise((resolve, reject) => {
    const archive = Archiver('zip')
    archive.on('error', reject)
    streamPassThrough.on('close', resolve)
    streamPassThrough.on('end', resolve)
    streamPassThrough.on('error', reject)

    archive.pipe(streamPassThrough)
    objects.forEach((o) => {
      archive.append(getObjectReadStream(o.key), { name: o.filename })
    })

    archive.finalize()
  })

  // wait for the upload to complete
  return await s3Upload.promise()
}

public async *streamAccountsNotification(market: Market, marketName: string): AsyncIterable<AccountsNotification> {
    const wsEndpoint = new URL(this._options.nodeEndpoint)
    wsEndpoint.protocol = this._options.nodeEndpoint.startsWith('https') ? 'wss' : 'ws'

    if (this._options.wsEndpointPort !== undefined) {
      wsEndpoint.port = this._options.wsEndpointPort.toString()
    }

    const notificationsStream = new PassThrough({
      objectMode: true,
      highWaterMark: 8096
    })

    const options = {
      nodeWsEndpoint: wsEndpoint.toString(),
      nodeRestEndpoint: this._options.nodeEndpoint,
      marketName,
      commitment: this._options.commitment
    }

    const accountsChangeNotifications = new AccountsChangeNotifications(market, options)

    logger.log('info', 'Starting RPC client', options)

    accountsChangeNotifications.onAccountsChange = (notification) => {
      notificationsStream.write(notification)
    }

    try {
      for await (const notification of notificationsStream) {
        yield notification as AccountsNotification
      }
    } finally {
      accountsChangeNotifications.dispose()
    }
  }

imports = {
    ...(process.env.NODE_ENV === 'test'
        ? {
              'test-utils/write-to-file': writeToFile,
          }
        : {}),
    '@google-cloud/bigquery': bigquery,
    '@google-cloud/pubsub': pubsub,
    '@google-cloud/storage': gcs,
    '@posthog/plugin-contrib': contrib,
    '@posthog/plugin-scaffold': scaffold,
    'aws-sdk': AWS,
    ethers: ethers,
    'generic-pool': genericPool,
    'node-fetch': fetch,
    'snowflake-sdk': snowflake,
    crypto: crypto,
    jsonwebtoken: jsonwebtoken,
    faker: faker,
    pg: pg,
    stream: { PassThrough },
    url: url,
    zlib: zlib,
}

constructor() {
    super()

    this.input = new PassThrough()

    this.processIncommingUDPData = this.processIncommingUDPData.bind(this)
    this.processIncommingPacket = this.processIncommingPacket.bind(this)

    // Create the reader as usual by piping the source stream through the splitter
    // and packet parser
    const reader = this.input
      .pipe(new MavLinkPacketSplitter())
      .pipe(new MavLinkPacketParser())

    reader.on('data', this.processIncommingPacket)
  }

sign(input: string | Buffer | Readable, extraOptions?: DKIM.Options): PassThrough;

describe('run:yeoman', () => {
  const mockTmpDir = os.tmpdir();

  let mockContext: ActionContext<{
    namespace: string;
    args?: string[];
    options?: JsonObject;
  }>;

  const action = createRunYeomanAction();

  beforeEach(() => {
    jest.resetAllMocks();
  });

  it('should call yeomanRun with the correct variables', async () => {
    const namespace = 'whatever:app';
    const args = ['aa', 'bb'];
    const options = {
      code: 'owner',
    };
    mockContext = {
      input: {
        namespace,
        args,
        options,
      },
      workspacePath: mockTmpDir,
      logger: getVoidLogger(),
      logStream: new PassThrough(),
      output: jest.fn(),
      createTemporaryDirectory: jest.fn().mockResolvedValue(mockTmpDir),
    };

    await action.handler(mockContext);
    expect(yeomanRun).toHaveBeenCalledWith(
      mockTmpDir,
      namespace,
      args,
      options,
    );
  });
});

public send = async () => {
    const {
      method,
      followRedirect,
      maxRedirectCount,
      requestURL,
      parsedURL,
      size,
      username,
      password,
      timeout,
      body: requestBody,
    } = this.options;

    /** Create electron request */
    const clientRequest = await this.createRequest();
    /** Cancel electron request */
    const cancelRequest = () => {
      // In electron, `request.destroy()` does not send abort to server
      clientRequest.abort();
    };
    /** Write body to electron request */
    const writeToRequest = () => {
      if (requestBody === null) {
        clientRequest.end();
      } else if (requestBody instanceof Stream) {
        // TODO remove as
        requestBody.pipe(new PassThrough()).pipe(clientRequest as unknown as Writable);
      } else {
        clientRequest.write(requestBody);
        clientRequest.end();
      }
    };
    /** Bind electron request event */
    const bindRequestEvent = (
      onFulfilled: (value: Response | PromiseLike<Response>) => void,
      onRejected: (reason: Error) => void,
    ) => {
      /** Set electron request timeout */
      if (timeout) {
        this.timeoutId = setTimeout(() => {
          onRejected(new Error(`Electron request timeout in ${timeout} ms`));
        }, timeout);
      }

      /** Bind electron request error event */
      clientRequest.on(REQUEST_EVENT.ERROR, onRejected);

      /** Bind electron request abort event */
      clientRequest.on(REQUEST_EVENT.ABORT, () => {
        onRejected(new Error('Electron request was aborted by the server'));
      });

      /** Bind electron request login event */
      clientRequest.on(REQUEST_EVENT.LOGIN, (authInfo, callback) => {
        if (username && password) {
          callback(username, password);
        } else {
          onRejected(
            new Error(`Login event received from ${authInfo.host} but no credentials provided`),
          );
        }
      });

      /** Bind electron request response event */
      clientRequest.on(REQUEST_EVENT.RESPONSE, (res) => {
        this.clearRequestTimeout();

        const { statusCode = 200, headers: responseHeaders } = res;
        const headers = new Headers(responseHeaders);

        if (isRedirect(statusCode) && followRedirect) {
          if (maxRedirectCount && this.redirectCount >= maxRedirectCount) {
            onRejected(new Error(`Maximum redirect reached at: ${requestURL}`));
          }

          if (!headers.get(HEADER_MAP.LOCATION)) {
            onRejected(new Error(`Redirect location header missing at: ${requestURL}`));
          }

          if (
            statusCode === 303 ||
            ((statusCode === 301 || statusCode === 302) && method === METHOD_MAP.POST)
          ) {
            this.options.method = METHOD_MAP.GET;
            this.options.body = null;
            this.options.headers.delete(HEADER_MAP.CONTENT_LENGTH);
          }

          this.redirectCount += 1;
          this.options.parsedURL = new URL(
            String(headers.get(HEADER_MAP.LOCATION)),
            parsedURL.toString(),
          );
          onFulfilled(this.send());
        }

        const responseBody = pump(res, new PassThrough(), (error) => {
          if (error !== null) {
            onRejected(error);
          }
        });

        responseBody.on(RESPONSE_EVENT.CANCEL_REQUEST, cancelRequest);

        onFulfilled(
          new ResponseImpl(responseBody, {
            requestURL,
            statusCode,
            headers,
            size,
          }),
        );
      });
    };

    return new Promise<Response>((resolve, reject) => {
      const onRejected = (reason: Error) => {
        this.clearRequestTimeout();
        cancelRequest();
        reject(reason);
      };
      bindRequestEvent(resolve, onRejected);
      writeToRequest();
    });
  };

describe('github:actions:dispatch', () => {
  const config = new ConfigReader({
    integrations: {
      github: [
        { host: 'github.com', token: 'tokenlols' },
        { host: 'ghe.github.com' },
      ],
    },
  });

  const integrations = ScmIntegrations.fromConfig(config);
  let githubCredentialsProvider: GithubCredentialsProvider;
  let action: TemplateAction<any>;

  const mockContext = {
    input: {
      repoUrl: 'github.com?repo=repo&owner=owner',
      workflowId: 'a-workflow-id',
      branchOrTagName: 'main',
    },
    workspacePath: 'lol',
    logger: getVoidLogger(),
    logStream: new PassThrough(),
    output: jest.fn(),
    createTemporaryDirectory: jest.fn(),
  };

  beforeEach(() => {
    jest.resetAllMocks();
    githubCredentialsProvider =
      DefaultGithubCredentialsProvider.fromIntegrations(integrations);
    action = createGithubActionsDispatchAction({
      integrations,
      githubCredentialsProvider,
    });
  });

  it('should call the githubApis for creating WorkflowDispatch without an input object', async () => {
    mockOctokit.rest.actions.createWorkflowDispatch.mockResolvedValue({
      data: {
        foo: 'bar',
      },
    });

    const repoUrl = 'github.com?repo=repo&owner=owner';
    const workflowId = 'dispatch_workflow';
    const branchOrTagName = 'main';
    const ctx = Object.assign({}, mockContext, {
      input: { repoUrl, workflowId, branchOrTagName },
    });
    await action.handler(ctx);

    expect(
      mockOctokit.rest.actions.createWorkflowDispatch,
    ).toHaveBeenCalledWith({
      owner: 'owner',
      repo: 'repo',
      workflow_id: workflowId,
      ref: branchOrTagName,
    });
  });

  it('should call the githubApis for creating WorkflowDispatch with an input object', async () => {
    mockOctokit.rest.actions.createWorkflowDispatch.mockResolvedValue({
      data: {
        foo: 'bar',
      },
    });

    const repoUrl = 'github.com?repo=repo&owner=owner';
    const workflowId = 'dispatch_workflow';
    const branchOrTagName = 'main';
    const workflowInputs = '{ "foo": "bar" }';
    const ctx = Object.assign({}, mockContext, {
      input: { repoUrl, workflowId, branchOrTagName, workflowInputs },
    });
    await action.handler(ctx);

    expect(
      mockOctokit.rest.actions.createWorkflowDispatch,
    ).toHaveBeenCalledWith({
      owner: 'owner',
      repo: 'repo',
      workflow_id: workflowId,
      ref: branchOrTagName,
      inputs: workflowInputs,
    });
  });
});

describe('UIServer apis', () => {
  let app: UIServer;
  const tagName = '1.0.0';
  const commitHash = 'abcdefg';
  const { argv, buildDate, indexHtmlContent } = commonSetup({ tagName, commitHash });

  beforeEach(() => {
    const consoleInfoSpy = jest.spyOn(global.console, 'info');
    consoleInfoSpy.mockImplementation(() => null);
    const consoleLogSpy = jest.spyOn(global.console, 'log');
    consoleLogSpy.mockImplementation(() => null);
  });

  afterEach(() => {
    if (app) {
      app.close();
    }
  });

  describe('/', () => {
    it('responds with unmodified index.html if it is not a kubeflow deployment', done => {
      const configs = loadConfigs(argv, {});
      app = new UIServer(configs);

      const request = requests(app.start());
      request
        .get('/')
        .expect('Content-Type', 'text/html; charset=utf-8')
        .expect(200, indexHtmlContent, done);
    });

    it('responds with a modified index.html if it is a kubeflow deployment', done => {
      const expectedIndexHtml = `
<html>
<head>
  <script>
  window.KFP_FLAGS.DEPLOYMENT="KUBEFLOW"
  </script>
  <script id="kubeflow-client-placeholder" src="/dashboard_lib.bundle.js"></script>
</head>
</html>`;
      const configs = loadConfigs(argv, { DEPLOYMENT: 'kubeflow' });
      app = new UIServer(configs);

      const request = requests(app.start());
      request
        .get('/')
        .expect('Content-Type', 'text/html; charset=utf-8')
        .expect(200, expectedIndexHtml, done);
    });

    it('responds with flag DEPLOYMENT=MARKETPLACE if it is a marketplace deployment', done => {
      const expectedIndexHtml = `
<html>
<head>
  <script>
  window.KFP_FLAGS.DEPLOYMENT="MARKETPLACE"
  </script>
  <script id="kubeflow-client-placeholder"></script>
</head>
</html>`;
      const configs = loadConfigs(argv, { DEPLOYMENT: 'marketplace' });
      app = new UIServer(configs);

      const request = requests(app.start());
      request
        .get('/')
        .expect('Content-Type', 'text/html; charset=utf-8')
        .expect(200, expectedIndexHtml, done);
    });
  });

  describe('/apis/v1beta1/healthz', () => {
    it('responds with apiServerReady to be false if ml-pipeline api server is not ready.', done => {
      (fetch as any).mockImplementationOnce((_url: string, _opt: any) => ({
        json: () => Promise.reject('Unknown error'),
      }));

      const configs = loadConfigs(argv, {});
      app = new UIServer(configs);
      requests(app.start())
        .get('/apis/v1beta1/healthz')
        .expect(
          200,
          {
            apiServerReady: false,
            buildDate,
            frontendCommitHash: commitHash,
            frontendTagName: tagName,
          },
          done,
        );
    });

    it('responds with both ui server and ml-pipeline api state if ml-pipeline api server is also ready.', done => {
      (fetch as any).mockImplementationOnce((_url: string, _opt: any) => ({
        json: () =>
          Promise.resolve({
            commit_sha: 'commit_sha',
            tag_name: '1.0.0',
          }),
      }));

      const configs = loadConfigs(argv, {});
      app = new UIServer(configs);
      requests(app.start())
        .get('/apis/v1beta1/healthz')
        .expect(
          200,
          {
            apiServerCommitHash: 'commit_sha',
            apiServerTagName: '1.0.0',
            apiServerReady: true,
            buildDate,
            frontendCommitHash: commitHash,
            frontendTagName: tagName,
          },
          done,
        );
    });
  });

  describe('/artifacts/get', () => {
    it('responds with a minio artifact if source=minio', done => {
      const artifactContent = 'hello world';
      const mockedMinioClient: jest.Mock = MinioClient as any;
      const mockedGetObjectStream: jest.Mock = minioHelper.getObjectStream as any;
      const objStream = new PassThrough();
      objStream.end(artifactContent);

      mockedGetObjectStream.mockImplementationOnce(opt =>
        opt.bucket === 'ml-pipeline' && opt.key === 'hello/world.txt'
          ? Promise.resolve(objStream)
          : Promise.reject('Unable to retrieve minio artifact.'),
      );
      const configs = loadConfigs(argv, {
        MINIO_ACCESS_KEY: 'minio',
        MINIO_HOST: 'minio-service',
        MINIO_NAMESPACE: 'kubeflow',
        MINIO_PORT: '9000',
        MINIO_SECRET_KEY: 'minio123',
        MINIO_SSL: 'false',
      });
      app = new UIServer(configs);

      const request = requests(app.start());
      request
        .get('/artifacts/get?source=minio&bucket=ml-pipeline&key=hello%2Fworld.txt')
        .expect(200, artifactContent, err => {
          expect(mockedMinioClient).toBeCalledWith({
            accessKey: 'minio',
            endPoint: 'minio-service.kubeflow',
            port: 9000,
            secretKey: 'minio123',
            useSSL: false,
          });
          done(err);
        });
    });

    it('responds with a s3 artifact if source=s3', done => {
      const artifactContent = 'hello world';
      const mockedMinioClient: jest.Mock = minioHelper.createMinioClient as any;
      const mockedGetObjectStream: jest.Mock = minioHelper.getObjectStream as any;
      const stream = new PassThrough();
      stream.write(artifactContent);
      stream.end();

      mockedGetObjectStream.mockImplementationOnce(opt =>
        opt.bucket === 'ml-pipeline' && opt.key === 'hello/world.txt'
          ? Promise.resolve(stream)
          : Promise.reject('Unable to retrieve s3 artifact.'),
      );
      const configs = loadConfigs(argv, {
        AWS_ACCESS_KEY_ID: 'aws123',
        AWS_SECRET_ACCESS_KEY: 'awsSecret123',
      });
      app = new UIServer(configs);

      const request = requests(app.start());
      request
        .get('/artifacts/get?source=s3&bucket=ml-pipeline&key=hello%2Fworld.txt')
        .expect(200, artifactContent, err => {
          expect(mockedMinioClient).toBeCalledWith({
            accessKey: 'aws123',
            endPoint: 's3.amazonaws.com',
            secretKey: 'awsSecret123',
          });
          done(err);
        });
    });

    it('responds with partial s3 artifact if peek=5 flag is set', done => {
      const artifactContent = 'hello world';
      const mockedMinioClient: jest.Mock = minioHelper.createMinioClient as any;
      const mockedGetObjectStream: jest.Mock = minioHelper.getObjectStream as any;
      const stream = new PassThrough();
      stream.write(artifactContent);
      stream.end();

      mockedGetObjectStream.mockImplementationOnce(opt =>
        opt.bucket === 'ml-pipeline' && opt.key === 'hello/world.txt'
          ? Promise.resolve(stream)
          : Promise.reject('Unable to retrieve s3 artifact.'),
      );
      const configs = loadConfigs(argv, {
        AWS_ACCESS_KEY_ID: 'aws123',
        AWS_SECRET_ACCESS_KEY: 'awsSecret123',
      });
      app = new UIServer(configs);

      const request = requests(app.start());
      request
        .get('/artifacts/get?source=s3&bucket=ml-pipeline&key=hello%2Fworld.txt&peek=5')
        .expect(200, artifactContent.slice(0, 5), err => {
          expect(mockedMinioClient).toBeCalledWith({
            accessKey: 'aws123',
            endPoint: 's3.amazonaws.com',
            secretKey: 'awsSecret123',
          });
          done(err);
        });
    });

    it('responds with a http artifact if source=http', done => {
      const artifactContent = 'hello world';
      mockedFetch.mockImplementationOnce((url: string, opts: any) =>
        url === 'http://foo.bar/ml-pipeline/hello/world.txt'
          ? Promise.resolve({
              buffer: () => Promise.resolve(artifactContent),
              body: new PassThrough().end(artifactContent),
            })
          : Promise.reject('Unable to retrieve http artifact.'),
      );
      const configs = loadConfigs(argv, {
        HTTP_BASE_URL: 'foo.bar/',
      });
      app = new UIServer(configs);

      const request = requests(app.start());
      request
        .get('/artifacts/get?source=http&bucket=ml-pipeline&key=hello%2Fworld.txt')
        .expect(200, artifactContent, err => {
          expect(mockedFetch).toBeCalledWith('http://foo.bar/ml-pipeline/hello/world.txt', {
            headers: {},
          });
          done(err);
        });
    });

    it('responds with partial http artifact if peek=5 flag is set', done => {
      const artifactContent = 'hello world';
      const mockedFetch: jest.Mock = fetch as any;
      mockedFetch.mockImplementationOnce((url: string, opts: any) =>
        url === 'http://foo.bar/ml-pipeline/hello/world.txt'
          ? Promise.resolve({
              buffer: () => Promise.resolve(artifactContent),
              body: new PassThrough().end(artifactContent),
            })
          : Promise.reject('Unable to retrieve http artifact.'),
      );
      const configs = loadConfigs(argv, {
        HTTP_BASE_URL: 'foo.bar/',
      });
      app = new UIServer(configs);

      const request = requests(app.start());
      request
        .get('/artifacts/get?source=http&bucket=ml-pipeline&key=hello%2Fworld.txt&peek=5')
        .expect(200, artifactContent.slice(0, 5), err => {
          expect(mockedFetch).toBeCalledWith('http://foo.bar/ml-pipeline/hello/world.txt', {
            headers: {},
          });
          done(err);
        });
    });

    it('responds with a https artifact if source=https', done => {
      const artifactContent = 'hello world';
      mockedFetch.mockImplementationOnce((url: string, opts: any) =>
        url === 'https://foo.bar/ml-pipeline/hello/world.txt' &&
        opts.headers.Authorization === 'someToken'
          ? Promise.resolve({
              buffer: () => Promise.resolve(artifactContent),
              body: new PassThrough().end(artifactContent),
            })
          : Promise.reject('Unable to retrieve http artifact.'),
      );
      const configs = loadConfigs(argv, {
        HTTP_AUTHORIZATION_DEFAULT_VALUE: 'someToken',
        HTTP_AUTHORIZATION_KEY: 'Authorization',
        HTTP_BASE_URL: 'foo.bar/',
      });
      app = new UIServer(configs);

      const request = requests(app.start());
      request
        .get('/artifacts/get?source=https&bucket=ml-pipeline&key=hello%2Fworld.txt')
        .expect(200, artifactContent, err => {
          expect(mockedFetch).toBeCalledWith('https://foo.bar/ml-pipeline/hello/world.txt', {
            headers: {
              Authorization: 'someToken',
            },
          });
          done(err);
        });
    });

    it('responds with a https artifact using the inherited header if source=https and http authorization key is provided.', done => {
      const artifactContent = 'hello world';
      mockedFetch.mockImplementationOnce((url: string, _opts: any) =>
        url === 'https://foo.bar/ml-pipeline/hello/world.txt'
          ? Promise.resolve({
              buffer: () => Promise.resolve(artifactContent),
              body: new PassThrough().end(artifactContent),
            })
          : Promise.reject('Unable to retrieve http artifact.'),
      );
      const configs = loadConfigs(argv, {
        HTTP_AUTHORIZATION_KEY: 'Authorization',
        HTTP_BASE_URL: 'foo.bar/',
      });
      app = new UIServer(configs);

      const request = requests(app.start());
      request
        .get('/artifacts/get?source=https&bucket=ml-pipeline&key=hello%2Fworld.txt')
        .set('Authorization', 'inheritedToken')
        .expect(200, artifactContent, err => {
          expect(mockedFetch).toBeCalledWith('https://foo.bar/ml-pipeline/hello/world.txt', {
            headers: {
              Authorization: 'inheritedToken',
            },
          });
          done(err);
        });
    });

    it('responds with a gcs artifact if source=gcs', done => {
      const artifactContent = 'hello world';
      const mockedGcsStorage: jest.Mock = GCSStorage as any;
      const stream = new PassThrough();
      stream.write(artifactContent);
      stream.end();
      mockedGcsStorage.mockImplementationOnce(() => ({
        bucket: () => ({
          getFiles: () =>
            Promise.resolve([[{ name: 'hello/world.txt', createReadStream: () => stream }]]),
        }),
      }));
      const configs = loadConfigs(argv, {});
      app = new UIServer(configs);

      const request = requests(app.start());
      request
        .get('/artifacts/get?source=gcs&bucket=ml-pipeline&key=hello%2Fworld.txt')
        .expect(200, artifactContent + '\n', done);
    });

    it('responds with a partial gcs artifact if peek=5 is set', done => {
      const artifactContent = 'hello world';
      const mockedGcsStorage: jest.Mock = GCSStorage as any;
      const stream = new PassThrough();
      stream.end(artifactContent);
      mockedGcsStorage.mockImplementationOnce(() => ({
        bucket: () => ({
          getFiles: () =>
            Promise.resolve([[{ name: 'hello/world.txt', createReadStream: () => stream }]]),
        }),
      }));
      const configs = loadConfigs(argv, {});
      app = new UIServer(configs);

      const request = requests(app.start());
      request
        .get('/artifacts/get?source=gcs&bucket=ml-pipeline&key=hello%2Fworld.txt&peek=5')
        .expect(200, artifactContent.slice(0, 5), done);
    });
  });

  describe('/system', () => {
    describe('/cluster-name', () => {
      it('responds with cluster name data from gke metadata', done => {
        mockedFetch.mockImplementationOnce((url: string, _opts: any) =>
          url === 'http://metadata/computeMetadata/v1/instance/attributes/cluster-name'
            ? Promise.resolve({ ok: true, text: () => Promise.resolve('test-cluster') })
            : Promise.reject('Unexpected request'),
        );
        app = new UIServer(loadConfigs(argv, {}));

        const request = requests(app.start());
        request
          .get('/system/cluster-name')
          .expect('Content-Type', 'text/html; charset=utf-8')
          .expect(200, 'test-cluster', done);
      });
      it('responds with 500 status code if corresponding endpoint is not ok', done => {
        mockedFetch.mockImplementationOnce((url: string, _opts: any) =>
          url === 'http://metadata/computeMetadata/v1/instance/attributes/cluster-name'
            ? Promise.resolve({ ok: false, text: () => Promise.resolve('404 not found') })
            : Promise.reject('Unexpected request'),
        );
        app = new UIServer(loadConfigs(argv, {}));

        const request = requests(app.start());
        request.get('/system/cluster-name').expect(500, 'Failed fetching GKE cluster name', done);
      });
      it('responds with endpoint disabled if DISABLE_GKE_METADATA env is true', done => {
        const configs = loadConfigs(argv, { DISABLE_GKE_METADATA: 'true' });
        app = new UIServer(configs);

        const request = requests(app.start());
        request
          .get('/system/cluster-name')
          .expect(500, 'GKE metadata endpoints are disabled.', done);
      });
    });

    describe('/project-id', () => {
      it('responds with project id data from gke metadata', done => {
        mockedFetch.mockImplementationOnce((url: string, _opts: any) =>
          url === 'http://metadata/computeMetadata/v1/project/project-id'
            ? Promise.resolve({ ok: true, text: () => Promise.resolve('test-project') })
            : Promise.reject('Unexpected request'),
        );
        app = new UIServer(loadConfigs(argv, {}));

        const request = requests(app.start());
        request.get('/system/project-id').expect(200, 'test-project', done);
      });
      it('responds with 500 status code if metadata request is not ok', done => {
        mockedFetch.mockImplementationOnce((url: string, _opts: any) =>
          url === 'http://metadata/computeMetadata/v1/project/project-id'
            ? Promise.resolve({ ok: false, text: () => Promise.resolve('404 not found') })
            : Promise.reject('Unexpected request'),
        );
        app = new UIServer(loadConfigs(argv, {}));

        const request = requests(app.start());
        request.get('/system/project-id').expect(500, 'Failed fetching GKE project id', done);
      });
      it('responds with endpoint disabled if DISABLE_GKE_METADATA env is true', done => {
        app = new UIServer(loadConfigs(argv, { DISABLE_GKE_METADATA: 'true' }));

        const request = requests(app.start());
        request.get('/system/project-id').expect(500, 'GKE metadata endpoints are disabled.', done);
      });
    });
  });

  describe('/k8s/pod', () => {
    let request: requests.SuperTest<requests.Test>;
    beforeEach(() => {
      app = new UIServer(loadConfigs(argv, {}));
      request = requests(app.start());
    });

    it('asks for podname if not provided', done => {
      request.get('/k8s/pod').expect(422, 'podname argument is required', done);
    });

    it('asks for podnamespace if not provided', done => {
      request
        .get('/k8s/pod?podname=test-pod')
        .expect(422, 'podnamespace argument is required', done);
    });

    it('responds with pod info in JSON', done => {
      const readPodSpy = jest.spyOn(K8S_TEST_EXPORT.k8sV1Client, 'readNamespacedPod');
      readPodSpy.mockImplementation(() =>
        Promise.resolve({
          body: { kind: 'Pod' }, // only body is used
        } as any),
      );
      request
        .get('/k8s/pod?podname=test-pod&podnamespace=test-ns')
        .expect(200, '{"kind":"Pod"}', err => {
          expect(readPodSpy).toHaveBeenCalledWith('test-pod', 'test-ns');
          done(err);
        });
    });

    it('responds with error when failed to retrieve pod info', done => {
      const readPodSpy = jest.spyOn(K8S_TEST_EXPORT.k8sV1Client, 'readNamespacedPod');
      readPodSpy.mockImplementation(() =>
        Promise.reject({
          body: {
            message: 'pod not found',
            code: 404,
          },
        } as any),
      );
      const spyError = jest.spyOn(console, 'error').mockImplementation(() => null);
      request
        .get('/k8s/pod?podname=test-pod&podnamespace=test-ns')
        .expect(500, 'Could not get pod test-pod in namespace test-ns: pod not found', () => {
          expect(spyError).toHaveBeenCalledTimes(1);
          done();
        });
    });
  });

  describe('/k8s/pod/events', () => {
    let request: requests.SuperTest<requests.Test>;
    beforeEach(() => {
      app = new UIServer(loadConfigs(argv, {}));
      request = requests(app.start());
    });

    it('asks for podname if not provided', done => {
      request.get('/k8s/pod/events').expect(422, 'podname argument is required', done);
    });

    it('asks for podnamespace if not provided', done => {
      request
        .get('/k8s/pod/events?podname=test-pod')
        .expect(422, 'podnamespace argument is required', done);
    });

    it('responds with pod info in JSON', done => {
      const listEventSpy = jest.spyOn(K8S_TEST_EXPORT.k8sV1Client, 'listNamespacedEvent');
      listEventSpy.mockImplementation(() =>
        Promise.resolve({
          body: { kind: 'EventList' }, // only body is used
        } as any),
      );
      request
        .get('/k8s/pod/events?podname=test-pod&podnamespace=test-ns')
        .expect(200, '{"kind":"EventList"}', err => {
          expect(listEventSpy).toHaveBeenCalledWith(
            'test-ns',
            undefined,
            undefined,
            undefined,
            'involvedObject.namespace=test-ns,involvedObject.name=test-pod,involvedObject.kind=Pod',
          );
          done(err);
        });
    });

    it('responds with error when failed to retrieve pod info', done => {
      const listEventSpy = jest.spyOn(K8S_TEST_EXPORT.k8sV1Client, 'listNamespacedEvent');
      listEventSpy.mockImplementation(() =>
        Promise.reject({
          body: {
            message: 'no events',
            code: 404,
          },
        } as any),
      );
      const spyError = jest.spyOn(console, 'error').mockImplementation(() => null);
      request
        .get('/k8s/pod/events?podname=test-pod&podnamespace=test-ns')
        .expect(
          500,
          'Error when listing pod events for pod "test-pod" in "test-ns" namespace: no events',
          err => {
            expect(spyError).toHaveBeenCalledTimes(1);
            done(err);
          },
        );
    });
  });

  describe('/apps/tensorboard', () => {
    let k8sGetCustomObjectSpy: jest.SpyInstance;
    let k8sDeleteCustomObjectSpy: jest.SpyInstance;
    let k8sCreateCustomObjectSpy: jest.SpyInstance;
    let kfpApiServer: Server;

    function newGetTensorboardResponse({
      name = 'viewer-example',
      logDir = 'log-dir-example',
      tensorflowImage = 'tensorflow:2.0.0',
      type = 'tensorboard',
    }: {
      name?: string;
      logDir?: string;
      tensorflowImage?: string;
      type?: string;
    } = {}) {
      return {
        response: undefined as any, // unused
        body: {
          metadata: {
            name,
          },
          spec: {
            tensorboardSpec: { logDir, tensorflowImage },
            type,
          },
        },
      };
    }

    beforeEach(() => {
      k8sGetCustomObjectSpy = jest.spyOn(
        K8S_TEST_EXPORT.k8sV1CustomObjectClient,
        'getNamespacedCustomObject',
      );
      k8sDeleteCustomObjectSpy = jest.spyOn(
        K8S_TEST_EXPORT.k8sV1CustomObjectClient,
        'deleteNamespacedCustomObject',
      );
      k8sCreateCustomObjectSpy = jest.spyOn(
        K8S_TEST_EXPORT.k8sV1CustomObjectClient,
        'createNamespacedCustomObject',
      );
    });

    afterEach(() => {
      if (kfpApiServer) {
        kfpApiServer.close();
      }
    });

    describe('get', () => {
      it('requires logdir for get tensorboard', done => {
        app = new UIServer(loadConfigs(argv, {}));
        requests(app.start())
          .get('/apps/tensorboard')
          .expect(400, 'logdir argument is required', done);
      });

      it('requires namespace for get tensorboard', done => {
        app = new UIServer(loadConfigs(argv, {}));
        requests(app.start())
          .get('/apps/tensorboard?logdir=some-log-dir')
          .expect(400, 'namespace argument is required', done);
      });

      it('does not crash with a weird query', done => {
        app = new UIServer(loadConfigs(argv, {}));
        k8sGetCustomObjectSpy.mockImplementation(() =>
          Promise.resolve(newGetTensorboardResponse()),
        );
        // The special case is that, decodeURIComponent('%2') throws an
        // exception, so this can verify handler doesn't do extra
        // decodeURIComponent on queries.
        const weirdLogDir = encodeURIComponent('%2');
        requests(app.start())
          .get(`/apps/tensorboard?logdir=${weirdLogDir}&namespace=test-ns`)
          .expect(200, done);
      });

      function setupMockKfpApiService({ port = 3001 }: { port?: number } = {}) {
        const receivedHeaders: any[] = [];
        kfpApiServer = express()
          .get('/apis/v1beta1/auth', (req, res) => {
            receivedHeaders.push(req.headers);
            res.status(200).send('{}'); // Authorized
          })
          .listen(port);
        return { receivedHeaders, host: 'localhost', port };
      }

      it('authorizes user requests from KFP auth api', done => {
        const { receivedHeaders, host, port } = setupMockKfpApiService();
        app = new UIServer(
          loadConfigs(argv, {
            ENABLE_AUTHZ: 'true',
            ML_PIPELINE_SERVICE_PORT: `${port}`,
            ML_PIPELINE_SERVICE_HOST: host,
          }),
        );
        k8sGetCustomObjectSpy.mockImplementation(() =>
          Promise.resolve(newGetTensorboardResponse()),
        );
        requests(app.start())
          .get(`/apps/tensorboard?logdir=some-log-dir&namespace=test-ns`)
          .set('x-goog-authenticated-user-email', 'accounts.google.com:[email protected]')
          .expect(200, err => {
            expect(receivedHeaders).toHaveLength(1);
            expect(receivedHeaders[0]).toMatchInlineSnapshot(`
              Object {
                "accept": "*/*",
                "accept-encoding": "gzip,deflate",
                "connection": "close",
                "host": "localhost:3001",
                "user-agent": "node-fetch/1.0 (+https://github.com/bitinn/node-fetch)",
                "x-goog-authenticated-user-email": "accounts.google.com:[email protected]",
              }
            `);
            done(err);
          });
      });

      it('uses configured KUBEFLOW_USERID_HEADER for user identity', done => {
        const { receivedHeaders, host, port } = setupMockKfpApiService();
        app = new UIServer(
          loadConfigs(argv, {
            ENABLE_AUTHZ: 'true',
            KUBEFLOW_USERID_HEADER: 'x-kubeflow-userid',
            ML_PIPELINE_SERVICE_PORT: `${port}`,
            ML_PIPELINE_SERVICE_HOST: host,
          }),
        );
        k8sGetCustomObjectSpy.mockImplementation(() =>
          Promise.resolve(newGetTensorboardResponse()),
        );
        requests(app.start())
          .get(`/apps/tensorboard?logdir=some-log-dir&namespace=test-ns`)
          .set('x-kubeflow-userid', '[email protected]')
          .expect(200, err => {
            expect(receivedHeaders).toHaveLength(1);
            expect(receivedHeaders[0]).toHaveProperty('x-kubeflow-userid', '[email protected]');
            done(err);
          });
      });

      it('rejects user requests when KFP auth api rejected', done => {
        const errorSpy = jest.spyOn(console, 'error');
        errorSpy.mockImplementation();

        const apiServerPort = 3001;
        kfpApiServer = express()
          .get('/apis/v1beta1/auth', (_, res) => {
            res.status(400).send(
              JSON.stringify({
                error: 'User xxx is not unauthorized to list viewers',
                details: ['unauthorized', 'callstack'],
              }),
            );
          })
          .listen(apiServerPort);
        app = new UIServer(
          loadConfigs(argv, {
            ENABLE_AUTHZ: 'true',
            ML_PIPELINE_SERVICE_PORT: `${apiServerPort}`,
            ML_PIPELINE_SERVICE_HOST: 'localhost',
          }),
        );
        k8sGetCustomObjectSpy.mockImplementation(() =>
          Promise.resolve(newGetTensorboardResponse()),
        );
        requests(app.start())
          .get(`/apps/tensorboard?logdir=some-log-dir&namespace=test-ns`)
          .expect(
            401,
            'User is not authorized to GET VIEWERS in namespace test-ns: User xxx is not unauthorized to list viewers',
            err => {
              expect(errorSpy).toHaveBeenCalledTimes(1);
              expect(
                errorSpy,
              ).toHaveBeenCalledWith(
                'User is not authorized to GET VIEWERS in namespace test-ns: User xxx is not unauthorized to list viewers',
                ['unauthorized', 'callstack'],
              );
              done(err);
            },
          );
      });

      it('gets tensorboard url and version', done => {
        app = new UIServer(loadConfigs(argv, {}));
        k8sGetCustomObjectSpy.mockImplementation(() =>
          Promise.resolve(
            newGetTensorboardResponse({
              name: 'viewer-abcdefg',
              logDir: 'log-dir-1',
              tensorflowImage: 'tensorflow:2.0.0',
            }),
          ),
        );

        requests(app.start())
          .get(`/apps/tensorboard?logdir=${encodeURIComponent('log-dir-1')}&namespace=test-ns`)
          .expect(
            200,
            JSON.stringify({
              podAddress:
                'http://viewer-abcdefg-service.test-ns.svc.cluster.local:80/tensorboard/viewer-abcdefg/',
              tfVersion: '2.0.0',
            }),
            err => {
              expect(k8sGetCustomObjectSpy.mock.calls[0]).toMatchInlineSnapshot(`
                              Array [
                                "kubeflow.org",
                                "v1beta1",
                                "test-ns",
                                "viewers",
                                "viewer-5e1404e679e27b0f0b8ecee8fe515830eaa736c5",
                              ]
                          `);
              done(err);
            },
          );
      });
    });

    describe('post (create)', () => {
      it('requires logdir', done => {
        app = new UIServer(loadConfigs(argv, {}));
        requests(app.start())
          .post('/apps/tensorboard')
          .expect(400, 'logdir argument is required', done);
      });

      it('requires namespace', done => {
        app = new UIServer(loadConfigs(argv, {}));
        requests(app.start())
          .post('/apps/tensorboard?logdir=some-log-dir')
          .expect(400, 'namespace argument is required', done);
      });

      it('requires tfversion', done => {
        app = new UIServer(loadConfigs(argv, {}));
        requests(app.start())
          .post('/apps/tensorboard?logdir=some-log-dir&namespace=test-ns')
          .expect(400, 'tfversion (tensorflow version) argument is required', done);
      });

      it('creates tensorboard viewer custom object and waits for it', done => {
        let getRequestCount = 0;
        k8sGetCustomObjectSpy.mockImplementation(() => {
          ++getRequestCount;
          switch (getRequestCount) {
            case 1:
              return Promise.reject('Not found');
            case 2:
              return Promise.resolve(
                newGetTensorboardResponse({
                  name: 'viewer-abcdefg',
                  logDir: 'log-dir-1',
                  tensorflowImage: 'tensorflow:2.0.0',
                }),
              );
            default:
              throw new Error('only expected to be called twice in this test');
          }
        });
        k8sCreateCustomObjectSpy.mockImplementation(() => Promise.resolve());

        app = new UIServer(loadConfigs(argv, {}));
        requests(app.start())
          .post(
            `/apps/tensorboard?logdir=${encodeURIComponent(
              'log-dir-1',
            )}&namespace=test-ns&tfversion=2.0.0`,
          )
          .expect(
            200,
            'http://viewer-abcdefg-service.test-ns.svc.cluster.local:80/tensorboard/viewer-abcdefg/',
            err => {
              expect(k8sGetCustomObjectSpy.mock.calls[0]).toMatchInlineSnapshot(`
                Array [
                  "kubeflow.org",
                  "v1beta1",
                  "test-ns",
                  "viewers",
                  "viewer-5e1404e679e27b0f0b8ecee8fe515830eaa736c5",
                ]
              `);
              expect(k8sCreateCustomObjectSpy.mock.calls[0]).toMatchInlineSnapshot(`
                Array [
                  "kubeflow.org",
                  "v1beta1",
                  "test-ns",
                  "viewers",
                  Object {
                    "apiVersion": "kubeflow.org/v1beta1",
                    "kind": "Viewer",
                    "metadata": Object {
                      "name": "viewer-5e1404e679e27b0f0b8ecee8fe515830eaa736c5",
                      "namespace": "test-ns",
                    },
                    "spec": Object {
                      "podTemplateSpec": Object {
                        "spec": Object {
                          "containers": Array [
                            Object {},
                          ],
                        },
                      },
                      "tensorboardSpec": Object {
                        "logDir": "log-dir-1",
                        "tensorflowImage": "tensorflow/tensorflow:2.0.0",
                      },
                      "type": "tensorboard",
                    },
                  },
                ]
              `);
              expect(k8sGetCustomObjectSpy.mock.calls[1]).toMatchInlineSnapshot(`
                Array [
                  "kubeflow.org",
                  "v1beta1",
                  "test-ns",
                  "viewers",
                  "viewer-5e1404e679e27b0f0b8ecee8fe515830eaa736c5",
                ]
              `);
              done(err);
            },
          );
      });

      it('returns error when there is an existing tensorboard with different version', done => {
        const errorSpy = jest.spyOn(console, 'error');
        errorSpy.mockImplementation();
        k8sGetCustomObjectSpy.mockImplementation(() =>
          Promise.resolve(
            newGetTensorboardResponse({
              name: 'viewer-abcdefg',
              logDir: 'log-dir-1',
              tensorflowImage: 'tensorflow:2.1.0',
            }),
          ),
        );
        k8sCreateCustomObjectSpy.mockImplementation(() => Promise.resolve());

        app = new UIServer(loadConfigs(argv, {}));
        requests(app.start())
          .post(
            `/apps/tensorboard?logdir=${encodeURIComponent(
              'log-dir-1',
            )}&namespace=test-ns&tfversion=2.0.0`,
          )
          .expect(
            500,
            `Failed to start Tensorboard app: There's already an existing tensorboard instance with a different version 2.1.0`,
            err => {
              expect(errorSpy).toHaveBeenCalledTimes(1);
              done(err);
            },
          );
      });

      it('returns existing pod address if there is an existing tensorboard with the same version', done => {
        k8sGetCustomObjectSpy.mockImplementation(() =>
          Promise.resolve(
            newGetTensorboardResponse({
              name: 'viewer-abcdefg',
              logDir: 'log-dir-1',
              tensorflowImage: 'tensorflow:2.0.0',
            }),
          ),
        );
        k8sCreateCustomObjectSpy.mockImplementation(() => Promise.resolve());

        app = new UIServer(loadConfigs(argv, {}));
        requests(app.start())
          .post(
            `/apps/tensorboard?logdir=${encodeURIComponent(
              'log-dir-1',
            )}&namespace=test-ns&tfversion=2.0.0`,
          )
          .expect(
            200,
            'http://viewer-abcdefg-service.test-ns.svc.cluster.local:80/tensorboard/viewer-abcdefg/',
            done,
          );
      });
    });

    describe('delete', () => {
      it('requires logdir', done => {
        app = new UIServer(loadConfigs(argv, {}));
        requests(app.start())
          .delete('/apps/tensorboard')
          .expect(400, 'logdir argument is required', done);
      });

      it('requires namespace', done => {
        app = new UIServer(loadConfigs(argv, {}));
        requests(app.start())
          .delete('/apps/tensorboard?logdir=some-log-dir')
          .expect(400, 'namespace argument is required', done);
      });

      it('deletes tensorboard viewer custom object', done => {
        k8sGetCustomObjectSpy.mockImplementation(() =>
          Promise.resolve(
            newGetTensorboardResponse({
              name: 'viewer-abcdefg',
              logDir: 'log-dir-1',
              tensorflowImage: 'tensorflow:2.0.0',
            }),
          ),
        );
        k8sDeleteCustomObjectSpy.mockImplementation(() => Promise.resolve());

        app = new UIServer(loadConfigs(argv, {}));
        requests(app.start())
          .delete(`/apps/tensorboard?logdir=${encodeURIComponent('log-dir-1')}&namespace=test-ns`)
          .expect(200, 'Tensorboard deleted.', err => {
            expect(k8sDeleteCustomObjectSpy.mock.calls[0]).toMatchInlineSnapshot(`
              Array [
                "kubeflow.org",
                "v1beta1",
                "test-ns",
                "viewers",
                "viewer-5e1404e679e27b0f0b8ecee8fe515830eaa736c5",
                V1DeleteOptions {},
              ]
            `);
            done(err);
          });
      });
    });
  });

  // TODO: Add integration tests for k8s helper related endpoints
  // describe('/k8s/pod/logs', () => {});

  describe('/apis/v1beta1/', () => {
    let request: requests.SuperTest<requests.Test>;
    let kfpApiServer: Server;

    beforeEach(() => {
      const kfpApiPort = 3001;
      kfpApiServer = express()
        .all('/*', (_, res) => {
          res.status(200).send('KFP API is working');
        })
        .listen(kfpApiPort);
      app = new UIServer(
        loadConfigs(argv, {
          ML_PIPELINE_SERVICE_PORT: `${kfpApiPort}`,
          ML_PIPELINE_SERVICE_HOST: 'localhost',
        }),
      );
      request = requests(app.start());
    });

    afterEach(() => {
      if (kfpApiServer) {
        kfpApiServer.close();
      }
    });

    it('rejects reportMetrics because it is not public kfp api', done => {
      const runId = 'a-random-run-id';
      request
        .post(`/apis/v1beta1/runs/${runId}:reportMetrics`)
        .expect(
          403,
          '/apis/v1beta1/runs/a-random-run-id:reportMetrics endpoint is not meant for external usage.',
          done,
        );
    });

    it('rejects reportWorkflow because it is not public kfp api', done => {
      const workflowId = 'a-random-workflow-id';
      request
        .post(`/apis/v1beta1/workflows/${workflowId}`)
        .expect(
          403,
          '/apis/v1beta1/workflows/a-random-workflow-id endpoint is not meant for external usage.',
          done,
        );
    });

    it('rejects reportScheduledWorkflow because it is not public kfp api', done => {
      const swf = 'a-random-swf-id';
      request
        .post(`/apis/v1beta1/scheduledworkflows/${swf}`)
        .expect(
          403,
          '/apis/v1beta1/scheduledworkflows/a-random-swf-id endpoint is not meant for external usage.',
          done,
        );
    });

    it('does not reject similar apis', done => {
      request // use reportMetrics as runId to see if it can confuse route parsing
        .post(`/apis/v1beta1/runs/xxx-reportMetrics:archive`)
        .expect(200, 'KFP API is working', done);
    });

    it('proxies other run apis', done => {
      request
        .post(`/apis/v1beta1/runs/a-random-run-id:archive`)
        .expect(200, 'KFP API is working', done);
    });
  });
});

describe('fs:rename', () => {
  const action = createFilesystemRenameAction();

  const mockInputFiles = [
    {
      from: 'unit-test-a.js',
      to: 'new-a.js',
    },
    {
      from: 'unit-test-b.js',
      to: 'new-b.js',
    },
    {
      from: 'a-folder',
      to: 'brand-new-folder',
    },
  ];
  const mockContext = {
    input: {
      files: mockInputFiles,
    },
    workspacePath,
    logger: getVoidLogger(),
    logStream: new PassThrough(),
    output: jest.fn(),
    createTemporaryDirectory: jest.fn(),
  };

  beforeEach(() => {
    jest.restoreAllMocks();

    mockFs({
      [workspacePath]: {
        'unit-test-a.js': 'hello',
        'unit-test-b.js': 'world',
        'unit-test-c.js': 'i will be overwritten :-(',
        'a-folder': {
          'file.md': 'content',
        },
      },
    });
  });

  afterEach(() => {
    mockFs.restore();
  });

  it('should throw an error when files is not an array', async () => {
    await expect(
      action.handler({
        ...mockContext,
        input: { files: undefined } as any,
      }),
    ).rejects.toThrow(/files must be an Array/);

    await expect(
      action.handler({
        ...mockContext,
        input: { files: {} } as any,
      }),
    ).rejects.toThrow(/files must be an Array/);

    await expect(
      action.handler({
        ...mockContext,
        input: { files: '' } as any,
      }),
    ).rejects.toThrow(/files must be an Array/);

    await expect(
      action.handler({
        ...mockContext,
        input: { files: null } as any,
      }),
    ).rejects.toThrow(/files must be an Array/);
  });

  it('should throw an error when files have missing from/to', async () => {
    await expect(
      action.handler({
        ...mockContext,
        input: { files: ['old.md'] } as any,
      }),
    ).rejects.toThrow(/each file must have a from and to property/);

    await expect(
      action.handler({
        ...mockContext,
        input: { files: [{ from: 'old.md' }] } as any,
      }),
    ).rejects.toThrow(/each file must have a from and to property/);

    await expect(
      action.handler({
        ...mockContext,
        input: { files: [{ to: 'new.md' }] } as any,
      }),
    ).rejects.toThrow(/each file must have a from and to property/);
  });

  it('should throw when file name is not relative to the workspace', async () => {
    await expect(
      action.handler({
        ...mockContext,
        input: { files: [{ from: 'index.js', to: '/core/../../../index.js' }] },
      }),
    ).rejects.toThrow(
      /Relative path is not allowed to refer to a directory outside its parent/,
    );

    await expect(
      action.handler({
        ...mockContext,
        input: { files: [{ from: '/core/../../../index.js', to: 'index.js' }] },
      }),
    ).rejects.toThrow(
      /Relative path is not allowed to refer to a directory outside its parent/,
    );
  });

  it('should throw is trying to override by mistake', async () => {
    const destFile = 'unit-test-c.js';
    const filePath = resolvePath(workspacePath, destFile);
    const beforeContent = fs.readFileSync(filePath, 'utf-8');

    await expect(
      action.handler({
        ...mockContext,
        input: {
          files: [
            {
              from: 'unit-test-a.js',
              to: 'unit-test-c.js',
            },
          ],
        },
      }),
    ).rejects.toThrow(/dest already exists/);

    const afterContent = fs.readFileSync(filePath, 'utf-8');

    expect(beforeContent).toEqual(afterContent);
  });

  it('should call fs.move with the correct values', async () => {
    mockInputFiles.forEach(file => {
      const filePath = resolvePath(workspacePath, file.from);
      const fileExists = fs.existsSync(filePath);
      expect(fileExists).toBe(true);
    });

    await action.handler(mockContext);

    mockInputFiles.forEach(file => {
      const filePath = resolvePath(workspacePath, file.from);
      const fileExists = fs.existsSync(filePath);
      expect(fileExists).toBe(false);
    });
  });

  it('should override when requested', async () => {
    const sourceFile = 'unit-test-a.js';
    const destFile = 'unit-test-c.js';
    const sourceFilePath = resolvePath(workspacePath, sourceFile);
    const destFilePath = resolvePath(workspacePath, destFile);

    const sourceBeforeContent = fs.readFileSync(sourceFilePath, 'utf-8');
    const destBeforeContent = fs.readFileSync(destFilePath, 'utf-8');

    expect(sourceBeforeContent).not.toEqual(destBeforeContent);

    await action.handler({
      ...mockContext,
      input: {
        files: [
          {
            from: sourceFile,
            to: destFile,
            overwrite: true,
          },
        ],
      },
    });

    const destAfterContent = fs.readFileSync(destFilePath, 'utf-8');

    expect(sourceBeforeContent).toEqual(destAfterContent);
  });
});

// see the interface declaration's explanation of ReceiveData
    async ReceiveData(payloadStream: Readable, user: User, options?: ReceiveDataOptions): Promise<Result<Import | DataStaging[]>> {
        if (!this.DataSourceRecord || !this.DataSourceRecord.id) {
            return Promise.resolve(Result.Failure('cannot receive data, no underlying or saved data source record'));
        }

        let internalTransaction = false;
        let transaction: PoolClient;

        if (options && options.transaction) {
            transaction = options.transaction;
        } else {
            const newTransaction = await this.#mapper.startTransaction();
            if (newTransaction.isError) return Promise.resolve(Result.Failure('unable to initiated db transaction'));

            transaction = newTransaction.value;
            internalTransaction = true;
        }

        if (!this.DataSourceRecord || !this.DataSourceRecord.id) {
            if (internalTransaction) await this.#mapper.rollbackTransaction(transaction);
            return Promise.resolve(
                Result.Failure(
                    `unable to receive data, data source either doesn't have a record present or data source needs to be saved prior to data being received`,
                ),
            );
        }

        let importID: string;

        if (options && options.importID) {
            importID = options.importID;
        } else {
            // we're not making the import as part of the transaction because even if we error, we want to record the
            // problem - and if we have 0 data retention on the source we need the import created prior to loading up
            // the data as messages for the process queue
            const newImport = await ImportMapper.Instance.CreateImport(
                user.id!,
                new Import({
                    data_source_id: this.DataSourceRecord.id,
                    reference: 'manual upload',
                }),
            );

            if (newImport.isError) {
                if (internalTransaction) await this.#mapper.rollbackTransaction(transaction);
                return Promise.resolve(Result.Failure(`unable to create import for data ${newImport.error?.error}`));
            }

            importID = newImport.value.id!;
        }

        // we used to lock this for receiving data, but it makes no sense as this is an additive process which does not
        // modify the import in any way. Locking prevented the mapper from running correctly.
        const retrievedImport = await this.#importRepo.findByID(importID, transaction);
        if (retrievedImport.isError) {
            if (internalTransaction) await this.#mapper.rollbackTransaction(transaction);
            Logger.error(`unable to retrieve and lock import ${retrievedImport.error}`);
            return Promise.resolve(Result.Failure(`unable to retrieve ${retrievedImport.error?.error}`));
        }

        // a buffer, once it's full we'll write these records to the database and wipe to start again
        let recordBuffer: DataStaging[] = [];

        // lets us wait for all save operations to complete - we can still fail fast on a bad import since all the
        // save operations will share the same database transaction under the hood - that is if we're saving and not
        // emitting the data straight to the queue
        const saveOperations: Promise<Result<boolean>>[] = [];

        // our PassThrough stream is what actually processes the data, it's the last step in our eventual pipe
        const pass = new PassThrough({objectMode: true});

        // default to storing the raw data
        if (
            !this.DataSourceRecord.config?.data_retention_days ||
            (this.DataSourceRecord.config?.data_retention_days && this.DataSourceRecord.config.data_retention_days !== 0)
        ) {
            pass.on('data', (data) => {
                recordBuffer.push(
                    new DataStaging({
                        data_source_id: this.DataSourceRecord!.id!,
                        import_id: retrievedImport.value.id!,
                        data,
                        shape_hash: TypeMapping.objectToShapeHash(data, {
                            value_nodes: this.DataSourceRecord?.config?.value_nodes,
                            stop_nodes: this.DataSourceRecord?.config?.stop_nodes,
                        }),
                    }),
                );

                // if we've reached the process record limit, insert into the database and wipe the records array
                // make sure to COPY the array into bulkSave function so that we can push it into the array of promises
                // and not modify the underlying array on save, allowing us to move asynchronously,
                if (recordBuffer.length >= Config.data_source_receive_buffer) {
                    // if we are returning
                    // the staging records, don't wipe the buffer just keep adding
                    if (!options || !options.returnStagingRecords) {
                        const toSave = [...recordBuffer];
                        recordBuffer = [];

                        saveOperations.push(this.#stagingRepo.bulkSave(toSave, transaction));
                    }
                }
            });

            // catch any records remaining in the buffer
            pass.on('end', () => {
                saveOperations.push(this.#stagingRepo.bulkSave(recordBuffer, transaction));
            });
        } else {
            // if data retention isn't configured, or it is 0 - we do not retain the data permanently
            // instead of our pipe saving data staging records to the database and having them emitted
            // later we immediately put the full message on the queue for processing - we also only log
            // errors that we encounter when putting on the queue, we don't fail outright
            const queue = await QueueFactory();

            pass.on('data', (data) => {
                const staging = new DataStaging({
                    data_source_id: this.DataSourceRecord!.id!,
                    import_id: retrievedImport.value.id!,
                    data,
                    shape_hash: TypeMapping.objectToShapeHash(data, {
                        value_nodes: this.DataSourceRecord?.config?.value_nodes,
                        stop_nodes: this.DataSourceRecord?.config?.stop_nodes,
                    }),
                });

                queue.Put(Config.process_queue, staging).catch((err) => Logger.error(`unable to put data staging record on the queue ${err}`));
            });
        }

        // the JSONStream pipe is simple, parsing a single array of json objects into parts
        const fromJSON: Writable = JSONStream.parse('*');
        let errorMessage: any | undefined;

        // handle all transform streams, piping each in order
        if (options && options.transformStreams && options.transformStreams.length > 0) {
            let pipeline = payloadStream;

            for (const pipe of options.transformStreams) {
                pipeline = pipeline.pipe(pipe);
            }

            // for the pipe process to work correctly you must wait for the pipe to finish reading all data
            await new Promise((fulfill) =>
                pipeline
                    .pipe(fromJSON)
                    .on('error', (err: any) => {
                        errorMessage = err;
                        fulfill(err);
                    })
                    .pipe(pass)
                    .on('finish', fulfill),
            );
        } else if (options && options.overrideJsonStream) {
            await new Promise((fulfill) =>
                payloadStream
                    .pipe(pass)
                    .on('error', (err: any) => {
                        errorMessage = err;
                        fulfill(err);
                    })
                    .on('finish', fulfill),
            );
        } else {
            await new Promise((fulfill) =>
                payloadStream
                    .pipe(fromJSON)
                    .on('error', (err: any) => {
                        errorMessage = err;
                        fulfill(err);
                    })
                    .pipe(pass)
                    .on('finish', fulfill),
            );
        }

        if (errorMessage) {
            if (internalTransaction) await this.#mapper.rollbackTransaction(transaction);
            return Promise.resolve(Result.Failure(`unable to parse JSON: ${errorMessage}`));
        }

        // we have to wait until any save operations are complete before we can act on the pipe's results
        const saveResults = await Promise.all(saveOperations);

        // if any of the save operations have an error, fail and rollback the transaction etc
        if (saveResults.filter((result) => result.isError || !result.value).length > 0) {
            if (internalTransaction) await this.#mapper.rollbackTransaction(transaction);
            return Promise.resolve(
                Result.Failure(
                    `one or more attempts to save data to the database failed, encountered the following errors: ${saveResults
                        .filter((r) => r.isError)
                        .map((r) => r.error)}`,
                ),
            );
        }

        if (internalTransaction) {
            const commit = await this.#mapper.completeTransaction(transaction);
            if (commit.isError) return Promise.resolve(Result.Pass(commit));
        }

        // now that we've committed to the database, send the records to the queue for the first time so they can attempt
        // to be processed if we have data retention enabled - we're doing this so we don't have the emitter constantly
        // sending records to the queue for processing
        if (
            !this.DataSourceRecord.config?.data_retention_days ||
            (this.DataSourceRecord.config?.data_retention_days && this.DataSourceRecord.config.data_retention_days !== 0)
        ) {
            const sent = await ImportMapper.Instance.SendToQueue(retrievedImport.value.id!);
            if (sent.isError) {
                Logger.error(`unable to queue new import ${sent}`);
            }
        }

        // return the saved buffer as we haven't wiped it, should contain all records with their updated IDs
        if (options && options.returnStagingRecords) {
            return new Promise((resolve) => resolve(Result.Success(recordBuffer)));
        }

        return new Promise((resolve) => resolve(Result.Success(retrievedImport.value)));
    }

describe('workflow-helper', () => {
  const minioConfig = {
    accessKey: 'minio',
    endPoint: 'minio-service.kubeflow',
    secretKey: 'minio123',
  };

  beforeEach(() => {
    jest.resetAllMocks();
  });

  describe('composePodLogsStreamHandler', () => {
    it('returns the stream from the default handler if there is no errors.', async () => {
      const defaultStream = new PassThrough();
      const defaultHandler = jest.fn((_podName: string, _namespace?: string) =>
        Promise.resolve(defaultStream),
      );
      const stream = await composePodLogsStreamHandler(defaultHandler)('podName', 'namespace');
      expect(defaultHandler).toBeCalledWith('podName', 'namespace');
      expect(stream).toBe(defaultStream);
    });

    it('returns the stream from the fallback handler if there is any error.', async () => {
      const fallbackStream = new PassThrough();
      const defaultHandler = jest.fn((_podName: string, _namespace?: string) =>
        Promise.reject('unknown error'),
      );
      const fallbackHandler = jest.fn((_podName: string, _namespace?: string) =>
        Promise.resolve(fallbackStream),
      );
      const stream = await composePodLogsStreamHandler(defaultHandler, fallbackHandler)(
        'podName',
        'namespace',
      );
      expect(defaultHandler).toBeCalledWith('podName', 'namespace');
      expect(fallbackHandler).toBeCalledWith('podName', 'namespace');
      expect(stream).toBe(fallbackStream);
    });

    it('throws error if both handler and fallback fails.', async () => {
      const defaultHandler = jest.fn((_podName: string, _namespace?: string) =>
        Promise.reject('unknown error for default'),
      );
      const fallbackHandler = jest.fn((_podName: string, _namespace?: string) =>
        Promise.reject('unknown error for fallback'),
      );
      await expect(
        composePodLogsStreamHandler(defaultHandler, fallbackHandler)('podName', 'namespace'),
      ).rejects.toEqual('unknown error for fallback');
    });
  });

  describe('getPodLogsStreamFromK8s', () => {
    it('returns the pod log stream using k8s api.', async () => {
      const mockedGetPodLogs: jest.Mock = getPodLogs as any;
      mockedGetPodLogs.mockResolvedValueOnce('pod logs');

      const stream = await getPodLogsStreamFromK8s('podName', 'namespace');
      expect(mockedGetPodLogs).toBeCalledWith('podName', 'namespace');
      expect(stream.read().toString()).toBe('pod logs');
    });
  });

  describe('toGetPodLogsStream', () => {
    it('wraps a getMinioRequestConfig function to return the corresponding object stream.', async () => {
      const objStream = new PassThrough();
      objStream.end('some fake logs.');

      const client = new MinioClient(minioConfig);
      const mockedClientGetObject: jest.Mock = client.getObject as any;
      mockedClientGetObject.mockResolvedValueOnce(objStream);
      const configs = {
        bucket: 'bucket',
        client,
        key: 'folder/key',
      };
      const createRequest = jest.fn((_podName: string, _namespace?: string) =>
        Promise.resolve(configs),
      );
      const stream = await toGetPodLogsStream(createRequest)('podName', 'namespace');
      expect(mockedClientGetObject).toBeCalledWith('bucket', 'folder/key');
    });
  });

  describe('createPodLogsMinioRequestConfig', () => {
    it('returns a MinioRequestConfig factory with the provided minioClientOptions, bucket, and prefix.', async () => {
      const mockedClient: jest.Mock = MinioClient as any;
      const requestFunc = await createPodLogsMinioRequestConfig(minioConfig, 'bucket', 'prefix');
      const request = await requestFunc('workflow-name-abc', 'namespace');

      expect(mockedClient).toBeCalledWith(minioConfig);
      expect(request.client).toBeInstanceOf(MinioClient);
      expect(request.bucket).toBe('bucket');
      expect(request.key).toBe('prefix/workflow-name/workflow-name-abc/main.log');
    });
  });

  describe('getPodLogsStreamFromWorkflow', () => {
    it('returns a getPodLogsStream function that retrieves an object stream using the workflow status corresponding to the pod name.', async () => {
      const sampleWorkflow = {
        apiVersion: 'argoproj.io/v1alpha1',
        kind: 'Workflow',
        status: {
          nodes: {
            'workflow-name-abc': {
              outputs: {
                artifacts: [
                  {
                    name: 'some-artifact.csv',
                    s3: {
                      accessKeySecret: { key: 'accessKey', name: 'accessKeyName' },
                      bucket: 'bucket',
                      endpoint: 'minio-service.kubeflow',
                      insecure: true,
                      key: 'prefix/workflow-name/workflow-name-abc/some-artifact.csv',
                      secretKeySecret: { key: 'secretKey', name: 'secretKeyName' },
                    },
                  },
                  {
                    archiveLogs: true,
                    name: 'main.log',
                    s3: {
                      accessKeySecret: { key: 'accessKey', name: 'accessKeyName' },
                      bucket: 'bucket',
                      endpoint: 'minio-service.kubeflow',
                      insecure: true,
                      key: 'prefix/workflow-name/workflow-name-abc/main.log',
                      secretKeySecret: { key: 'secretKey', name: 'secretKeyName' },
                    },
                  },
                ],
              },
            },
          },
        },
      };

      const mockedGetArgoWorkflow: jest.Mock = getArgoWorkflow as any;
      mockedGetArgoWorkflow.mockResolvedValueOnce(sampleWorkflow);

      const mockedGetK8sSecret: jest.Mock = getK8sSecret as any;
      mockedGetK8sSecret.mockResolvedValue('someSecret');

      const objStream = new PassThrough();
      const mockedClient: jest.Mock = MinioClient as any;
      const mockedClientGetObject: jest.Mock = MinioClient.prototype.getObject as any;
      mockedClientGetObject.mockResolvedValueOnce(objStream);
      objStream.end('some fake logs.');

      const stream = await getPodLogsStreamFromWorkflow('workflow-name-abc');

      expect(mockedGetArgoWorkflow).toBeCalledWith('workflow-name');

      expect(mockedGetK8sSecret).toBeCalledTimes(2);
      expect(mockedGetK8sSecret).toBeCalledWith('accessKeyName', 'accessKey');
      expect(mockedGetK8sSecret).toBeCalledWith('secretKeyName', 'secretKey');

      expect(mockedClient).toBeCalledTimes(1);
      expect(mockedClient).toBeCalledWith({
        accessKey: 'someSecret',
        endPoint: 'minio-service.kubeflow',
        port: 80,
        secretKey: 'someSecret',
        useSSL: false,
      });
      expect(mockedClientGetObject).toBeCalledTimes(1);
      expect(mockedClientGetObject).toBeCalledWith(
        'bucket',
        'prefix/workflow-name/workflow-name-abc/main.log',
      );
    });
  });
});

describe('fetch:template', () => {
  let action: TemplateAction<any>;

  const workspacePath = os.tmpdir();
  const createTemporaryDirectory: jest.MockedFunction<
    ActionContext<FetchTemplateInput>['createTemporaryDirectory']
  > = jest.fn(() =>
    Promise.resolve(
      joinPath(workspacePath, `${createTemporaryDirectory.mock.calls.length}`),
    ),
  );

  const logger = getVoidLogger();

  const mockContext = (inputPatch: Partial<FetchTemplateInput> = {}) => ({
    templateInfo: {
      baseUrl: 'base-url',
      entityRef: 'template:default/test-template',
    },
    input: {
      url: './skeleton',
      targetPath: './target',
      values: {
        test: 'value',
      },
      ...inputPatch,
    },
    output: jest.fn(),
    logStream: new PassThrough(),
    logger,
    workspacePath,
    createTemporaryDirectory,
  });

  beforeEach(() => {
    mockFs({
      ...realFiles,
    });

    action = createFetchTemplateAction({
      reader: Symbol('UrlReader') as unknown as UrlReader,
      integrations: Symbol('Integrations') as unknown as ScmIntegrations,
    });
  });

  afterEach(() => {
    mockFs.restore();
  });

  it(`returns a TemplateAction with the id 'fetch:template'`, () => {
    expect(action.id).toEqual('fetch:template');
  });

  describe('handler', () => {
    it('throws if output directory is outside the workspace', async () => {
      await expect(() =>
        action.handler(mockContext({ targetPath: '../' })),
      ).rejects.toThrowError(
        /relative path is not allowed to refer to a directory outside its parent/i,
      );
    });

    it('throws if copyWithoutRender parameter is not an array', async () => {
      await expect(() =>
        action.handler(
          mockContext({ copyWithoutRender: 'abc' as unknown as string[] }),
        ),
      ).rejects.toThrowError(/copyWithoutRender must be an array/i);
    });

    it('throws if copyWithoutRender is used with extension', async () => {
      await expect(() =>
        action.handler(
          mockContext({
            copyWithoutRender: ['abc'],
            templateFileExtension: true,
          }),
        ),
      ).rejects.toThrowError(
        /input extension incompatible with copyWithoutRender and cookiecutterCompat/,
      );
    });

    it('throws if cookiecutterCompat is used with extension', async () => {
      await expect(() =>
        action.handler(
          mockContext({
            cookiecutterCompat: true,
            templateFileExtension: true,
          }),
        ),
      ).rejects.toThrowError(
        /input extension incompatible with copyWithoutRender and cookiecutterCompat/,
      );
    });

    describe('with valid input', () => {
      let context: ActionContext<FetchTemplateInput>;

      beforeEach(async () => {
        context = mockContext({
          values: {
            name: 'test-project',
            count: 1234,
            itemList: ['first', 'second', 'third'],
            showDummyFile: false,
          },
        });

        mockFetchContents.mockImplementation(({ outputPath }) => {
          mockFs({
            ...realFiles,
            [outputPath]: {
              'an-executable.sh': mockFs.file({
                content: '#!/usr/bin/env bash',
                mode: parseInt('100755', 8),
              }),
              'empty-dir-${{ values.count }}': {},
              'static.txt': 'static content',
              '${{ values.name }}.txt': 'static content',
              subdir: {
                'templated-content.txt':
                  '${{ values.name }}: ${{ values.count }}',
              },
              '.${{ values.name }}': '${{ values.itemList | dump }}',
              'a-binary-file.png': aBinaryFile,
              '{% if values.showDummyFile %}dummy-file.txt{% else %}{% endif %}':
                'dummy file',
              '${{ "dummy-file2.txt" if values.showDummyFile else "" }}':
                'some dummy file',
            },
          });

          return Promise.resolve();
        });

        await action.handler(context);
      });

      it('uses fetchContents to retrieve the template content', () => {
        expect(mockFetchContents).toHaveBeenCalledWith(
          expect.objectContaining({
            baseUrl: context.templateInfo?.baseUrl,
            fetchUrl: context.input.url,
          }),
        );
      });

      it('skips empty filename', async () => {
        await expect(
          fs.pathExists(`${workspacePath}/target/dummy-file.txt`),
        ).resolves.toEqual(false);
      });

      it('skips empty filename syntax #2', async () => {
        await expect(
          fs.pathExists(`${workspacePath}/target/dummy-file2.txt`),
        ).resolves.toEqual(false);
      });

      it('copies files with no templating in names or content successfully', async () => {
        await expect(
          fs.readFile(`${workspacePath}/target/static.txt`, 'utf-8'),
        ).resolves.toEqual('static content');
      });

      it('copies files with templated names successfully', async () => {
        await expect(
          fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'),
        ).resolves.toEqual('static content');
      });

      it('copies files with templated content successfully', async () => {
        await expect(
          fs.readFile(
            `${workspacePath}/target/subdir/templated-content.txt`,
            'utf-8',
          ),
        ).resolves.toEqual('test-project: 1234');
      });

      it('processes dotfiles', async () => {
        await expect(
          fs.readFile(`${workspacePath}/target/.test-project`, 'utf-8'),
        ).resolves.toEqual('["first","second","third"]');
      });

      it('copies empty directories', async () => {
        await expect(
          fs.readdir(`${workspacePath}/target/empty-dir-1234`, 'utf-8'),
        ).resolves.toEqual([]);
      });

      it('copies binary files as-is without processing them', async () => {
        await expect(
          fs.readFile(`${workspacePath}/target/a-binary-file.png`),
        ).resolves.toEqual(aBinaryFile);
      });
      it('copies files and maintains the original file permissions', async () => {
        await expect(
          fs
            .stat(`${workspacePath}/target/an-executable.sh`)
            .then(fObj => fObj.mode),
        ).resolves.toEqual(parseInt('100755', 8));
      });
    });

    describe('copyWithoutRender', () => {
      let context: ActionContext<FetchTemplateInput>;

      beforeEach(async () => {
        context = mockContext({
          values: {
            name: 'test-project',
            count: 1234,
          },
          copyWithoutRender: ['.unprocessed'],
        });

        mockFetchContents.mockImplementation(({ outputPath }) => {
          mockFs({
            ...realFiles,
            [outputPath]: {
              processed: {
                'templated-content-${{ values.name }}.txt':
                  '${{ values.count }}',
              },
              '.unprocessed': {
                'templated-content-${{ values.name }}.txt':
                  '${{ values.count }}',
              },
            },
          });

          return Promise.resolve();
        });

        await action.handler(context);
      });

      it('ignores template syntax in files matched in copyWithoutRender', async () => {
        await expect(
          fs.readFile(
            `${workspacePath}/target/.unprocessed/templated-content-\${{ values.name }}.txt`,
            'utf-8',
          ),
        ).resolves.toEqual('${{ values.count }}');
      });

      it('processes files not matched in copyWithoutRender', async () => {
        await expect(
          fs.readFile(
            `${workspacePath}/target/processed/templated-content-test-project.txt`,
            'utf-8',
          ),
        ).resolves.toEqual('1234');
      });
    });
  });

  describe('cookiecutter compatibility mode', () => {
    let context: ActionContext<FetchTemplateInput>;

    beforeEach(async () => {
      context = mockContext({
        values: {
          name: 'test-project',
          count: 1234,
          itemList: ['first', 'second', 'third'],
        },
        cookiecutterCompat: true,
      });

      mockFetchContents.mockImplementation(({ outputPath }) => {
        mockFs({
          ...realFiles,
          [outputPath]: {
            '{{ cookiecutter.name }}.txt': 'static content',
            subdir: {
              'templated-content.txt':
                '{{ cookiecutter.name }}: {{ cookiecutter.count }}',
            },
            '{{ cookiecutter.name }}.json':
              '{{ cookiecutter.itemList | jsonify }}',
          },
        });

        return Promise.resolve();
      });

      await action.handler(context);
    });

    it('copies files with cookiecutter-style templated names successfully', async () => {
      await expect(
        fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'),
      ).resolves.toEqual('static content');
    });

    it('copies files with cookiecutter-style templated content successfully', async () => {
      await expect(
        fs.readFile(
          `${workspacePath}/target/subdir/templated-content.txt`,
          'utf-8',
        ),
      ).resolves.toEqual('test-project: 1234');
    });

    it('includes the jsonify filter', async () => {
      await expect(
        fs.readFile(`${workspacePath}/target/test-project.json`, 'utf-8'),
      ).resolves.toEqual('["first","second","third"]');
    });
  });

  describe('with extension=true', () => {
    let context: ActionContext<FetchTemplateInput>;

    beforeEach(async () => {
      context = mockContext({
        values: {
          name: 'test-project',
          count: 1234,
          itemList: ['first', 'second', 'third'],
        },
        templateFileExtension: true,
      });

      mockFetchContents.mockImplementation(({ outputPath }) => {
        mockFs({
          ...realFiles,
          [outputPath]: {
            'empty-dir-${{ values.count }}': {},
            'static.txt': 'static content',
            '${{ values.name }}.txt': 'static content',
            '${{ values.name }}.txt.jinja2':
              '${{ values.name }}: ${{ values.count }}',
            subdir: {
              'templated-content.txt.njk':
                '${{ values.name }}: ${{ values.count }}',
            },
            '.${{ values.name }}.njk': '${{ values.itemList | dump }}',
            'a-binary-file.png': aBinaryFile,
          },
        });

        return Promise.resolve();
      });

      await action.handler(context);
    });

    it('copies files with no templating in names or content successfully', async () => {
      await expect(
        fs.readFile(`${workspacePath}/target/static.txt`, 'utf-8'),
      ).resolves.toEqual('static content');
    });

    it('copies files with templated names successfully', async () => {
      await expect(
        fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'),
      ).resolves.toEqual('static content');
    });

    it('copies jinja2 files with templated names successfully', async () => {
      await expect(
        fs.readFile(`${workspacePath}/target/test-project.txt.jinja2`, 'utf-8'),
      ).resolves.toEqual('${{ values.name }}: ${{ values.count }}');
    });

    it('copies files with templated content successfully', async () => {
      await expect(
        fs.readFile(
          `${workspacePath}/target/subdir/templated-content.txt`,
          'utf-8',
        ),
      ).resolves.toEqual('test-project: 1234');
    });

    it('processes dotfiles', async () => {
      await expect(
        fs.readFile(`${workspacePath}/target/.test-project`, 'utf-8'),
      ).resolves.toEqual('["first","second","third"]');
    });

    it('copies empty directories', async () => {
      await expect(
        fs.readdir(`${workspacePath}/target/empty-dir-1234`, 'utf-8'),
      ).resolves.toEqual([]);
    });

    it('copies binary files as-is without processing them', async () => {
      await expect(
        fs.readFile(`${workspacePath}/target/a-binary-file.png`),
      ).resolves.toEqual(aBinaryFile);
    });
  });

  describe('with specified .jinja2 extension', () => {
    let context: ActionContext<FetchTemplateInput>;

    beforeEach(async () => {
      context = mockContext({
        templateFileExtension: '.jinja2',
        values: {
          name: 'test-project',
          count: 1234,
        },
      });

      mockFetchContents.mockImplementation(({ outputPath }) => {
        mockFs({
          ...realFiles,
          [outputPath]: {
            '${{ values.name }}.njk': '${{ values.name }}: ${{ values.count }}',
            '${{ values.name }}.txt.jinja2':
              '${{ values.name }}: ${{ values.count }}',
          },
        });

        return Promise.resolve();
      });

      await action.handler(context);
    });

    it('does not process .njk files', async () => {
      await expect(
        fs.readFile(`${workspacePath}/target/test-project.njk`, 'utf-8'),
      ).resolves.toEqual('${{ values.name }}: ${{ values.count }}');
    });

    it('does process .jinja2 files', async () => {
      await expect(
        fs.readFile(`${workspacePath}/target/test-project.txt`, 'utf-8'),
      ).resolves.toEqual('test-project: 1234');
    });
  });
});

async fetch(url?:boolean):Promise<StreamInfo>{
    try{
      let info = this.ytdlInfo;
      if(!info){
        // no cache then get
        const agent = Util.config.proxy && HttpsProxyAgent.default(Util.config.proxy);
        const requestOptions = agent ? {agent} : undefined;  
        const t = Util.time.timer.start("YouTube(AudioSource)#fetch->GetInfo");
        info = await ytdl.getInfo(this.Url, {
          lang: "ja", requestOptions
        })
        t.end();
      }
      // store related videos
      this.relatedVideos = info.related_videos.map(video => ({
        url: "https://www.youtube.com/watch?v=" + video.id,
        title: video.title,
        description: "関連動画として取得したため詳細は表示されません",
        length: video.length_seconds,
        channel: (video.author as ytdl.Author)?.name,
        channelUrl: (video.author as ytdl.Author)?.channel_url,
        thumbnail: video.thumbnails[0].url,
        isLive: video.isLive
      })).filter(v => !v.isLive);
      // update description
      this.Description = info.videoDetails.description ?? "不明";
      // update is live status
      this.LiveStream = info.videoDetails.liveBroadcastDetails && info.videoDetails.liveBroadcastDetails.isLiveNow;
      // update title
      this.Title = info.videoDetails.title;
      // format selection
      const format = ytdl.chooseFormat(info.formats, {
        filter: this.LiveStream ? null : "audioonly",
        quality: this.LiveStream ? null : "highestaudio",
        isHLS: this.LiveStream,
      } as ytdl.chooseFormatOptions);
      Util.logger.log(`[AudioSource:youtube]Format: ${format.itag}, Bitrate: ${format.bitrate}bps, Audio codec:${format.audioCodec}, Container: ${format.container}`);
      if(url){
        // return url when forced it
        this.fallback = false;
        Util.logger.log("[AudioSource:youtube]Returning the url instead of stream");
        return {
          type: "url",
          url: format.url,
          userAgent: ua
        }
      }else{
        // otherwise return readable stream
        let readable = null as Readable;
        if(info.videoDetails.liveBroadcastDetails && info.videoDetails.liveBroadcastDetails.isLiveNow){
          readable = ytdl.downloadFromInfo(info, {format, lang: "ja"});
        }else{
          readable = createChunkedYTStream(info, format, {lang: "ja"}, 1 * 1024 * 1024);
        }
        this.fallback = false;
        return {
          type: "readable",
          stream: readable,
          streamType: format.container === "webm" && format.audioCodec === "opus" ? voice.StreamType.WebmOpus : undefined
        };
      }
    }
    catch{
      this.fallback = true;
      Util.logger.log("ytdl.getInfo() failed, fallback to youtube-dl", "warn");
      const t = Util.time.timer.start("YouTube(AudioSource)#fetch->GetInfo(Fallback)");
      const info = this.youtubeDlInfo ?? JSON.parse(await getYouTubeDlInfo(this.Url)) as YoutubeDlInfo;
      t.end();
      this.Description = info.description ?? "不明";
      if(info.title) this.Title = info.title;
      if(info.is_live){
        const format = info.formats.filter(f => f.format_id === info.format_id);
        if(url){
          return {
            type: "url",
            url: format[0].url
          }
        }
        const stream = new PassThrough({highWaterMark: 1024 * 512});
        stream._destroy = () => { stream.destroyed = true; };
        const req = m3u8stream(format[0].url, {
          begin: Date.now(),
          parser: "m3u8"
        });
        req
          .on("error", (e) => stream.emit("error", e))
          .pipe(stream)
          .on('error', (e) => stream.emit("error", e));
        return {
          type: "readable",
          stream,
        };
      }else{
        const format = info.formats.filter(f => f.format_note==="tiny");
        format.sort((fa, fb) => fb.abr - fa.tbr);
        return {
          type: "url",
          url: format[0].url
        };
      }
    }
  }

describe('catalog:register', () => {
  const integrations = ScmIntegrations.fromConfig(
    new ConfigReader({
      integrations: {
        github: [{ host: 'github.com', token: 'token' }],
      },
    }),
  );

  const addLocation = jest.fn();
  const catalogClient = {
    addLocation: addLocation,
  };

  const action = createCatalogRegisterAction({
    integrations,
    catalogClient: catalogClient as unknown as CatalogApi,
  });

  const mockContext = {
    workspacePath: os.tmpdir(),
    logger: getVoidLogger(),
    logStream: new PassThrough(),
    output: jest.fn(),
    createTemporaryDirectory: jest.fn(),
  };
  beforeEach(() => {
    jest.resetAllMocks();
  });

  it('should reject registrations for locations that does not match any integration', async () => {
    await expect(
      action.handler({
        ...mockContext,
        input: {
          repoContentsUrl: 'https://google.com/foo/bar',
        },
      }),
    ).rejects.toThrow(
      /No integration found for host https:\/\/google.com\/foo\/bar/,
    );
  });

  it('should register location in catalog', async () => {
    addLocation
      .mockResolvedValueOnce({
        entities: [],
      })
      .mockResolvedValueOnce({
        entities: [
          {
            metadata: {
              namespace: 'default',
              name: 'test',
            },
            kind: 'Component',
          } as Entity,
        ],
      });
    await action.handler({
      ...mockContext,
      input: {
        catalogInfoUrl: 'http://foo/var',
      },
    });

    expect(addLocation).toHaveBeenNthCalledWith(
      1,
      {
        type: 'url',
        target: 'http://foo/var',
      },
      {},
    );
    expect(addLocation).toHaveBeenNthCalledWith(
      2,
      {
        dryRun: true,
        type: 'url',
        target: 'http://foo/var',
      },
      {},
    );

    expect(mockContext.output).toBeCalledWith(
      'entityRef',
      'component:default/test',
    );
    expect(mockContext.output).toBeCalledWith(
      'catalogInfoUrl',
      'http://foo/var',
    );
  });

  it('should return entityRef with the Component entity and not the generated location', async () => {
    addLocation
      .mockResolvedValueOnce({
        entities: [],
      })
      .mockResolvedValueOnce({
        entities: [
          {
            metadata: {
              namespace: 'default',
              name: 'generated-1238',
            },
            kind: 'Location',
          } as Entity,
          {
            metadata: {
              namespace: 'default',
              name: 'test',
            },
            kind: 'Api',
          } as Entity,
          {
            metadata: {
              namespace: 'default',
              name: 'test',
            },
            kind: 'Component',
          } as Entity,
          {
            metadata: {
              namespace: 'default',
              name: 'test',
            },
            kind: 'Template',
          } as Entity,
        ],
      });
    await action.handler({
      ...mockContext,
      input: {
        catalogInfoUrl: 'http://foo/var',
      },
    });
    expect(mockContext.output).toBeCalledWith(
      'entityRef',
      'component:default/test',
    );
  });

  it('should return entityRef with the next non-generated entity if no Component kind can be found', async () => {
    addLocation
      .mockResolvedValueOnce({
        entities: [],
      })
      .mockResolvedValueOnce({
        entities: [
          {
            metadata: {
              namespace: 'default',
              name: 'generated-1238',
            },
            kind: 'Location',
          } as Entity,
          {
            metadata: {
              namespace: 'default',
              name: 'test',
            },
            kind: 'Api', // should return this one
          } as Entity,
          {
            metadata: {
              namespace: 'default',
              name: 'test',
            },
            kind: 'Template',
          } as Entity,
        ],
      });
    await action.handler({
      ...mockContext,
      input: {
        catalogInfoUrl: 'http://foo/var',
      },
    });
    expect(mockContext.output).toBeCalledWith('entityRef', 'api:default/test');
  });

  it('should return entityRef with the first entity if no non-generated entities can be found', async () => {
    addLocation
      .mockResolvedValueOnce({
        entities: [],
      })
      .mockResolvedValueOnce({
        entities: [
          {
            metadata: {
              namespace: 'default',
              name: 'generated-1238',
            },
            kind: 'Location',
          } as Entity,
          {
            metadata: {
              namespace: 'default',
              name: 'generated-1238',
            },
            kind: 'Template',
          } as Entity,
        ],
      });
    await action.handler({
      ...mockContext,
      input: {
        catalogInfoUrl: 'http://foo/var',
      },
    });
    expect(mockContext.output).toBeCalledWith(
      'entityRef',
      'location:default/generated-1238',
    );
  });

  it('should not return entityRef if there are no entites', async () => {
    addLocation
      .mockResolvedValueOnce({
        entities: [],
      })
      .mockResolvedValueOnce({
        entities: [],
      });
    await action.handler({
      ...mockContext,
      input: {
        catalogInfoUrl: 'http://foo/var',
      },
    });
    expect(mockContext.output).not.toBeCalledWith(
      'entityRef',
      expect.any(String),
    );
  });

  it('should ignore failures when dry running the location in the catalog if `optional` is set', async () => {
    addLocation
      .mockResolvedValueOnce({
        entities: [],
      })
      .mockRejectedValueOnce(new Error('Not found'));
    await action.handler({
      ...mockContext,
      input: {
        catalogInfoUrl: 'http://foo/var',
        optional: true,
      },
    });

    expect(addLocation).toHaveBeenNthCalledWith(
      1,
      {
        type: 'url',
        target: 'http://foo/var',
      },
      {},
    );
    expect(addLocation).toHaveBeenNthCalledWith(
      2,
      {
        dryRun: true,
        type: 'url',
        target: 'http://foo/var',
      },
      {},
    );

    expect(mockContext.output).toBeCalledWith(
      'catalogInfoUrl',
      'http://foo/var',
    );
  });
});

async findAllToFile(fileOptions: FileOptions, queryOptions?: QueryOptions, options?: Options<any>): Promise<Result<File>> {
        const storage = new Mapper();

        if (queryOptions && queryOptions.groupBy) {
            this._rawQuery.push(`GROUP BY ${queryOptions.groupBy}`);
        }

        if (queryOptions && queryOptions.sortBy) {
            if (queryOptions.sortDesc) {
                this._rawQuery.push(`ORDER BY "${queryOptions.sortBy}" DESC`);
            } else {
                this._rawQuery.push(`ORDER BY "${queryOptions.sortBy}" ASC`);
            }
        }

        if (queryOptions && queryOptions.offset) {
            this._values.push(queryOptions.offset);
            this._rawQuery.push(`OFFSET $${this._values.length}`);
        }

        if (queryOptions && queryOptions.limit) {
            this._values.push(queryOptions.limit);
            this._rawQuery.push(`LIMIT $${this._values.length}`);
        }

        const query = {
            text: this._rawQuery.join(' '),
            values: this._values,
        };

        // reset the filter
        this._rawQuery = [`SELECT * FROM ${this._tableName}`];
        this._values = [];

        try {
            // blob storage will pick the proper provider based on environment variable, no need to specify here unless
            // fileOptions contains a provider
            const fileMapper = new FileMapper();
            let contentType = '';
            const blob = BlobStorageProvider(fileOptions.blob_provider);
            let stream: Readable = await storage.rowsStreaming(query, options);

            if (fileOptions.transformStreams && fileOptions.transformStreams.length > 0) {
                for (const pipe of fileOptions.transformStreams) {
                    stream = stream.pipe(pipe);
                }
            }

            // pass through stream needed in order to pipe the results through transformative streams prior to piping
            // them to the blob storage
            const pass = new PassThrough();
            const fileName = fileOptions.file_name ? fileOptions.file_name : `${short.generate()}-${new Date().toDateString()}`;

            switch (fileOptions.file_type) {
                case 'json': {
                    stream.pipe(JSONStream.stringify()).pipe(pass);
                    contentType = 'application/json';
                    break;
                }

                case 'csv': {
                    stream.pipe(csvStringify({header: true})).pipe(pass);
                    contentType = 'text/csv';
                    break;
                }

                default: {
                    return Promise.resolve(Result.Failure('no file type specified, aborting'));
                }
            }

            if (blob) {
                const result = await blob?.uploadPipe(`containers/${fileOptions.containerID}/queryResults`, fileName, pass, contentType, 'utf8');

                const file = new File({
                    file_name: fileName,
                    file_size: result.value.size,
                    md5hash: result.value.md5hash,
                    adapter_file_path: result.value.filepath,
                    adapter: blob.name(),
                    metadata: result.value.metadata,
                    container_id: fileOptions.containerID,
                });

                return fileMapper.Create(fileOptions.userID ? fileOptions.userID : 'system', file);
            } else {
                return Promise.resolve(Result.Failure('unable to initialize blob storage client'));
            }
        } catch (e: any) {
            return Promise.resolve(Result.Error(e));
        }
    }

export async function loader({
  request,
  params,
}: {
  request: Request
  params: { path: string }
}) {
  if (request.method !== "GET") {
    return new Response("405", { status: 405 })
  }

  const match = route.exec(params.path)
  if (!match) {
    return new Response("404", { status: 404 })
  }
  const [, tweetId, mode] = match
  const requestUrl = new URL(request.url)
  const lang = requestUrl.searchParams.get("lang") || "ja"
  if (!lang.match(/^[a-z-]{2,5}$/)) {
    return new Response("400 lang", { status: 400 })
  }
  const tz = timezones.find(
    (t) => t.offset === parseInt(requestUrl.searchParams.get("tz") || "9")
  )
  if (!tz) {
    return new Response("400 tz", { status: 400 })
  }
  const theme = requestUrl.searchParams.get("theme") || "light"
  if (!theme.match(/^[a-z]+$/)) {
    return new Response("400 theme", { status: 400 })
  }
  const scale = parseFloat(requestUrl.searchParams.get("scale") || "2")
  if (Number.isNaN(scale) || scale < 1 || 5 < scale) {
    return new Response("400 scale", { status: 400 })
  }

  const headers: Record<string, string> = {
    "access-control-allow-origin": "*",
  }
  if (scale !== 1) {
    headers["content-dpr"] = scale.toFixed(1)
  }
  const hideCard =
    parseInt(requestUrl.searchParams.get("hideCard") || "0") === 1 ||
    requestUrl.searchParams.get("hideCard") === "true"
  const hideThread =
    parseInt(requestUrl.searchParams.get("hideThread") || "0") === 1 ||
    requestUrl.searchParams.get("hideThread") === "true"

  const r = await axios.get<{
    id_str?: string
    user?: { screen_name?: string }
  }>(`https://cdn.syndication.twimg.com/tweet?id=${tweetId}`, {
    validateStatus: () => true,
  })
  if (![301, 200].includes(r.status)) {
    return new Response("remote is" + r.status, { status: 400 })
  }
  const { user, id_str } = r.data
  if (user?.screen_name && id_str) {
    headers[
      "link"
    ] = `<https://twitter.com/${user.screen_name}/status/${id_str}>; rel="canonical"`
  }

  // CJK統合漢字 CJK Unified Ideographs
  // 日本語が描画可能に
  await chromium.font(
    "https://cdn.jsdelivr.net/gh/googlefonts/noto-cjk@165c01b46ea533872e002e0785ff17e44f6d97d8/Sans/OTF/Japanese/NotoSansCJKjp-Regular.otf"
  )
  // 数学用英数字記号 Mathematical Alphanumeric Symbols
  // ????ℳℴ???? などが描画可能に
  await chromium.font(
    "https://cdn.jsdelivr.net/gh/googlefonts/noto-fonts@736e6b8f886cae4664e78edb0880fbb5af7d50b7/hinted/ttf/NotoSansMath/NotoSansMath-Regular.ttf"
  )
  // 基本ラテン文字 Basic Latin
  // 下付き文字などが描画可能に
  await chromium.font(
    "https://cdn.jsdelivr.net/gh/googlefonts/noto-fonts@7697007fcb3563290d73f41f56a70d5d559d828c/hinted/ttf/NotoSans/NotoSans-Regular.ttf"
  )
  const mime = `image/${mode.replace("jpg", "jpeg")}`

  if (imageCacheUrl && imageCacheUA) {
    const ua = request.headers.get("user-agent")
    if (!ua || !ua.includes(imageCacheUA)) {
      const qs = querystring.stringify(
        Object.fromEntries(
          Object.entries({
            hideCard,
            hideThread,
            scale,
            lang,
            theme,
            tz: tz.offset,
          }).filter(([k]) => requestUrl.searchParams.has(k))
        )
      )
      const cacheUrl = url.resolve(
        imageCacheUrl,
        `${tweetId}.${mode}${0 < qs.length ? `?${qs}` : ""}`
      )
      try {
        const s = new PassThrough()
        const r = nodeRequest(cacheUrl)
        r.pipe(s, { end: true })
        let isReceived = false
        await new Promise<void>((res) => {
          r.on("data", () => {
            isReceived = true
            res()
          })
          setTimeout(() => res(), 500)
        })
        if (isReceived) {
          if (r.response?.statusCode === 200) {
            return new Response(s as any, {
              headers: {
                ...headers,
                "content-type": mime,
                "cache-control":
                  "public, max-age=31536000, stale-while-revalidate",
              },
            })
          } else {
            console.warn(`cache status_code: ${r.response?.statusCode}`)
          }
        }
      } catch (e) {
        console.error(e)
      }
    }
  }

  const tzString = tz?.utc.pop()

  const browser = await chromium.puppeteer.launch({
    args: chromium.args,
    defaultViewport: {
      ...chromium.defaultViewport,
      deviceScaleFactor: scale,
      width: 1280,
      height: 1280,
    },
    executablePath: await chromium.executablePath,
    headless: chromium.headless,
    env: {
      ...process.env,
      TZ: tzString ? tzString : "Asia/Tokyo",
    },
  })

  try {
    const pages = await browser.pages()
    const page = 0 < pages.length ? pages[0] : await browser.newPage()
    await page.evaluateOnNewDocument(() => {
      const timeout = setTimeout
      // @ts-ignore
      window.setTimeout = function (fn: Function, ms?: number) {
        const s = fn.toString()
        if (s.includes("timeout") || s.includes("native code")) return
        timeout(() => fn(), 0)
      }
    })
    const params = {
      widgetsVersion: "9066bb2:1593540614199",
      origin: "file:///Users/ci7lus/tweet2image.html",
      embedId: "twitter-widget-0",
      hideCard: hideCard.toString(),
      hideThread: hideThread.toString(),
      lang,
      theme,
      id: tweetId,
    }
    await Promise.all([
      page.goto(
        `https://platform.twitter.com/embed/index.html?${querystring.stringify(
          params
        )}`
      ),
      page.waitForNavigation({ waitUntil: ["networkidle0"] }),
    ])
    const rect = await page.evaluate(() => {
      const { x, y, width, height } = document
        .querySelector("article")
        ?.getBoundingClientRect()!
      return {
        x: x - 1,
        y: y - 1,
        width: Math.round(width + 2),
        height: Math.round(height + 2),
      }
    })
    await page.setViewport({
      ...chromium.defaultViewport,
      deviceScaleFactor: scale,
      height: rect.height,
      width: rect.width,
    })
    const buffer = await page.screenshot({
      clip: rect,
      type: mode === "png" ? "png" : "jpeg",
      omitBackground: mode === "png",
    })
    return new Response(buffer, {
      headers: {
        ...headers,
        "content-type": mime,
        "cache-control": "max-age=600, public, stale-while-revalidate",
      },
    })
  } catch (error) {
    console.error(error)
    return new Response("500", { status: 500 })
  } finally {
    await browser.close()
  }
}

/**
 * Creates a techdocs router.
 *
 * @public
 */
export async function createRouter(
  options: RouterOptions,
): Promise<express.Router> {
  const router = Router();
  const { publisher, config, logger, discovery } = options;
  const catalogClient = new CatalogClient({ discoveryApi: discovery });
  const docsBuildStrategy =
    options.docsBuildStrategy ?? DefaultDocsBuildStrategy.fromConfig(config);
  const buildLogTransport =
    options.buildLogTransport ??
    new winston.transports.Stream({ stream: new PassThrough() });

  // Entities are cached to optimize the /static/docs request path, which can be called many times
  // when loading a single techdocs page.
  const entityLoader = new CachedEntityLoader({
    catalog: catalogClient,
    cache: options.cache.getClient(),
  });

  // Set up a cache client if configured.
  let cache: TechDocsCache | undefined;
  const defaultTtl = config.getOptionalNumber('techdocs.cache.ttl');
  if (defaultTtl) {
    const cacheClient = options.cache.getClient({ defaultTtl });
    cache = TechDocsCache.fromConfig(config, { cache: cacheClient, logger });
  }

  const scmIntegrations = ScmIntegrations.fromConfig(config);
  const docsSynchronizer = new DocsSynchronizer({
    publisher,
    logger,
    buildLogTransport,
    config,
    scmIntegrations,
    cache,
  });

  router.get('/metadata/techdocs/:namespace/:kind/:name', async (req, res) => {
    const { kind, namespace, name } = req.params;
    const entityName = { kind, namespace, name };
    const token = getBearerToken(req.headers.authorization);

    // Verify that the related entity exists and the current user has permission to view it.
    const entity = await entityLoader.load(entityName, token);

    if (!entity) {
      throw new NotFoundError(
        `Unable to get metadata for '${stringifyEntityRef(entityName)}'`,
      );
    }

    try {
      const techdocsMetadata = await publisher.fetchTechDocsMetadata(
        entityName,
      );

      res.json(techdocsMetadata);
    } catch (err) {
      logger.info(
        `Unable to get metadata for '${stringifyEntityRef(
          entityName,
        )}' with error ${err}`,
      );
      throw new NotFoundError(
        `Unable to get metadata for '${stringifyEntityRef(entityName)}'`,
        err,
      );
    }
  });

  router.get('/metadata/entity/:namespace/:kind/:name', async (req, res) => {
    const { kind, namespace, name } = req.params;
    const entityName = { kind, namespace, name };
    const token = getBearerToken(req.headers.authorization);

    const entity = await entityLoader.load(entityName, token);

    if (!entity) {
      throw new NotFoundError(
        `Unable to get metadata for '${stringifyEntityRef(entityName)}'`,
      );
    }

    try {
      const locationMetadata = getLocationForEntity(entity, scmIntegrations);
      res.json({ ...entity, locationMetadata });
    } catch (err) {
      logger.info(
        `Unable to get metadata for '${stringifyEntityRef(
          entityName,
        )}' with error ${err}`,
      );
      throw new NotFoundError(
        `Unable to get metadata for '${stringifyEntityRef(entityName)}'`,
        err,
      );
    }
  });

  // Check if docs are the latest version and trigger rebuilds if not
  // Responds with an event-stream that closes after the build finished
  // Responds with an immediate success if rebuild not needed
  // If a build is required, responds with a success when finished
  router.get('/sync/:namespace/:kind/:name', async (req, res) => {
    const { kind, namespace, name } = req.params;
    const token = getBearerToken(req.headers.authorization);

    const entity = await entityLoader.load({ kind, namespace, name }, token);

    if (!entity?.metadata?.uid) {
      throw new NotFoundError('Entity metadata UID missing');
    }

    const responseHandler: DocsSynchronizerSyncOpts = createEventStream(res);

    // By default, techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to
    // 'local'. If set to 'external', it will assume that an external process (e.g. CI/CD pipeline
    // of the repository) is responsible for building and publishing documentation to the storage provider.
    // Altering the implementation of the injected docsBuildStrategy allows for more complex behaviours, based on
    // either config or the properties of the entity (e.g. annotations, labels, spec fields etc.).
    const shouldBuild = await docsBuildStrategy.shouldBuild({ entity });
    if (!shouldBuild) {
      // However, if caching is enabled, take the opportunity to check and
      // invalidate stale cache entries.
      if (cache) {
        await docsSynchronizer.doCacheSync({
          responseHandler,
          discovery,
          token,
          entity,
        });
        return;
      }
      responseHandler.finish({ updated: false });
      return;
    }

    // Set the synchronization and build process if "out-of-the-box" configuration is provided.
    if (isOutOfTheBoxOption(options)) {
      const { preparers, generators } = options;

      await docsSynchronizer.doSync({
        responseHandler,
        entity,
        preparers,
        generators,
      });
      return;
    }

    responseHandler.error(
      new Error(
        "Invalid configuration. docsBuildStrategy.shouldBuild returned 'true', but no 'preparer' was provided to the router initialization.",
      ),
    );
  });

  // Ensures that the related entity exists and the current user has permission to view it.
  if (config.getOptionalBoolean('permission.enabled')) {
    router.use(
      '/static/docs/:namespace/:kind/:name',
      async (req, _res, next) => {
        const { kind, namespace, name } = req.params;
        const entityName = { kind, namespace, name };
        const token = getBearerToken(req.headers.authorization);

        const entity = await entityLoader.load(entityName, token);

        if (!entity) {
          throw new NotFoundError(
            `Entity not found for ${stringifyEntityRef(entityName)}`,
          );
        }

        next();
      },
    );
  }

  // If a cache manager was provided, attach the cache middleware.
  if (cache) {
    router.use(createCacheMiddleware({ logger, cache }));
  }

  // Route middleware which serves files from the storage set in the publisher.
  router.use('/static/docs', publisher.docsRouter());

  return router;
}