aws-lambda#KinesisStreamEvent TypeScript Examples

kinesisStreamHandler = async (event: KinesisStreamEvent, context: Context): Promise<void> => {
  return myContainer.get<KinesisStreamService>(TYPES.KinesisStreamService).handle(event, context)
    .catch((error) => {
      console.error(error);
    })
    .then(() => {
      // Return void promise
      return Promise.resolve();
    });
}

/**
   * Handle received data by wrapping in a mock KinesisStreamRecord and forwarding on
   * @param data The data to wrap
   * @returns Promise with the result
   */
  async handleData(data: OsDocumentData, print: boolean): Promise<OpenSearchBulkResult> {
    const event: KinesisStreamEvent = {
      Records: [this.wrapDataIntoRecord(data)],
    };
    // unused so any value will work
    const context: Context = {} as Context;
    const kss = await this.kinesisStreamService.handle(event, context);
    if (print) {
      this.logger.log(JSON.stringify(kss, null, '  '));
    }
    return kss;
  }

/* eslint-disable @typescript-eslint/no-unused-vars */
  /**
   * Handle the Kinesis event by transforming and then bulk uploading to OpenSearch
   * @param event The event containing the data to transform
   * @param context The lambda context
   * @returns A promise to wait on
   */
  public async handle(event: KinesisStreamEvent, context: Context): Promise<OpenSearchBulkResult> {
    this.logger.log(`Transforming ${event.Records.length} kinesis records to ES documents`);
    const docs = this.ecsTransformService.transform(event);
    this.logger.log(`Submitting ${docs.length} documents to ES`);
    return this.openSearch.bulk(docs).then((value) => {
      this.logger.log(`${docs.length - value.errors.length} documents added`);
      this.logger.log(`${value.errors.length} documents failed`);
      return value;
    });
  }

handler = async (event: KinesisStreamEvent, context: Context): Promise<void> => {
  const sqs = new AWS.SQS();

  if (!_applicationContainer) {
    _applicationContainer = ApplicationContainer.instance();
  }

  if (!_queueEndpointCache) {
    _queueEndpointCache = await buildEndpointCache(sqs);
  }

  _applicationContainer.logger.debug('Event Source Broker: %o', event);

  const batches = new Map<string, AWS.SQS.Types.SendMessageBatchRequest>();
  const messageGroupId = uuid();

  // fan out Kinesis stream Event by event name convention to a SQS FIFO event handler
  event.Records.forEach(record => {
    const currentEvent: Event = JSON.parse(Buffer.from(record.kinesis.data, 'base64').toString('utf8'));
    const endpoint = _queueEndpointCache.get(getQueueNameFromEvent(currentEvent));
    const id = uuid();

    if (!batches.get(endpoint)) {
      batches.set(endpoint, { Entries: [], QueueUrl: endpoint });
    }

    batches.get(endpoint).Entries.push({ Id: id, MessageBody: JSON.stringify(currentEvent), MessageDeduplicationId: id, MessageGroupId: messageGroupId });
  });

  for (const batch of Array.from(batches.values())) {
    await sqs.sendMessageBatch(batch).promise();
    _applicationContainer.logger.debug('Sent SQS Batch: %o', batch);
  }
}

/**
   * Transform
   * @param event
   * @returns
   */
  public transform(event: KinesisStreamEvent): OsDocument[] {
    if (event.Records) {
      let badDocs = 0;
      this.logger.log(`Received ${event.Records.length} records`);

      const docs = event.Records
        .map((record) => {
          try {
            return this.ksrMapper.toOpensearchDocument(record);
          } catch (e: unknown) {
            badDocs++;
          }
        })
        .filter((document): document is OsDocument => document !== undefined)
        .map((document) => {
          try {
            return this.parseDocumentData(document);
          } catch (error: unknown) {
            const parser: string = error instanceof ParserError ? error.parser : 'unknown';
            const message: string = error instanceof ParserError || error instanceof GenericError ?
              error.message : '';
            const team: string = document.data.organization?.id ? document.data.organization.id : 'unknown';
            const hostName: string = document.data.host?.hostname ? document.data.host?.hostname : '';
            const serviceName: string = document.data.service?.name ? document.data.service?.name : '';
            const sequence: string = document.data.event?.sequence ? document.data.event?.sequence : '';
            const path: string = document.data.log?.file?.path ? document.data.log?.file?.path : '';
            // eslint-disable-next-line max-len
            this.logger.log(`PARSE_ERROR:${parser} ${team} ${hostName} ${serviceName} ${path}:${sequence} ${document.fingerprint.name} : ${message}`);
            badDocs++;
            return undefined;
          }
        })
        .filter((document): document is OsDocument => document !== undefined);

      if (badDocs > 0) {
        this.logger.log(`Rejected ${badDocs} records`);
      }
      return docs;
    }
    return [];
  }

describe('KinesisStreamService', () => {
  it('transforms and then sends data', async () => {
    const docs = ['docs', 'docs', 'docs!'];
    const etService = {
      transform: jest.fn().mockReturnValue(docs),
    } as unknown as EcsTransformService;
    const osService = {
      bulk: jest.fn().mockReturnValue({
        then: jest.fn().mockImplementation((cb) => {
          cb({
            errors: ['err'],
          });
        }),
      }),
    } as unknown as OpenSearchService;
    const logger = {
      log: jest.fn(),
      debug: jest.fn(),
    } as LoggerService;

    const ks = new KinesisStreamService(
      etService,
      osService,
      logger,
    );
    const fakeEvent = {Records: []} as KinesisStreamEvent;
    const fakeContext = {} as Context;

    await ks.handle(fakeEvent, fakeContext);

    expect(etService.transform).toBeCalledTimes(1);
    expect(etService.transform).toBeCalledWith(fakeEvent);

    expect(osService.bulk).toBeCalledTimes(1);
    expect(osService.bulk).toBeCalledWith(docs);
    expect(logger.log).toBeCalledTimes(4);
    expect(logger.log).toBeCalledWith('Transforming 0 kinesis records to ES documents');
    expect(logger.log).toBeCalledWith('Submitting 3 documents to ES');
    expect(logger.log).toBeCalledWith('2 documents added');
    expect(logger.log).toBeCalledWith('1 documents failed');
  });
});