@grafana/data#Labels TypeScript Examples

row: LogRowModel = {
  entryFieldIndex: 0,
  rowIndex: 0,
  dataFrame: new MutableDataFrame(),
  entry: '4',
  labels: (null as any) as Labels,
  hasAnsi: false,
  raw: '4',
  logLevel: LogLevel.info,
  timeEpochMs: 4,
  timeFromNow: '',
  timeLocal: '',
  timeUtc: '',
  uid: '1',
}

/**
 * Create target (query to run). Url is what is used as cache key.
 */
function makeTarget(url: string, labels?: Labels) {
  labels = labels || { job: 'varlogs' };
  return {
    url,
    size: 10,
    query: formatLabels(labels),
    refId: 'A',
    regexp: '',
  };
}

/**
 * Transforms LokiLogStream structure into a dataFrame. Used when doing standard queries and older version of Loki.
 */
export function legacyLogStreamToDataFrame(
  stream: LokiLegacyStreamResult,
  reverse?: boolean,
  refId?: string
): DataFrame {
  let labels: Labels = stream.parsedLabels;
  if (!labels && stream.labels) {
    labels = parseLabels(stream.labels);
  }

  const times = new ArrayVector<string>([]);
  const timesNs = new ArrayVector<string>([]);
  const lines = new ArrayVector<string>([]);
  const uids = new ArrayVector<string>([]);

  for (const entry of stream.entries) {
    const ts = entry.ts || entry.timestamp;
    // iso string with nano precision, will be truncated but is parse-able
    times.add(ts);
    // So this matches new format, we are loosing precision here, which sucks but no easy way to keep it and this
    // is for old pre 1.0.0 version Loki so probably does not affect that much.
    timesNs.add(dateTime(ts).valueOf() + '000000');
    lines.add(entry.line);
    uids.add(createUid(ts, stream.labels, entry.line));
  }

  return constructDataFrame(times, timesNs, lines, uids, labels, reverse, refId);
}

/**
 * Transforms LokiStreamResult structure into a dataFrame. Used when doing standard queries and newer version of Loki.
 */
export function lokiStreamResultToDataFrame(stream: LokiStreamResult, reverse?: boolean, refId?: string): DataFrame {
  const labels: Labels = stream.stream;
  const labelsString = Object.entries(labels)
    .map(([key, val]) => `${key}="${val}"`)
    .sort()
    .join('');

  const times = new ArrayVector<string>([]);
  const timesNs = new ArrayVector<string>([]);
  const lines = new ArrayVector<string>([]);
  const uids = new ArrayVector<string>([]);

  for (const [ts, line] of stream.values) {
    // num ns epoch in string, we convert it to iso string here so it matches old format
    times.add(new Date(parseInt(ts.substr(0, ts.length - 6), 10)).toISOString());
    timesNs.add(ts);
    lines.add(line);
    uids.add(createUid(ts, labelsString, line));
  }

  return constructDataFrame(times, timesNs, lines, uids, labels, reverse, refId);
}

/**
 * Constructs dataFrame with supplied fields and other data. Also makes sure it is properly reversed if needed.
 */
function constructDataFrame(
  times: ArrayVector<string>,
  timesNs: ArrayVector<string>,
  lines: ArrayVector<string>,
  uids: ArrayVector<string>,
  labels: Labels,
  reverse?: boolean,
  refId?: string
) {
  const dataFrame = {
    refId,
    fields: [
      { name: 'ts', type: FieldType.time, config: { title: 'Time' }, values: times }, // Time
      { name: 'line', type: FieldType.string, config: {}, values: lines, labels }, // Line
      { name: 'id', type: FieldType.string, config: {}, values: uids },
      { name: 'tsNs', type: FieldType.time, config: { title: 'Time ns' }, values: timesNs }, // Time
    ],
    length: times.length,
  };

  if (reverse) {
    const mutableDataFrame = new MutableDataFrame(dataFrame);
    mutableDataFrame.reverse();
    return mutableDataFrame;
  }

  return dataFrame;
}

export function appendResponseToBufferedData(response: LokiTailResponse, data: MutableDataFrame) {
  // Should we do anything with: response.dropped_entries?

  const streams: LokiStreamResult[] = response.streams;
  if (!streams || !streams.length) {
    return;
  }

  let baseLabels: Labels = {};
  for (const f of data.fields) {
    if (f.type === FieldType.string) {
      if (f.labels) {
        baseLabels = f.labels;
      }
      break;
    }
  }

  for (const stream of streams) {
    // Find unique labels
    const unique = findUniqueLabels(stream.stream, baseLabels);
    const allLabelsString = Object.entries(stream.stream)
      .map(([key, val]) => `${key}="${val}"`)
      .sort()
      .join('');

    // Add each line
    for (const [ts, line] of stream.values) {
      data.values.ts.add(new Date(parseInt(ts.substr(0, ts.length - 6), 10)).toISOString());
      data.values.tsNs.add(ts);
      data.values.line.add(line);
      data.values.labels.add(unique);
      data.values.id.add(createUid(ts, allLabelsString, line));
    }
  }
}

/**
 * Transform LokiResponse data and appends it to MutableDataFrame. Used for streaming where the dataFrame can be
 * a CircularDataFrame creating a fixed size rolling buffer.
 * TODO: Probably could be unified with the logStreamToDataFrame function.
 * @param response
 * @param data Needs to have ts, line, labels, id as fields
 */
export function appendLegacyResponseToBufferedData(response: LokiLegacyStreamResponse, data: MutableDataFrame) {
  // Should we do anything with: response.dropped_entries?

  const streams: LokiLegacyStreamResult[] = response.streams;
  if (!streams || !streams.length) {
    return;
  }

  let baseLabels: Labels = {};
  for (const f of data.fields) {
    if (f.type === FieldType.string) {
      if (f.labels) {
        baseLabels = f.labels;
      }
      break;
    }
  }

  for (const stream of streams) {
    // Find unique labels
    const labels = parseLabels(stream.labels);
    const unique = findUniqueLabels(labels, baseLabels);

    // Add each line
    for (const entry of stream.entries) {
      const ts = entry.ts || entry.timestamp;
      data.values.ts.add(ts);
      data.values.line.add(entry.line);
      data.values.labels.add(unique);
      data.values.id.add(createUid(ts, stream.labels, entry.line));
    }
  }
}

/**
 * Converts dataFrames into LogsModel. This involves merging them into one list, sorting them and computing metadata
 * like common labels.
 */
export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefined {
  if (logSeries.length === 0) {
    return undefined;
  }
  const allLabels: Labels[] = [];

  // Find the fields we care about and collect all labels
  const allSeries: LogFields[] = logSeries.map(series => {
    const fieldCache = new FieldCache(series);

    // Assume the first string field in the dataFrame is the message. This was right so far but probably needs some
    // more explicit checks.
    const stringField = fieldCache.getFirstFieldOfType(FieldType.string);
    if (stringField.labels) {
      allLabels.push(stringField.labels);
    }
    return {
      series,
      timeField: fieldCache.getFirstFieldOfType(FieldType.time),
      stringField,
      logLevelField: fieldCache.getFieldByName('level'),
      idField: getIdField(fieldCache),
    };
  });

  const commonLabels = allLabels.length > 0 ? findCommonLabels(allLabels) : {};

  const rows: LogRowModel[] = [];
  let hasUniqueLabels = false;

  for (const info of allSeries) {
    const { timeField, stringField, logLevelField, idField, series } = info;
    const labels = stringField.labels;
    const uniqueLabels = findUniqueLabels(labels, commonLabels);
    if (Object.keys(uniqueLabels).length > 0) {
      hasUniqueLabels = true;
    }

    let seriesLogLevel: LogLevel | undefined = undefined;
    if (labels && Object.keys(labels).indexOf('level') !== -1) {
      seriesLogLevel = getLogLevelFromKey(labels['level']);
    }

    for (let j = 0; j < series.length; j++) {
      const ts = timeField.values.get(j);
      const time = dateTime(ts);

      const messageValue: unknown = stringField.values.get(j);
      // This should be string but sometimes isn't (eg elastic) because the dataFrame is not strongly typed.
      const message: string = typeof messageValue === 'string' ? messageValue : JSON.stringify(messageValue);

      const hasAnsi = hasAnsiCodes(message);
      const searchWords = series.meta && series.meta.searchWords ? series.meta.searchWords : [];

      let logLevel = LogLevel.unknown;
      if (logLevelField && logLevelField.values.get(j)) {
        logLevel = getLogLevelFromKey(logLevelField.values.get(j));
      } else if (seriesLogLevel) {
        logLevel = seriesLogLevel;
      } else {
        logLevel = getLogLevel(message);
      }

      rows.push({
        entryFieldIndex: stringField.index,
        rowIndex: j,
        dataFrame: series,
        logLevel,
        timeFromNow: time.fromNow(),
        timeEpochMs: time.valueOf(),
        timeLocal: time.format(logTimeFormat),
        timeUtc: toUtc(time.valueOf()).format(logTimeFormat),
        uniqueLabels,
        hasAnsi,
        searchWords,
        entry: hasAnsi ? ansicolor.strip(message) : message,
        raw: message,
        labels: stringField.labels,
        uid: idField ? idField.values.get(j) : j.toString(),
      });
    }
  }

  const deduplicatedLogRows = deduplicateLogRowsById(rows);

  // Meta data to display in status
  const meta: LogsMetaItem[] = [];
  if (_.size(commonLabels) > 0) {
    meta.push({
      label: 'Common labels',
      value: commonLabels,
      kind: LogsMetaKind.LabelsMap,
    });
  }

  const limits = logSeries.filter(series => series.meta && series.meta.limit);
  const limitValue = Object.values(
    limits.reduce((acc: any, elem: any) => {
      acc[elem.refId] = elem.meta.limit;
      return acc;
    }, {})
  ).reduce((acc: number, elem: any) => (acc += elem), 0);

  if (limits.length > 0) {
    meta.push({
      label: 'Limit',
      value: `${limitValue} (${deduplicatedLogRows.length} returned)`,
      kind: LogsMetaKind.String,
    });
  }

  return {
    hasUniqueLabels,
    meta,
    rows: deduplicatedLogRows,
  };
}

describe('Live Stream Tests', () => {
  afterAll(() => {
    jest.restoreAllMocks();
  });

  const msg0: any = {
    streams: [
      {
        labels: '{filename="/var/log/sntpc.log", job="varlogs"}',
        entries: [
          {
            ts: '2019-08-28T20:50:40.118944705Z',
            line: 'Kittens',
          },
        ],
      },
    ],
    dropped_entries: null,
  };

  it('reads the values into the buffer', done => {
    fakeSocket = new Subject<any>();
    const labels: Labels = { job: 'varlogs' };
    const target = makeTarget('fake', labels);
    const stream = new LiveStreams().getLegacyStream(target);
    expect.assertions(4);

    const tests = [
      (val: DataFrame[]) => {
        expect(val[0].length).toEqual(7);
        expect(val[0].fields[1].labels).toEqual(labels);
      },
      (val: DataFrame[]) => {
        expect(val[0].length).toEqual(8);
        const view = new DataFrameView(val[0]);
        const last = { ...view.get(view.length - 1) };
        expect(last).toEqual({
          ts: '2019-08-28T20:50:40.118944705Z',
          id: '81d963f31c276ad2ea1af38b38436237',
          line: 'Kittens',
          labels: { filename: '/var/log/sntpc.log' },
        });
      },
    ];
    stream.subscribe({
      next: val => {
        const test = tests.shift();
        test(val);
      },
      complete: () => done(),
    });

    // Send it the initial list of things
    fakeSocket.next(initialRawResponse);
    // Send it a single update
    fakeSocket.next(msg0);
    fakeSocket.complete();
  });

  it('returns the same subscription if the url matches existing one', () => {
    fakeSocket = new Subject<any>();
    const liveStreams = new LiveStreams();
    const stream1 = liveStreams.getLegacyStream(makeTarget('url_to_match'));
    const stream2 = liveStreams.getLegacyStream(makeTarget('url_to_match'));
    expect(stream1).toBe(stream2);
  });

  it('returns new subscription when the previous unsubscribed', () => {
    fakeSocket = new Subject<any>();
    const liveStreams = new LiveStreams();
    const stream1 = liveStreams.getLegacyStream(makeTarget('url_to_match'));
    const subscription = stream1.subscribe({
      next: noop,
    });
    subscription.unsubscribe();

    const stream2 = liveStreams.getLegacyStream(makeTarget('url_to_match'));
    expect(stream1).not.toBe(stream2);
  });

  it('returns new subscription when the previous is unsubscribed and correctly unsubscribes from source', () => {
    let unsubscribed = false;
    fakeSocket = new Observable(() => {
      return () => (unsubscribed = true);
    }) as any;
    const spy = spyOn(rxJsWebSocket, 'webSocket');
    spy.and.returnValue(fakeSocket);

    const liveStreams = new LiveStreams();
    const stream1 = liveStreams.getLegacyStream(makeTarget('url_to_match'));
    const subscription = stream1.subscribe({
      next: noop,
    });
    subscription.unsubscribe();
    expect(unsubscribed).toBe(true);
  });
});