org.rrd4j.core.RrdDb Java Examples

@Test
//Commented 14-Oct-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 15-Sep-2018
public void testBasic() throws Exception {
  timeSource.sleep(10000);
  List<Pair<String, Long>> list = handler.getFactory().list(100);
  // solr.jvm, solr.node, solr.collection..system
  assertEquals(list.toString(), 3, list.size());
  for (Pair<String, Long> p : list) {
    RrdDb db = new RrdDb(MetricsHistoryHandler.URI_PREFIX + p.first(), true, handler.getFactory());
    int dsCount = db.getDsCount();
    int arcCount = db.getArcCount();
    assertTrue("dsCount should be > 0, was " + dsCount, dsCount > 0);
    assertEquals("arcCount", 5, arcCount);
    db.close();
  }
}
 

/**
 * Periodically dumps the new mbean state to the data base
 */
public void run() {
    try {

        RrdDb dataBase = new RrdDb(dataBaseFile);

        logger.debug("RRD database configuration:\n" + dataBase.getRrdDef().dump());

        while (!terminate) {
            synchronized (dataSources) {

                dataSources.wait(step * 1000);

                if (terminate) {
                    break;
                }
                sample(dataBase, System.currentTimeMillis());
            }
        }
        dataBase.close();
    } catch (Exception e) {
        logger.error(e.getMessage(), e);
    }
}
 

@Test
public void samplesAreCreated_2Beans() throws Exception {
    File rrdFile = createTempRRDFile();

    Fake fakeBean = new Fake();
    Fake fakeBean2 = new Fake();
    MBeanServer beanServer = MBeanServerFactory.createMBeanServer();
    beanServer.registerMBean(fakeBean, new ObjectName("java.lang:type=Memory"));
    beanServer.registerMBean(fakeBean2, new ObjectName("sigar:Type=Mem"));

    RRDSigarDataStore store = new RRDSigarDataStore(beanServer, rrdFile.getPath(), 4, Logger.getLogger("test"));
    RrdDb dataBase = new RrdDb(rrdFile.getPath());

    // sample 5 times every 10 seconds
    long firstSampleTime = System.currentTimeMillis();
    for (int i = 1; i <= 5; i++) {
        store.sample(dataBase, firstSampleTime + i * TEN_SECONDS);
    }

    assertEquals((firstSampleTime + 5 * TEN_SECONDS) / 1000, dataBase.getLastUpdateTime());

    assertEquals(42, dataBase.getDatasource("ValueMemory").getLastValue(), 0.001);
    assertEquals(42, dataBase.getDatasource("ValueMem").getLastValue(), 0.001);
}
 

private void doDump(final RrdDb db) {
    if (savedData) {
        log.info("Saving RRD");
        try {
            db.exportXml(getExportFile().toString());
            savedData = false;
        } catch (final IOException e) {
            log.error("Can't dump the rrd, next restart will likely not be able to reuse it: " + e.getMessage(), e);
        }
    } else {
        log.debug("No project created, no RRD dump update");
    }
}
 

private RrdAccessor(RrdDb rrdDb) {
    this.rrdDb = rrdDb;
    try {
        this.logger.info("Opened RRD with {} archives and data sources {}.", rrdDb.getArcCount(), Arrays.toString(rrdDb.getDsNames()));
    } catch (Throwable t) {
        throw new RuntimeException(t);
    }
}
 

public static RrdAccessor open(String rrdPath) {
    try {
        final RrdDb rrdDb = new RrdDb("", "rrdtool:/" + rrdPath, RrdBackendFactory.getFactory("MEMORY"));
        rrdDb.getLastUpdateTime();
        return new RrdAccessor(rrdDb);
    } catch (IOException e) {
        throw new RuntimeException("Could not open RRD.", e);
    }
}
 

private RrdDb getOrCreateDb(String registry, Group group) {
  RrdDb db = knownDbs.computeIfAbsent(registry, r -> {
    RrdDef def = createDef(r, group);
    try {
      RrdDb newDb = new RrdDb(def, factory);
      return newDb;
    } catch (IOException e) {
      log.warn("Can't create RrdDb for registry {}, group {}: {}", registry, group, e);
      return null;
    }
  });
  return db;
}
 

private String dumpData(RrdDb db, FetchData fd) throws Exception {
  Map<String, Object> map = new LinkedHashMap<>();
  map.put("dbLastUpdateTime", db.getLastUpdateTime());
  map.put("firstTimestamp", fd.getFirstTimestamp());
  map.put("lastTimestamp", fd.getLastTimestamp());
  map.put("timestamps", fd.getTimestamps());
  map.put("data", fd.dump());
  return Utils.toJSONString(map);
}
 

@Override
public MetricsDatabase build() {
    checkNotNull(metricName, METRIC_NAME_MSG);
    checkNotNull(resourceName, RESOURCE_NAME_MSG);
    checkArgument(!dsDefs.isEmpty(), METRIC_TYPE_MSG);

    // define the resolution of monitored metrics
    rrdDef = new RrdDef(DB_PATH + SPLITTER + metricName +
             SPLITTER + resourceName, RESOLUTION_IN_SECOND);

    try {
        DsDef[] dsDefArray = new DsDef[dsDefs.size()];
        IntStream.range(0, dsDefs.size()).forEach(i -> dsDefArray[i] = dsDefs.get(i));

        rrdDef.addDatasource(dsDefArray);
        rrdDef.setStep(RESOLUTION_IN_SECOND);

        // raw archive, no aggregation is required
        ArcDef rawArchive = new ArcDef(CONSOL_FUNCTION, XFF_VALUE,
                STEP_VALUE, ROW_VALUE);
        rrdDef.addArchive(rawArchive);

        // always store the metric data in memory...
        rrdDb = new RrdDb(rrdDef, RrdBackendFactory.getFactory(STORING_METHOD));
    } catch (IOException e) {
        log.warn("Failed to create a new round-robin database due to {}", e);
    }

    return new DefaultMetricsDatabase(metricName, resourceName, rrdDb);
}
 

/**
 * Adds a line for the item to the graph definition.
 * The color of the line is determined by the counter, it simply picks the according index from LINECOLORS (and
 * rolls over if necessary).
 * 
 * @param graphDef the graph definition to fill
 * @param item the item to add a line for
 * @param counter defines the number of the datasource and is used to determine the line color
 */
protected void addLine(RrdGraphDef graphDef, Item item, int counter) {
    Color color = LINECOLORS[counter % LINECOLORS.length];
    String label = itemUIRegistry.getLabel(item.getName());
    String rrdName = RRD4jService.DB_FOLDER + File.separator + item.getName() + ".rrd";
    ConsolFun consolFun;
    if (label != null && label.contains("[") && label.contains("]")) {
        label = label.substring(0, label.indexOf('['));
    }
    try {
        RrdDb db = new RrdDb(rrdName);
        consolFun = db.getRrdDef().getArcDefs()[0].getConsolFun();
        db.close();
    } catch (IOException e) {
        consolFun = ConsolFun.MAX;
    }
    if (item instanceof NumberItem) {
        // we only draw a line
        graphDef.datasource(Integer.toString(counter), rrdName, "state", consolFun); // RRD4jService.getConsolidationFunction(item));
        graphDef.line(Integer.toString(counter), color, label, 2);
    } else {
        // we draw a line and fill the area beneath it with a transparent color
        graphDef.datasource(Integer.toString(counter), rrdName, "state", consolFun); // RRD4jService.getConsolidationFunction(item));
        Color areaColor = AREACOLORS[counter % LINECOLORS.length];

        graphDef.area(Integer.toString(counter), areaColor);
        graphDef.line(Integer.toString(counter), color, label, 2);
    }
}
 

public ConsolFun getConsolidationFunction(RrdDb db) {
    try {
        return db.getRrdDef().getArcDefs()[0].getConsolFun();
    } catch (IOException e) {
        return ConsolFun.MAX;
    }
}
 

private NamedList<Object> getDbData(RrdDb db, String[] dsNames, Format format, SolrParams params) throws IOException {
  NamedList<Object> res = new SimpleOrderedMap<>();
  if (dsNames == null || dsNames.length == 0) {
    dsNames = db.getDsNames();
  }
  StringBuilder str = new StringBuilder();
  RrdDef def = db.getRrdDef();
  ArcDef[] arcDefs = def.getArcDefs();
  for (ArcDef arcDef : arcDefs) {
    SimpleOrderedMap<Object> map = new SimpleOrderedMap<>();
    res.add(arcDef.dump(), map);
    Archive a = db.getArchive(arcDef.getConsolFun(), arcDef.getSteps());
    // startTime / endTime, arcStep are in seconds
    FetchRequest fr = db.createFetchRequest(arcDef.getConsolFun(),
        a.getStartTime() - a.getArcStep(),
        a.getEndTime() + a.getArcStep());
    FetchData fd = fr.fetchData();
    if (format != Format.GRAPH) {
      // add timestamps separately from values
      long[] timestamps = fd.getTimestamps();
      if (format == Format.LIST) {
        // Arrays.asList works only on arrays of Objects
        map.add("timestamps", Arrays.stream(timestamps).boxed().collect(Collectors.toList()));
      } else {
        str.setLength(0);
        for (int i = 0; i < timestamps.length; i++) {
          if (i > 0) {
            str.append('\n');
          }
          str.append(String.valueOf(timestamps[i]));
        }
        map.add("timestamps", str.toString());
      }
    }
    SimpleOrderedMap<Object> values = new SimpleOrderedMap<>();
    map.add("values", values);
    for (String name : dsNames) {
      double[] vals = fd.getValues(name);
      switch (format) {
        case GRAPH:
          RrdGraphDef graphDef = new RrdGraphDef();
          graphDef.setTitle(name);
          graphDef.datasource(name, fd);
          graphDef.setStartTime(a.getStartTime() - a.getArcStep());
          graphDef.setEndTime(a.getEndTime() + a.getArcStep());
          graphDef.setPoolUsed(false);
          graphDef.setAltAutoscale(true);
          graphDef.setAltYGrid(true);
          graphDef.setAltYMrtg(true);
          graphDef.setSignature("Apache Solr " + versionString);
          graphDef.setNoLegend(true);
          graphDef.setAntiAliasing(true);
          graphDef.setTextAntiAliasing(true);
          graphDef.setWidth(500);
          graphDef.setHeight(175);
          graphDef.setTimeZone(TimeZone.getDefault());
          graphDef.setLocale(Locale.ROOT);
          // redraw immediately
          graphDef.setLazy(false);
          // area with a border
          graphDef.area(name, new Color(0xffb860), null);
          graphDef.line(name, Color.RED, null, 1.0f);
          RrdGraph graph = new RrdGraph(graphDef);
          BufferedImage bi = new BufferedImage(
              graph.getRrdGraphInfo().getWidth(),
              graph.getRrdGraphInfo().getHeight(),
              BufferedImage.TYPE_INT_RGB);
          graph.render(bi.getGraphics());
          ByteArrayOutputStream baos = new ByteArrayOutputStream();
          ImageIO.write(bi, "png", baos);
          values.add(name, Base64.byteArrayToBase64(baos.toByteArray()));
          break;
        case STRING:
          str.setLength(0);
          for (int i = 0; i < vals.length; i++) {
            if (i > 0) {
              str.append('\n');
            }
            str.append(String.valueOf(vals[i]));
          }
          values.add(name, str.toString());
          break;
        case LIST:
          values.add(name, Arrays.stream(vals).boxed().collect(Collectors.toList()));
          break;
      }
    }
  }
  return res;
}
 

protected void initDatabase() throws IOException {
    if (!new File(dataBaseFile).exists()) {
        if (step <= 0) {
            logger.debug("Provided step is invalid, forcing it to " + DEFAULT_STEP_IN_SECONDS);
            step = DEFAULT_STEP_IN_SECONDS;
        }
        logger.info("Node's statistics are saved in " + dataBaseFile);

        RrdDef rrdDef = new RrdDef(dataBaseFile, System.currentTimeMillis() / 1000, step);

        for (String dataSource : dataSources.keySet()) {
            rrdDef.addDatasource(dataSource, DsType.GAUGE, 600, 0, Double.NaN);
        }

        // for step equals 4 seconds
        // Archive of 10 minutes = 600 seconds (4 * 1 * 150) of completely detailed data
        ConsolFun[] consolidationFunctions = new ConsolFun[] { ConsolFun.AVERAGE, ConsolFun.MIN, ConsolFun.MAX,
                                                               ConsolFun.LAST, ConsolFun.FIRST, ConsolFun.TOTAL };
        addArchives(rrdDef, consolidationFunctions, 0.5, 1, 150);

        // An archive of 1 hour = 3600 seconds (4 * 5 * 180) i.e. 180 averages of 5 steps
        addArchives(rrdDef, consolidationFunctions, 0.5, 5, 180);

        // An archive of 4 hours = 14400 seconds (4 * 10 * 360) i.e. 360 averages of 10 steps
        addArchives(rrdDef, consolidationFunctions, 0.5, 10, 360);

        // An archive of 8 hours = 28800 seconds (4 * 20 * 360) i.e. 360 averages of 20 steps
        addArchives(rrdDef, consolidationFunctions, 0.5, 20, 360);

        // An archive of 24 hours = 86400 seconds (4 * 30 * 720) i.e. 720 averages of 30 steps
        addArchives(rrdDef, consolidationFunctions, 0.5, 30, 720);

        // An archive of 1 week = 604800 seconds (4 * 210 * 720) i.e. 720 averages of 210 steps
        addArchives(rrdDef, consolidationFunctions, 0.5, 210, 720);

        // An archive of 1 month ~= 28 days = 604800 seconds (4 * 840 * 720) i.e. 720 averages of 840 steps
        addArchives(rrdDef, consolidationFunctions, 0.5, 840, 720);

        // An archive of 1 year = 364 days = 31449600 seconds (4 * 10920 * 720) i.e. 720 averages of 10920 steps
        addArchives(rrdDef, consolidationFunctions, 0.5, 10920, 720);

        RrdDb dataBase = new RrdDb(rrdDef);
        dataBase.close();
    } else {
        logger.info("Using existing RRD database: " + new File(dataBaseFile).getAbsolutePath());
    }
}
 

public static String rrdContent(byte[] rrd4j, String newRange, String[] dataSources, String function)
        throws IOException {

    File rrd4jDb = File.createTempFile("database", "rr4dj");
    // construct the JSON response directly in a String
    StringBuilder result = new StringBuilder();
    try {
        try (OutputStream out = new FileOutputStream(rrd4jDb)) {
            out.write(rrd4j);
        }

        // create RRD4J DB, should be identical to the one held by the RM
        RrdDb db = new RrdDb(rrd4jDb.getAbsolutePath(), true);

        long timeEnd = db.getLastUpdateTime();
        // force float separator for JSON parsing
        DecimalFormatSymbols otherSymbols = new DecimalFormatSymbols(Locale.US);
        otherSymbols.setDecimalSeparator('.');
        // formatting will greatly reduce response size
        DecimalFormat formatter = new DecimalFormat("###.###", otherSymbols);

        result.append("{");

        for (int i = 0; i < dataSources.length; i++) {
            String dataSource = dataSources[i];
            char zone = newRange.charAt(i);
            long timeStart = timeEnd - secondsInZone(zone);

            FetchRequest req = db.createFetchRequest(function != null ? ConsolFun.valueOf(function)
                                                                      : ConsolFun.AVERAGE,
                                                     timeStart,
                                                     timeEnd);
            req.setFilter(dataSource);
            FetchData fetchData = req.fetchData();
            result.append("\"").append(dataSource).append("\":[");

            double[] values = fetchData.getValues(dataSource);

            int nValuesToTake = values.length;
            // if the last value is NaN then we decide to not send it to the client
            // why? Because when we retrieve from RDD file, somehow the latest is
            // always NaN, which is not what we want.
            if (Double.compare(values[values.length - 1], Double.NaN) == 0) {
                --nValuesToTake;
            }

            String collect = Arrays.stream(values)
                                   .boxed()
                                   .collect(Collectors.toList())
                                   .subList(0, nValuesToTake)
                                   .stream()
                                   .map(value -> {
                                       if (Double.compare(Double.NaN, value) == 0) {
                                           return "null";
                                       } else {
                                           return formatter.format(value);
                                       }
                                   })
                                   .collect(Collectors.joining(","));
            result.append(collect);
            result.append(']');
            if (i < dataSources.length - 1) {
                result.append(',');
            }
        }
        result.append("}");

        db.close();
    } finally {
        FileUtils.deleteQuietly(rrd4jDb);
    }

    return result.toString();
}
 

@Override
public String getAttributesHistory(String objectName, String[] attrs, String range) throws IOException {

    RrdDb db = new RrdDb(statBaseName, true);

    long timeEnd = db.getLastUpdateTime();
    // force float separator for JSON parsing
    DecimalFormatSymbols otherSymbols = new DecimalFormatSymbols(Locale.US);
    otherSymbols.setDecimalSeparator('.');
    // formatting will greatly reduce response size
    DecimalFormat formatter = new DecimalFormat("###.###", otherSymbols);

    // construct the JSON response directly in a String
    StringBuilder result = new StringBuilder();
    result.append("{");

    for (int i = 0; i < attrs.length; i++) {

        String dataSource = RRDSigarDataStore.toDataStoreName(attrs[i] + "-" + objectName);

        char zone = range.charAt(0);
        long timeStart = timeEnd - MBeanInfoViewer.secondsInZone(zone);

        FetchRequest req = db.createFetchRequest(ConsolFun.AVERAGE, timeStart, timeEnd);
        req.setFilter(dataSource);
        FetchData fetchData = req.fetchData();
        result.append("\"").append(dataSource).append("\":[");

        double[] values = fetchData.getValues(dataSource);
        for (int j = 0; j < values.length - 1; j++) {
            if (Double.compare(Double.NaN, values[j]) == 0) {
                result.append("null");
            } else {
                result.append(formatter.format(values[j]));
            }
            if (j < values.length - 2)
                result.append(',');
        }
        result.append(']');
        if (i < attrs.length - 1)
            result.append(',');
    }
    result.append("}");

    db.close();

    return result.toString();
}
 

/**
 * Constructs a metrics database using the given metric name and
 * round robin database.
 *
 * @param metricName  metric name
 * @param rrdDb       round robin database
 */
private DefaultMetricsDatabase(String metricName, String resourceName, RrdDb rrdDb) {
    this.metricName = metricName;
    this.resourceName = resourceName;
    this.rrdDb = rrdDb;
}