Java Code Examples for org.apache.lucene.util.Constants#LINUX

@SuppressForbidden(reason = "tries to determine if disk is spinning")
// TODO: move PathUtils to be package-private here instead of 
// public+forbidden api!
ESFileStore(FileStore in) {
    this.in = in;
    Boolean spins;
    // Lucene's IOUtils.spins only works on Linux today:
    if (Constants.LINUX) {
        try {
            spins = IOUtils.spins(PathUtils.get(getMountPointLinux(in)));
        } catch (Exception e) {
            spins = null;
        }
    } else {
        spins = null;
    }
    this.spins = spins;
}
 

/**
 * Attempt to drop the capability to execute for the process.
 * <p>
 * This is best effort and OS and architecture dependent. It may throw any Throwable.
 * @return 0 if we can do this for application threads, 1 for the entire process
 */
static int init(Path tmpFile) throws Throwable {
    if (Constants.LINUX) {
        return linuxImpl();
    } else if (Constants.MAC_OS_X) {
        // try to enable both mechanisms if possible
        bsdImpl();
        macImpl(tmpFile);
        return 1;
    } else if (Constants.SUN_OS) {
        solarisImpl();
        return 1;
    } else if (Constants.FREE_BSD || OPENBSD) {
        bsdImpl();
        return 1;
    } else if (Constants.WINDOWS) {
        windowsImpl();
        return 1;
    } else {
        throw new UnsupportedOperationException("syscall filtering not supported for OS: '" + Constants.OS_NAME + "'");
    }
}
 

static void trySetMaxNumberOfThreads() {
    if (Constants.LINUX) {
        // this is only valid on Linux and the value *is* different on OS X
        // see /usr/include/sys/resource.h on OS X
        // on Linux the resource RLIMIT_NPROC means *the number of threads*
        // this is in opposition to BSD-derived OSes
        final int rlimit_nproc = 6;

        final JNACLibrary.Rlimit rlimit = new JNACLibrary.Rlimit();
        if (JNACLibrary.getrlimit(rlimit_nproc, rlimit) == 0) {
            MAX_NUMBER_OF_THREADS = rlimit.rlim_cur.longValue();
        } else {
            LOGGER.warn("unable to retrieve max number of threads [" + JNACLibrary.strerror(Native.getLastError()) + "]");
        }
    }
}
 

private IOException convertMapFailedIOException(IOException ioe, String resourceDescription, int bufSize) {
  final String originalMessage;
  final Throwable originalCause;
  if (ioe.getCause() instanceof OutOfMemoryError) {
    // nested OOM confuses users, because it's "incorrect", just print a plain message:
    originalMessage = "Map failed";
    originalCause = null;
  } else {
    originalMessage = ioe.getMessage();
    originalCause = ioe.getCause();
  }
  final String moreInfo;
  if (!Constants.JRE_IS_64BIT) {
    moreInfo = "MMapDirectory should only be used on 64bit platforms, because the address space on 32bit operating systems is too small. ";
  } else if (Constants.WINDOWS) {
    moreInfo = "Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. ";
  } else if (Constants.LINUX) {
    moreInfo = "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'), and 'sysctl vm.max_map_count'. ";
  } else {
    moreInfo = "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'). ";
  }
  final IOException newIoe = new IOException(String.format(Locale.ENGLISH,
      "%s: %s [this may be caused by lack of enough unfragmented virtual address space "+
      "or too restrictive virtual memory limits enforced by the operating system, "+
      "preventing us to map a chunk of %d bytes. %sMore information: "+
      "http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]",
      originalMessage, resourceDescription, bufSize, moreInfo), originalCause);
  newIoe.setStackTrace(ioe.getStackTrace());
  return newIoe;
}
 

@SuppressForbidden(reason = "tries to determine if disk is spinning")
// TODO: move PathUtils to be package-private here instead of
// public+forbidden api!
ESFileStore(final FileStore in) {
    this.in = in;
    if (Constants.LINUX) {
        try {
            final List<String> lines = Files.readAllLines(PathUtils.get("/proc/self/mountinfo"));
            for (final String line : lines) {
                final String[] fields = line.trim().split("\\s+");
                final String mountPoint = fields[4];
                if (mountPoint.equals(getMountPointLinux(in))) {
                    final String[] deviceNumbers = fields[2].split(":");
                    majorDeviceNumber = Integer.parseInt(deviceNumbers[0]);
                    minorDeviceNumber = Integer.parseInt(deviceNumbers[1]);
                    break;
                }
            }
        } catch (final Exception e) {
            majorDeviceNumber = -1;
            minorDeviceNumber = -1;
        }
    } else {
        majorDeviceNumber = -1;
        minorDeviceNumber = -1;
    }
}
 

public OsStats osStats() {
    final OsStats.Cpu cpu = new OsStats.Cpu(getSystemCpuPercent(), getSystemLoadAverage());
    final OsStats.Mem mem = new OsStats.Mem(getTotalPhysicalMemorySize(), getFreePhysicalMemorySize());
    final OsStats.Swap swap = new OsStats.Swap(getTotalSwapSpaceSize(), getFreeSwapSpaceSize());
    final OsStats.Cgroup cgroup = Constants.LINUX ? getCgroup() : null;
    return new OsStats(System.currentTimeMillis(), cpu, mem, swap, cgroup);
}
 

public FsInfo stats(FsInfo previous, @Nullable ClusterInfo clusterInfo) throws IOException {
    if (!nodeEnv.hasNodeFile()) {
        return new FsInfo(System.currentTimeMillis(), null, new FsInfo.Path[0]);
    }
    NodePath[] dataLocations = nodeEnv.nodePaths();
    FsInfo.Path[] paths = new FsInfo.Path[dataLocations.length];
    for (int i = 0; i < dataLocations.length; i++) {
        paths[i] = getFSInfo(dataLocations[i]);
    }
    FsInfo.IoStats ioStats = null;
    if (Constants.LINUX) {
        Set<Tuple<Integer, Integer>> devicesNumbers = new HashSet<>();
        for (int i = 0; i < dataLocations.length; i++) {
            if (dataLocations[i].majorDeviceNumber != -1 && dataLocations[i].minorDeviceNumber != -1) {
                devicesNumbers.add(Tuple.tuple(dataLocations[i].majorDeviceNumber, dataLocations[i].minorDeviceNumber));
            }
        }
        ioStats = ioStats(devicesNumbers, previous);
    }
    DiskUsage leastDiskEstimate = null;
    DiskUsage mostDiskEstimate = null;
    if (clusterInfo != null) {
        leastDiskEstimate = clusterInfo.getNodeLeastAvailableDiskUsages().get(nodeEnv.nodeId());
        mostDiskEstimate = clusterInfo.getNodeMostAvailableDiskUsages().get(nodeEnv.nodeId());
    }
    return new FsInfo(System.currentTimeMillis(), ioStats, paths, leastDiskEstimate, mostDiskEstimate);
}
 

static void trySetMaxSizeVirtualMemory() {
    if (Constants.LINUX || Constants.MAC_OS_X) {
        final JNACLibrary.Rlimit rlimit = new JNACLibrary.Rlimit();
        if (JNACLibrary.getrlimit(JNACLibrary.RLIMIT_AS, rlimit) == 0) {
            MAX_SIZE_VIRTUAL_MEMORY = rlimit.rlim_cur.longValue();
        } else {
            LOGGER.warn("unable to retrieve max size virtual memory [" + JNACLibrary.strerror(Native.getLastError()) + "]");
        }
    }
}
 

static void trySetMaxFileSize() {
    if (Constants.LINUX || Constants.MAC_OS_X) {
        final JNACLibrary.Rlimit rlimit = new JNACLibrary.Rlimit();
        if (JNACLibrary.getrlimit(JNACLibrary.RLIMIT_FSIZE, rlimit) == 0) {
            MAX_FILE_SIZE = rlimit.rlim_cur.longValue();
        } else {
            LOGGER.warn("unable to retrieve max file size [" + JNACLibrary.strerror(Native.getLastError()) + "]");
        }
    }
}
 

static void tryMlockall() {
    int errno = Integer.MIN_VALUE;
    String errMsg = null;
    boolean rlimitSuccess = false;
    long softLimit = 0;
    long hardLimit = 0;
    
    try {
        int result = JNACLibrary.mlockall(JNACLibrary.MCL_CURRENT);
        if (result == 0) {
            LOCAL_MLOCKALL = true;
            return;
        }
        
        errno = Native.getLastError();
        errMsg = JNACLibrary.strerror(errno);
        if (Constants.LINUX || Constants.MAC_OS_X) {
            // we only know RLIMIT_MEMLOCK for these two at the moment. 
            JNACLibrary.Rlimit rlimit = new JNACLibrary.Rlimit();
            if (JNACLibrary.getrlimit(JNACLibrary.RLIMIT_MEMLOCK, rlimit) == 0) {
                rlimitSuccess = true;
                softLimit = rlimit.rlim_cur.longValue();
                hardLimit = rlimit.rlim_max.longValue();
            } else {
                logger.warn("Unable to retrieve resource limits: " + JNACLibrary.strerror(Native.getLastError()));
            }
        }
    } catch (UnsatisfiedLinkError e) {
        // this will have already been logged by CLibrary, no need to repeat it
        return;
    }

    // mlockall failed for some reason
    logger.warn("Unable to lock JVM Memory: error=" + errno + ",reason=" + errMsg);
    logger.warn("This can result in part of the JVM being swapped out.");
    if (errno == JNACLibrary.ENOMEM) {
        if (rlimitSuccess) {
            logger.warn("Increase RLIMIT_MEMLOCK, soft limit: " + rlimitToString(softLimit) + ", hard limit: " + rlimitToString(hardLimit));
            if (Constants.LINUX) {
                // give specific instructions for the linux case to make it easy
                String user = System.getProperty("user.name");
                logger.warn("These can be adjusted by modifying /etc/security/limits.conf, for example: \n" +
                            "\t# allow user '" + user + "' mlockall\n" +
                            "\t" + user + " soft memlock unlimited\n" +
                            "\t" + user + " hard memlock unlimited"
                           );
                logger.warn("If you are logged in interactively, you will have to re-login for the new limits to take effect.");
            }
        } else {
            logger.warn("Increase RLIMIT_MEMLOCK (ulimit).");
        }
    }
}
 

static void tryMlockall() {
    int errno = Integer.MIN_VALUE;
    String errMsg = null;
    boolean rlimitSuccess = false;
    long softLimit = 0;
    long hardLimit = 0;

    try {
        int result = JNACLibrary.mlockall(JNACLibrary.MCL_CURRENT);
        if (result == 0) {
            LOCAL_MLOCKALL = true;
            return;
        }

        errno = Native.getLastError();
        errMsg = JNACLibrary.strerror(errno);
        if (Constants.LINUX || Constants.MAC_OS_X) {
            // we only know RLIMIT_MEMLOCK for these two at the moment.
            JNACLibrary.Rlimit rlimit = new JNACLibrary.Rlimit();
            if (JNACLibrary.getrlimit(JNACLibrary.RLIMIT_MEMLOCK, rlimit) == 0) {
                rlimitSuccess = true;
                softLimit = rlimit.rlim_cur.longValue();
                hardLimit = rlimit.rlim_max.longValue();
            } else {
                LOGGER.warn("Unable to retrieve resource limits: {}", JNACLibrary.strerror(Native.getLastError()));
            }
        }
    } catch (UnsatisfiedLinkError e) {
        // this will have already been logged by CLibrary, no need to repeat it
        return;
    }

    // mlockall failed for some reason
    LOGGER.warn("Unable to lock JVM Memory: error={}, reason={}", errno , errMsg);
    LOGGER.warn("This can result in part of the JVM being swapped out.");
    if (errno == JNACLibrary.ENOMEM) {
        if (rlimitSuccess) {
            LOGGER.warn("Increase RLIMIT_MEMLOCK, soft limit: {}, hard limit: {}", rlimitToString(softLimit), rlimitToString(hardLimit));
            if (Constants.LINUX) {
                // give specific instructions for the linux case to make it easy
                String user = System.getProperty("user.name");
                LOGGER.warn("These can be adjusted by modifying /etc/security/limits.conf, for example: \n" +
                            "\t# allow user '{}' mlockall\n" +
                            "\t{} soft memlock unlimited\n" +
                            "\t{} hard memlock unlimited",
                            user, user, user
                            );
                LOGGER.warn("If you are logged in interactively, you will have to re-login for the new limits to take effect.");
            }
        } else {
            LOGGER.warn("Increase RLIMIT_MEMLOCK (ulimit).");
        }
    }
}
 

@Test
public void testSysNodesCgroup() throws Exception {
    if (Constants.LINUX && !"true".equals(System.getenv("SHIPPABLE"))) { // cgroups are only available on Linux
        SQLResponse response = execute("select" +
                                       " os['cgroup']['cpuacct']['control_group']," +
                                       " os['cgroup']['cpuacct']['usage_nanos']," +
                                       " os['cgroup']['cpu']['control_group']," +
                                       " os['cgroup']['cpu']['cfs_period_micros']," +
                                       " os['cgroup']['cpu']['cfs_quota_micros']," +
                                       " os['cgroup']['cpu']['num_elapsed_periods']," +
                                       " os['cgroup']['cpu']['num_times_throttled']," +
                                       " os['cgroup']['cpu']['time_throttled_nanos']" +
                                       " from sys.nodes limit 1");
        assertThat(response.rowCount(), is(1L));
        assertThat(response.rows()[0][0], notNullValue());
        assertThat((long) response.rows()[0][1], greaterThanOrEqualTo(0L));
        assertThat(response.rows()[0][2], notNullValue());
        assertThat((long) response.rows()[0][3], greaterThanOrEqualTo(0L));
        assertThat((long) response.rows()[0][4], anyOf(equalTo(-1L), greaterThanOrEqualTo(0L)));
        assertThat((long) response.rows()[0][5], greaterThanOrEqualTo(0L));
        assertThat((long) response.rows()[0][6], greaterThanOrEqualTo(0L));
        assertThat((long) response.rows()[0][7], greaterThanOrEqualTo(0L));
    } else {
        // for all other OS cgroup fields should return `null`
        response = execute("select os['cgroup']," +
                           " os['cgroup']['cpuacct']," +
                           " os['cgroup']['cpuacct']['control_group']," +
                           " os['cgroup']['cpuacct']['usage_nanos']," +
                           " os['cgroup']['cpu']," +
                           " os['cgroup']['cpu']['control_group']," +
                           " os['cgroup']['cpu']['cfs_period_micros']," +
                           " os['cgroup']['cpu']['cfs_quota_micros']," +
                           " os['cgroup']['cpu']['num_elapsed_periods']," +
                           " os['cgroup']['cpu']['num_times_throttled']," +
                           " os['cgroup']['cpu']['time_throttled_nanos']" +
                           " from sys.nodes limit 1");
        assertThat(response.rowCount(), is(1L));
        for (int i = 0; i <= 10; i++) {
            assertThat(response.rows()[0][1], Matchers.is(Map.of()));
        }
    }

}