org.apache.hadoop.util.GSet Java Examples

The following examples show how to use org.apache.hadoop.util.GSet. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestHadoopNNAWithStreamEngine.java    From NNAnalytics with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new HadoopWebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "false");
  conf.set("nna.historical", "false");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #2
Source File: TestHadoopNNAOperations.java    From NNAnalytics with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new HadoopWebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "false");
  conf.set("nna.historical", "false");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #3
Source File: TestLdapAuth.java    From NNAnalytics with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new WebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "false");
  conf.set("nna.historical", "false");
  conf.set("nna.localonly.users", "hdfs:hdfs,hdfsW:hdfsW,hdfsR:hdfsR,testEmpty:");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #4
Source File: TestHadoopNNAWithStreamEngine.java    From NNAnalytics with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new HadoopWebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "false");
  conf.set("nna.historical", "false");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #5
Source File: TestHadoopNNAOperations.java    From NNAnalytics with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new HadoopWebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "false");
  conf.set("nna.historical", "false");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #6
Source File: TestNNAWithStreamEngine.java    From NNAnalytics with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new WebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "false");
  conf.set("nna.historical", "false");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #7
Source File: GSetGeneratorBase.java    From NNAnalytics with Apache License 2.0 6 votes vote down vote up 
public GSet<INode, INodeWithAdditionalFields> getGSet(
    short depth, int numDirsPerDepth, int numFilesPerDir) {
  if (gset == null || gset.size() == 0) {
    try {
      gset = getEmptyGSet();
      gset.put(root);
      DIRS_MADE++;
      generateGSet(gset, root, numFilesPerDir, numDirsPerDepth, depth);
      LOG.info("Generated GSet size is: " + gset.size());
      assert FILES_MADE + DIRS_MADE == gset.size();
    } catch (IOException ignored) {
      /* Defaults not expected to throw Exceptions */
    }
  }
  return gset;
}
Example #8
Source File: GSetGenerator.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
public void generateGSet(
    GSet<INode, INodeWithAdditionalFields> newGSet,
    INodeDirectory parent,
    int filesPerDir,
    int numOfDirsPerDepth,
    short depth)
    throws IOException {
  if (depth == 0) {
    return;
  }
  List<INodeDirectory> childrenDirs = new ArrayList<>();
  for (int j = 1; j <= numOfDirsPerDepth; j++) {
    long mAndAtime = now - rand.nextLong(TimeUnit.DAYS.toMillis(365)); // 1 year
    int dirId = ID++;
    INodeDirectory dir =
        new INodeDirectory(dirId, ("dir" + j).getBytes(CHARSET), status, mAndAtime);
    boolean childAdded = parent.addChild(dir);
    if (rand.nextBoolean()) {
      dir.setQuota(BlockStoragePolicySuite.createDefaultSuite(), 9000L, 9999999999L, null);
      dir.getDirectoryWithQuotaFeature()
          .setSpaceConsumed(rand.nextLong(5000L), rand.nextLong(9999999991L), null);
    }
    if (childAdded) {
      generateFilesForDirectory(gset, dir, filesPerDir);
      dir.setParent(parent);
      INodeWithAdditionalFields booted = newGSet.put(dir);
      if (booted == null) {
        DIRS_MADE++;
      }
    }
    childrenDirs.add(dir);
  }
  for (INodeDirectory childrenDir : childrenDirs) {
    generateGSet(newGSet, childrenDir, filesPerDir, numOfDirsPerDepth, (short) (depth - 1));
  }
}
Example #9
Source File: TestNoHistorical.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new WebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "false");
  conf.set("nna.historical", "false");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #10
Source File: TestWebServerMainAuthorization.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new WebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "true");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #11
Source File: TestWebServerMainOperations.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new WebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "false");
  conf.set("nna.historical", "false");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #12
Source File: GSetGeneratorBase.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
protected abstract void generateGSet(
GSet<INode, INodeWithAdditionalFields> newGSet,
INodeDirectory parent,
int filesPerDir,
int numOfDirsPerDepth,
short depth)
throws IOException;
Example #13
Source File: ConcurrentHashMapINodeCollection.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Override
public Map<INode, INodeWithAdditionalFields> filterDirs(
    GSet<INode, INodeWithAdditionalFields> gset) {
  return StreamSupport.stream(gset.spliterator(), true)
      .filter(INode::isDirectory)
      .collect(Collectors.toConcurrentMap(node -> node, node -> node));
}
Example #14
Source File: GSetGenerator.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
public void generateGSet(
    GSet<INode, INodeWithAdditionalFields> newGSet,
    INodeDirectory parent,
    int filesPerDir,
    int numOfDirsPerDepth,
    short depth)
    throws IOException {
  if (depth == 0) {
    return;
  }
  List<INodeDirectory> childrenDirs = new ArrayList<>();
  for (int j = 1; j <= numOfDirsPerDepth; j++) {
    long mAndAtime = now - rand.nextLong(TimeUnit.DAYS.toMillis(365)); // 1 year
    int dirId = ID++;
    INodeDirectory dir =
        new INodeDirectory(dirId, ("dir" + j).getBytes(CHARSET), status, mAndAtime);
    boolean childAdded = parent.addChild(dir);
    if (rand.nextBoolean()) {
      dir.setQuota(BlockStoragePolicySuite.createDefaultSuite(), 9000L, 9999999999L, null);
      dir.getDirectoryWithQuotaFeature()
          .setSpaceConsumed(rand.nextLong(5000L), rand.nextLong(9999999991L), null);
    }
    if (childAdded) {
      generateFilesForDirectory(gset, dir, filesPerDir);
      dir.setParent(parent);
      INodeWithAdditionalFields booted = newGSet.put(dir);
      if (booted == null) {
        DIRS_MADE++;
      }
    }
    childrenDirs.add(dir);
  }
  for (INodeDirectory childrenDir : childrenDirs) {
    generateGSet(newGSet, childrenDir, filesPerDir, numOfDirsPerDepth, (short) (depth - 1));
  }
}
Example #15
Source File: GSetGenerator.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
public void generateGSet(
    GSet<INode, INodeWithAdditionalFields> newGSet,
    INodeDirectory parent,
    int filesPerDir,
    int numOfDirsPerDepth,
    short depth)
    throws IOException {
  if (depth == 0) {
    return;
  }
  List<INodeDirectory> childrenDirs = new ArrayList<>();
  for (int j = 1; j <= numOfDirsPerDepth; j++) {
    long mAndAtime = now - rand.nextLong(TimeUnit.DAYS.toMillis(365)); // 1 year
    int dirId = ID++;
    INodeDirectory dir =
        new INodeDirectory(dirId, ("dir" + j).getBytes(CHARSET), status, mAndAtime);
    boolean childAdded = parent.addChild(dir);
    if (rand.nextBoolean()) {
      dir.setQuota(BlockStoragePolicySuite.createDefaultSuite(), 9000L, 9999999999L, null);
      dir.getDirectoryWithQuotaFeature()
          .setSpaceConsumed(rand.nextLong(5000L), rand.nextLong(9999999991L), null);
    }
    if (childAdded) {
      generateFilesForDirectory(gset, dir, filesPerDir);
      dir.setParent(parent);
      INodeWithAdditionalFields booted = newGSet.put(dir);
      if (booted == null) {
        DIRS_MADE++;
      }
    }
    childrenDirs.add(dir);
  }
  for (INodeDirectory childrenDir : childrenDirs) {
    generateGSet(newGSet, childrenDir, filesPerDir, numOfDirsPerDepth, (short) (depth - 1));
  }
}
Example #16
Source File: TestHadoopNNAAuthorization.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new HadoopWebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "true");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #17
Source File: GSetGenerator.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
public void generateGSet(
    GSet<INode, INodeWithAdditionalFields> newGSet,
    INodeDirectory parent,
    int filesPerDir,
    int numOfDirsPerDepth,
    short depth)
    throws IOException {
  if (depth == 0) {
    return;
  }
  List<INodeDirectory> childrenDirs = new ArrayList<>();
  for (int j = 1; j <= numOfDirsPerDepth; j++) {
    long mAndAtime = now - rand.nextLong(TimeUnit.DAYS.toMillis(365)); // 1 year
    int dirId = ID++;
    INodeDirectory dir =
        new INodeDirectory(dirId, ("dir" + j).getBytes(CHARSET), status, mAndAtime);
    if (rand.nextBoolean()) {
      dir.setQuota(BlockStoragePolicySuite.createDefaultSuite(), 9000L, 9999999999L, null);
      dir.getDirectoryWithQuotaFeature()
          .setSpaceConsumed(rand.nextLong(5000L), rand.nextLong(9999999991L), null);
    }
    boolean childAdded = parent.addChild(dir);
    if (childAdded) {
      generateFilesForDirectory(gset, dir, filesPerDir);
      dir.setParent(parent);
      INodeWithAdditionalFields booted = newGSet.put(dir);
      if (booted == null) {
        DIRS_MADE++;
      }
    }
    childrenDirs.add(dir);
  }
  for (INodeDirectory childrenDir : childrenDirs) {
    generateGSet(newGSet, childrenDir, filesPerDir, numOfDirsPerDepth, (short) (depth - 1));
  }
}
Example #18
Source File: INodeMap.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
static INodeMap newInstance(INodeDirectory rootDir) {
  // Compute the map capacity by allocating 1% of total memory
  int capacity = LightWeightGSet.computeCapacity(1, "INodeMap");
  GSet<INode, INodeWithAdditionalFields> map
      = new LightWeightGSet<INode, INodeWithAdditionalFields>(capacity);
  map.put(rootDir);
  return new INodeMap(map);
}
Example #19
Source File: INodeMap.java    From big-c with Apache License 2.0 5 votes vote down vote up 
static INodeMap newInstance(INodeDirectory rootDir) {
  // Compute the map capacity by allocating 1% of total memory
  int capacity = LightWeightGSet.computeCapacity(1, "INodeMap");
  GSet<INode, INodeWithAdditionalFields> map
      = new LightWeightGSet<INode, INodeWithAdditionalFields>(capacity);
  map.put(rootDir);
  return new INodeMap(map);
}
Example #20
Source File: GSetGenerator.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
public void generateGSet(
    GSet<INode, INodeWithAdditionalFields> newGSet,
    INodeDirectory parent,
    int filesPerDir,
    int numOfDirsPerDepth,
    short depth)
    throws IOException {
  if (depth == 0) {
    return;
  }
  List<INodeDirectory> childrenDirs = new ArrayList<>();
  for (int j = 1; j <= numOfDirsPerDepth; j++) {
    long mAndAtime = now - rand.nextLong(TimeUnit.DAYS.toMillis(365)); // 1 year
    int dirId = ID++;
    INodeDirectory dir =
        new INodeDirectory(dirId, ("dir" + j).getBytes(CHARSET), status, mAndAtime);
    boolean childAdded = parent.addChild(dir);
    if (rand.nextBoolean()) {
      dir.setQuota(BlockStoragePolicySuite.createDefaultSuite(), 9000L, 9999999999L, null);
      dir.getDirectoryWithQuotaFeature()
          .setSpaceConsumed(rand.nextLong(5000L), rand.nextLong(9999999991L), null);
    }
    if (childAdded) {
      generateFilesForDirectory(gset, dir, filesPerDir);
      dir.setParent(parent);
      INodeWithAdditionalFields booted = newGSet.put(dir);
      if (booted == null) {
        DIRS_MADE++;
      }
    }
    childrenDirs.add(dir);
  }
  for (INodeDirectory childrenDir : childrenDirs) {
    generateGSet(newGSet, childrenDir, filesPerDir, numOfDirsPerDepth, (short) (depth - 1));
  }
}
Example #21
Source File: ConcurrentHashMapINodeCollection.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Override
public Map<INode, INodeWithAdditionalFields> filterFiles(
    GSet<INode, INodeWithAdditionalFields> gset) {
  return StreamSupport.stream(gset.spliterator(), true)
      .filter(INode::isFile)
      .collect(Collectors.toConcurrentMap(node -> node, node -> node));
}
Example #22
Source File: EclipseINodeCollection.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Override
public Map<INode, INodeWithAdditionalFields> filterFiles(
    GSet<INode, INodeWithAdditionalFields> gset) {
  return StreamSupport.stream(gset.spliterator(), true)
      .filter(INode::isFile)
      .collect(
          Collectors.toConcurrentMap(
              node -> node,
              node -> node,
              throwingMerger(),
              org.eclipse.collections.impl.map.mutable.ConcurrentHashMap::new));
}
Example #23
Source File: EclipseINodeCollection.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Override
public Map<INode, INodeWithAdditionalFields> filterDirs(
    GSet<INode, INodeWithAdditionalFields> gset) {
  return StreamSupport.stream(gset.spliterator(), true)
      .filter(INode::isDirectory)
      .collect(
          Collectors.toConcurrentMap(
              node -> node,
              node -> node,
              throwingMerger(),
              org.eclipse.collections.impl.map.mutable.ConcurrentHashMap::new));
}
Example #24
Source File: HadoopWebServerMain.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
/**
 * Main method of initializing an NNA instance. Web server will start first. Then INodes will
 * load.
 *
 * @param conf the application configuration
 * @param inodes preloaded inodes (used by tests)
 * @param preloadedHadoopConf preloaded hadoop configuration (used by tests)
 * @throws Exception errors here will crash NNA
 */
@VisibleForTesting
public void init(
    ApplicationConfiguration conf,
    GSet<INode, INodeWithAdditionalFields> inodes,
    Configuration preloadedHadoopConf)
    throws Exception {
  // Initialize classes needed for NNA.
  SecurityContext securityContext = loadSecurityContext(conf);
  UsageMetrics usageMetrics = new UsageMetrics();

  // Start NameNodeAnalytics HTTP server.
  nnaHttpServer =
      new NameNodeAnalyticsHttpServer(
          preloadedHadoopConf,
          conf,
          getHttpServerBindAddress(conf),
          securityContext,
          nameNodeLoader,
          hsqlDriver,
          usageMetrics,
          runningQueries,
          runningOperations,
          internalService,
          operationService,
          queryLock,
          savingNamespace,
          cancelRequest);
  nnaHttpServer.start();

  // Bootstrap classes.
  nameNodeLoader.load(inodes, preloadedHadoopConf, conf);
  nameNodeLoader.initHistoryRecorder(hsqlDriver, conf, conf.getHistoricalEnabled());
  nameNodeLoader.initReloadThreads(internalService, conf);
}
Example #25
Source File: TestHadoopNNAAuthorization.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new HadoopWebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "true");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #26
Source File: AbstractQueryEngine.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked") /* We do unchecked casting to extract GSets */
private void filterINodes(
    GSet<INode, INodeWithAdditionalFields> gset, ApplicationConfiguration nnaConf)
    throws Exception {
  String inodeCollectionClass = nnaConf.getINodeCollectionImplementation();
  LOG.info("Filtering inodes with implementation: {}", inodeCollectionClass);
  Class<INodeFilterer> filtererClass = (Class<INodeFilterer>) Class.forName(inodeCollectionClass);
  INodeFilterer filterer = ReflectionUtils.newInstance(filtererClass, null);
  final long start = System.currentTimeMillis();
  files = filterer.filterFiles(gset);
  dirs = filterer.filterDirs(gset);
  all = CollectionsView.combine(files.keySet(), dirs.keySet());
  final long end = System.currentTimeMillis();
  LOG.info("Performing AbstractQE filtering of files and dirs took: {} ms.", (end - start));
}
Example #27
Source File: AbstractQueryEngine.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked") /* We do unchecked casting to extract GSets */
@Override // QueryEngine
public void handleGSet(
    GSet<INode, INodeWithAdditionalFields> preloaded,
    ApplicationConfiguration nnaConf,
    FSNamesystem namesystem)
    throws Exception {
  if (preloaded != null) {
    filterINodes(preloaded, nnaConf);
    return;
  }

  namesystem.writeLock();
  try {
    FSDirectory fsDirectory = namesystem.getFSDirectory();
    INodeMap inodeMap = fsDirectory.getINodeMap();
    Field mapField = inodeMap.getClass().getDeclaredField("map");
    mapField.setAccessible(true);
    GSet<INode, INodeWithAdditionalFields> gset =
        (GSet<INode, INodeWithAdditionalFields>) mapField.get(inodeMap);

    filterINodes(gset, nnaConf);

    GSet<INode, INodeWithAdditionalFields> newGSet = new GSetSeperatorWrapper(files, dirs);
    mapField.set(inodeMap, newGSet);
  } finally {
    namesystem.writeUnlock();
  }
}
Example #28
Source File: NonBlockingHashMapINodeCollection.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Override
public Map<INode, INodeWithAdditionalFields> filterFiles(
    GSet<INode, INodeWithAdditionalFields> gset) {
  return StreamSupport.stream(gset.spliterator(), true)
      .filter(INode::isFile)
      .collect(
          Collectors.toConcurrentMap(
              node -> node, node -> node, throwingMerger(), NonBlockingHashMap::new));
}
Example #29
Source File: NonBlockingHashMapINodeCollection.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Override
public Map<INode, INodeWithAdditionalFields> filterDirs(
    GSet<INode, INodeWithAdditionalFields> gset) {
  return StreamSupport.stream(gset.spliterator(), true)
      .filter(INode::isDirectory)
      .collect(
          Collectors.toConcurrentMap(
              node -> node, node -> node, throwingMerger(), NonBlockingHashMap::new));
}
Example #30
Source File: WebServerMain.java    From NNAnalytics with Apache License 2.0 4 votes vote down vote up 
@Override // ApplicationMain
@VisibleForTesting
public void init(ApplicationConfiguration conf, GSet<INode, INodeWithAdditionalFields> inodes)
    throws Exception {
  init(conf, inodes, null);
}