org.apache.hadoop.hdfs.server.namenode.INodeWithAdditionalFields Java Examples

The following examples show how to use org.apache.hadoop.hdfs.server.namenode.INodeWithAdditionalFields. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FSImageFormatPBSnapshot.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * save all the snapshot diff to fsimage
 */
public void serializeSnapshotDiffSection(OutputStream out)
    throws IOException {
  INodeMap inodesMap = fsn.getFSDirectory().getINodeMap();
  final List<INodeReference> refList = parent.getSaverContext()
      .getRefList();
  int i = 0;
  Iterator<INodeWithAdditionalFields> iter = inodesMap.getMapIterator();
  while (iter.hasNext()) {
    INodeWithAdditionalFields inode = iter.next();
    if (inode.isFile()) {
      serializeFileDiffList(inode.asFile(), out);
    } else if (inode.isDirectory()) {
      serializeDirDiffList(inode.asDirectory(), refList, out);
    }
    ++i;
    if (i % FSImageFormatProtobuf.Saver.CHECK_CANCEL_INTERVAL == 0) {
      context.checkCancelled();
    }
  }
  parent.commitSection(headers,
      FSImageFormatProtobuf.SectionName.SNAPSHOT_DIFF);
}
Example #2
Source File: FSImageFormatPBSnapshot.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * save all the snapshot diff to fsimage
 */
public void serializeSnapshotDiffSection(OutputStream out)
    throws IOException {
  INodeMap inodesMap = fsn.getFSDirectory().getINodeMap();
  final List<INodeReference> refList = parent.getSaverContext()
      .getRefList();
  int i = 0;
  Iterator<INodeWithAdditionalFields> iter = inodesMap.getMapIterator();
  while (iter.hasNext()) {
    INodeWithAdditionalFields inode = iter.next();
    if (inode.isFile()) {
      serializeFileDiffList(inode.asFile(), out);
    } else if (inode.isDirectory()) {
      serializeDirDiffList(inode.asDirectory(), refList, out);
    }
    ++i;
    if (i % FSImageFormatProtobuf.Saver.CHECK_CANCEL_INTERVAL == 0) {
      context.checkCancelled();
    }
  }
  parent.commitSection(headers,
      FSImageFormatProtobuf.SectionName.SNAPSHOT_DIFF);
}
Example #3
Source File: TestHadoopNNAWithStreamEngine.java    From NNAnalytics with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new HadoopWebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "false");
  conf.set("nna.historical", "false");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #4
Source File: TestHadoopNNAOperations.java    From NNAnalytics with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new HadoopWebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "false");
  conf.set("nna.historical", "false");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #5
Source File: TestNNAWithStreamEngine.java    From NNAnalytics with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new WebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "false");
  conf.set("nna.historical", "false");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #6
Source File: TestLdapAuth.java    From NNAnalytics with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new WebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "false");
  conf.set("nna.historical", "false");
  conf.set("nna.localonly.users", "hdfs:hdfs,hdfsW:hdfsW,hdfsR:hdfsR,testEmpty:");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #7
Source File: TestHadoopNNAOperations.java    From NNAnalytics with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new HadoopWebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "false");
  conf.set("nna.historical", "false");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #8
Source File: TestHadoopNNAWithStreamEngine.java    From NNAnalytics with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new HadoopWebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "false");
  conf.set("nna.historical", "false");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  conf.set("nna.query.engine.impl", JavaStreamQueryEngine.class.getCanonicalName());
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #9
Source File: GSetSeperatorWrapper.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Override
public INodeWithAdditionalFields put(INodeWithAdditionalFields element) {
  if (element.isFile()) {
    return fileSet.put(element, element);
  }
  return dirSet.put(element, element);
}
Example #10
Source File: GSetSeperatorWrapper.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Override
public INodeWithAdditionalFields remove(INode key) {
  INodeWithAdditionalFields removed;
  removed = fileSet.remove(key);
  if (removed != null) {
    return removed;
  }
  return dirSet.remove(key);
}
Example #11
Source File: GSetSeperatorWrapper.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Override
public INodeWithAdditionalFields get(INode key) {
  INodeWithAdditionalFields val;
  val = fileSet.get(key);
  if (val != null) {
    return val;
  }
  return dirSet.get(key);
}
Example #12
Source File: GSetSeperatorWrapper.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Override
public INodeWithAdditionalFields put(INodeWithAdditionalFields element) {
  if (element.isFile()) {
    return fileSet.put(element, element);
  }
  return dirSet.put(element, element);
}
Example #13
Source File: GSetSeperatorWrapper.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Override
public INodeWithAdditionalFields remove(INode key) {
  INodeWithAdditionalFields removed;
  removed = fileSet.remove(key);
  if (removed != null) {
    return removed;
  }
  return dirSet.remove(key);
}
Example #14
Source File: TestHadoopNNAAuthorization.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new HadoopWebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "true");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #15
Source File: TestINodeTransfer.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Test
public void filterAll() {
  long start = System.currentTimeMillis();
  List<INodeWithAdditionalFields> allNodes =
      StreamSupport.stream(gset.spliterator(), true).collect(Collectors.toList());
  long end = System.currentTimeMillis();
  System.out.println("Took " + (end - start) + " ms.");
  assertThat(allNodes.size(), is(gset.size()));
  assertThat(allNodes.size(), is(GSetGenerator.FILES_MADE + GSetGenerator.DIRS_MADE));
}
Example #16
Source File: TestINodeTransfer.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Test
public void filterFiles() {
  List<INodeWithAdditionalFields> allFiles =
      StreamSupport.stream(gset.spliterator(), true)
          .filter(INode::isFile)
          .collect(Collectors.toList());
  assertThat(allFiles.size(), is(GSetGenerator.FILES_MADE));
}
Example #17
Source File: TestINodeTransfer.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Test
public void filterDirs() {
  List<INodeWithAdditionalFields> allFiles =
      StreamSupport.stream(gset.spliterator(), true)
          .filter(INode::isDirectory)
          .collect(Collectors.toList());
  assertThat(allFiles.size(), is(GSetGenerator.DIRS_MADE));
}
Example #18
Source File: TestNoHistorical.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new WebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "false");
  conf.set("nna.historical", "false");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #19
Source File: BenchmarkGSetFiltering.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Benchmark
public void benchmarkConcurrentJavaFiltering() {
  ConcurrentHashMapINodeCollection concurrent = new ConcurrentHashMapINodeCollection();
  Map<INode, INodeWithAdditionalFields> files = concurrent.filterFiles(gset);
  Map<INode, INodeWithAdditionalFields> dirs = concurrent.filterDirs(gset);
  MemoryProfiler.keepReference(files);
  MemoryProfiler.keepReference(dirs);
}
Example #20
Source File: BenchmarkGSetFiltering.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Benchmark
public void benchmarkNonBlockingHashMapFiltering() {
  NonBlockingHashMapINodeCollection concurrent = new NonBlockingHashMapINodeCollection();
  Map<INode, INodeWithAdditionalFields> files = concurrent.filterFiles(gset);
  Map<INode, INodeWithAdditionalFields> dirs = concurrent.filterDirs(gset);
  MemoryProfiler.keepReference(files);
  MemoryProfiler.keepReference(dirs);
}
Example #21
Source File: BenchmarkGSetFiltering.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Benchmark
public void benchmarkEclipseConcurrentFiltering() {
  EclipseINodeCollection concurrent = new EclipseINodeCollection();
  Map<INode, INodeWithAdditionalFields> files = concurrent.filterFiles(gset);
  Map<INode, INodeWithAdditionalFields> dirs = concurrent.filterDirs(gset);
  MemoryProfiler.keepReference(files);
  MemoryProfiler.keepReference(dirs);
}
Example #22
Source File: TestWebServerMainAuthorization.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new WebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "true");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #23
Source File: TestWebServerMainOperations.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new WebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "false");
  conf.set("nna.historical", "false");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #24
Source File: TestHadoopNNAAuthorization.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  GSetGenerator gSetGenerator = new GSetGenerator();
  gSetGenerator.clear();
  GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
  nna = new HadoopWebServerMain();
  ApplicationConfiguration conf = new ApplicationConfiguration();
  conf.set("ldap.enable", "false");
  conf.set("authorization.enable", "true");
  conf.set("nna.base.dir", MiniDFSCluster.getBaseDirectory());
  conf.set("nna.web.base.dir", "src/main/resources/webapps/nna");
  nna.init(conf, gset);
  hostPort = new HttpHost("localhost", 4567);
}
Example #25
Source File: HadoopWebServerMain.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
/**
 * Main method of initializing an NNA instance. Web server will start first. Then INodes will
 * load.
 *
 * @param conf the application configuration
 * @param inodes preloaded inodes (used by tests)
 * @param preloadedHadoopConf preloaded hadoop configuration (used by tests)
 * @throws Exception errors here will crash NNA
 */
@VisibleForTesting
public void init(
    ApplicationConfiguration conf,
    GSet<INode, INodeWithAdditionalFields> inodes,
    Configuration preloadedHadoopConf)
    throws Exception {
  // Initialize classes needed for NNA.
  SecurityContext securityContext = loadSecurityContext(conf);
  UsageMetrics usageMetrics = new UsageMetrics();

  // Start NameNodeAnalytics HTTP server.
  nnaHttpServer =
      new NameNodeAnalyticsHttpServer(
          preloadedHadoopConf,
          conf,
          getHttpServerBindAddress(conf),
          securityContext,
          nameNodeLoader,
          hsqlDriver,
          usageMetrics,
          runningQueries,
          runningOperations,
          internalService,
          operationService,
          queryLock,
          savingNamespace,
          cancelRequest);
  nnaHttpServer.start();

  // Bootstrap classes.
  nameNodeLoader.load(inodes, preloadedHadoopConf, conf);
  nameNodeLoader.initHistoryRecorder(hsqlDriver, conf, conf.getHistoricalEnabled());
  nameNodeLoader.initReloadThreads(internalService, conf);
}
Example #26
Source File: GSetSeperatorWrapper.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Override
public INodeWithAdditionalFields get(INode key) {
  INodeWithAdditionalFields val;
  val = fileSet.get(key);
  if (val != null) {
    return val;
  }
  return dirSet.get(key);
}
Example #27
Source File: GSetSeperatorWrapper.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Override
public INodeWithAdditionalFields put(INodeWithAdditionalFields element) {
  if (element.isFile()) {
    return fileSet.put(element, element);
  }
  return dirSet.put(element, element);
}
Example #28
Source File: GSetSeperatorWrapper.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Override
public INodeWithAdditionalFields remove(INode key) {
  INodeWithAdditionalFields removed;
  removed = fileSet.remove(key);
  if (removed != null) {
    return removed;
  }
  return dirSet.remove(key);
}
Example #29
Source File: GSetSeperatorWrapper.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Override
public INodeWithAdditionalFields get(INode key) {
  INodeWithAdditionalFields val;
  val = fileSet.get(key);
  if (val != null) {
    return val;
  }
  return dirSet.get(key);
}
Example #30
Source File: GSetSeperatorWrapper.java    From NNAnalytics with Apache License 2.0 5 votes vote down vote up 
@Override
public INodeWithAdditionalFields get(INode key) {
  INodeWithAdditionalFields val;
  val = fileSet.get(key);
  if (val != null) {
    return val;
  }
  return dirSet.get(key);
}