org.apache.hadoop.hive.metastore.IHMSHandler Java Examples

private TServer thriftServer() throws IOException,
        TTransportException,
        MetaException,
        InvocationTargetException,
        NoSuchMethodException,
        IllegalAccessException,
        NoSuchFieldException {
  final TServerSocketKeepAlive socket = new TServerSocketKeepAlive(new TServerSocket(0));
  this.hiveConf = hiveConf(new Configuration(), socket.getServerSocket().getLocalPort());
  HiveMetaStore.HMSHandler baseHandler = new HiveMetaStore.HMSHandler("new db based metaserver", hiveConf);
  IHMSHandler handler = RetryingHMSHandler.getProxy(hiveConf, baseHandler, true);
  final TTransportFactory transportFactory = new TTransportFactory();
  final TSetIpAddressProcessor<IHMSHandler> processor = new TSetIpAddressProcessor<>(handler);

  TThreadPoolServer.Args args = new TThreadPoolServer.Args(socket)
          .processor(processor)
          .transportFactory(transportFactory)
          .protocolFactory(new TBinaryProtocol.Factory())
          .minWorkerThreads(3)
          .maxWorkerThreads(5);

  return new TThreadPoolServer(args);
}
 

@Override
public TProcessor getProcessor(TTransport transport) {
  try {
    if (transport instanceof TSocket) {
      Socket socket = ((TSocket) transport).getSocket();
      log.debug("Received a connection from ip: {}", socket.getInetAddress().getHostAddress());
    }
    CloseableIHMSHandler baseHandler = federatedHMSHandlerFactory.create();
    IHMSHandler handler = newRetryingHMSHandler(ExceptionWrappingHMSHandler.newProxyInstance(baseHandler), hiveConf,
        false);
    transportMonitor.monitor(transport, baseHandler);
    return new TSetIpAddressProcessor<>(handler);
  } catch (MetaException | ReflectiveOperationException | RuntimeException e) {
    throw new RuntimeException("Error creating TProcessor", e);
  }
}
 

private TServer newThriftServer(TServerSocket socket, HiveConf conf) throws Exception {
  HiveConf serverConf = new HiveConf(conf);
  serverConf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, "jdbc:derby:" + getDerbyPath() + ";create=true");
  HiveMetaStore.HMSHandler baseHandler = new HiveMetaStore.HMSHandler("new db based metaserver", serverConf);
  IHMSHandler handler = RetryingHMSHandler.getProxy(serverConf, baseHandler, false);

  TThreadPoolServer.Args args = new TThreadPoolServer.Args(socket)
      .processor(new TSetIpAddressProcessor<>(handler))
      .transportFactory(new TTransportFactory())
      .protocolFactory(new TBinaryProtocol.Factory())
      .minWorkerThreads(3)
      .maxWorkerThreads(5);

  return new TThreadPoolServer(args);
}
 

MetastoreCacheInitializer(IHMSHandler hmsHandler, Configuration conf) {
  this.hmsHandler = hmsHandler;
  this.maxPartitionsPerCall = conf.getInt(
          ServiceConstants.ServerConfig
                  .SENTRY_HDFS_SYNC_METASTORE_CACHE_MAX_PART_PER_RPC,
          ServiceConstants.ServerConfig
                  .SENTRY_HDFS_SYNC_METASTORE_CACHE_MAX_PART_PER_RPC_DEFAULT);
  this.maxTablesPerCall = conf.getInt(
          ServiceConstants.ServerConfig
                  .SENTRY_HDFS_SYNC_METASTORE_CACHE_MAX_TABLES_PER_RPC,
          ServiceConstants.ServerConfig
                  .SENTRY_HDFS_SYNC_METASTORE_CACHE_MAX_TABLES_PER_RPC_DEFAULT);
  threadPool = Executors.newFixedThreadPool(conf.getInt(
          ServiceConstants.ServerConfig
                  .SENTRY_HDFS_SYNC_METASTORE_CACHE_INIT_THREADS,
          ServiceConstants.ServerConfig
                  .SENTRY_HDFS_SYNC_METASTORE_CACHE_INIT_THREADS_DEFAULT));
  maxRetries = conf.getInt(
          ServiceConstants.ServerConfig
                  .SENTRY_HDFS_SYNC_METASTORE_CACHE_RETRY_MAX_NUM,
          ServiceConstants.ServerConfig
                  .SENTRY_HDFS_SYNC_METASTORE_CACHE_RETRY_MAX_NUM_DEFAULT);
  waitDurationMillis = conf.getInt(
          ServiceConstants.ServerConfig
                  .SENTRY_HDFS_SYNC_METASTORE_CACHE_RETRY_WAIT_DURAION_IN_MILLIS,
          ServiceConstants.ServerConfig
                  .SENTRY_HDFS_SYNC_METASTORE_CACHE_RETRY_WAIT_DURAION_IN_MILLIS_DEFAULT);
  failOnRetry = conf.getBoolean(
          ServiceConstants.ServerConfig
                  .SENTRY_HDFS_SYNC_METASTORE_CACHE_FAIL_ON_PARTIAL_UPDATE,
          ServiceConstants.ServerConfig
                  .SENTRY_HDFS_SYNC_METASTORE_CACHE_FAIL_ON_PARTIAL_UPDATE_DEFAULT);
}
 

@Test
public void get_databaseWaggleDanceServerException() throws Exception {
  expectedException.expect(MetaException.class);
  IHMSHandler handler = ExceptionWrappingHMSHandler.newProxyInstance(baseHandler);
  when(baseHandler.get_database("bdp")).thenThrow(new WaggleDanceServerException("waggle waggle!"));
  handler.get_database("bdp");
}
 

@Test
public void get_databasNotAllowedException() throws Exception {
  expectedException.expect(MetaException.class);
  IHMSHandler handler = ExceptionWrappingHMSHandler.newProxyInstance(baseHandler);
  when(baseHandler.get_database("bdp")).thenThrow(new NotAllowedException("waggle waggle!"));
  handler.get_database("bdp");
}
 

@Test
public void get_databaseRunTimeExceptionIsNotWrapped() throws Exception {
  expectedException.expect(RuntimeException.class);
  expectedException.expectMessage("generic non waggle dance exception");
  IHMSHandler handler = ExceptionWrappingHMSHandler.newProxyInstance(baseHandler);
  when(baseHandler.get_database("bdp")).thenThrow(new RuntimeException("generic non waggle dance exception"));
  handler.get_database("bdp");
}
 

@Test
public void get_databaseCheckedExceptionIsNotWrapped() throws Exception {
  expectedException.expect(NoSuchObjectException.class);
  expectedException.expectMessage("Does not exist!");
  IHMSHandler handler = ExceptionWrappingHMSHandler.newProxyInstance(baseHandler);
  when(baseHandler.get_database("bdp")).thenThrow(new NoSuchObjectException("Does not exist!"));
  handler.get_database("bdp");
}
 

public TServer startMetaStore(String forceBindIP, int port,
    HiveConf conf) throws IOException {
  try {
    // Server will create new threads up to max as necessary. After an idle
    // period, it will destory threads to keep the number of threads in the
    // pool to min.
    int minWorkerThreads = conf.getIntVar(HiveConf.ConfVars.METASTORESERVERMINTHREADS);
    int maxWorkerThreads = conf.getIntVar(HiveConf.ConfVars.METASTORESERVERMAXTHREADS);
    boolean tcpKeepAlive = conf.getBoolVar(HiveConf.ConfVars.METASTORE_TCP_KEEP_ALIVE);
    boolean useFramedTransport = conf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_FRAMED_TRANSPORT);

    // don't support SASL yet
    //boolean useSasl = conf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL);

    TServerTransport serverTransport;
    if (forceBindIP != null) {
      InetSocketAddress address = new InetSocketAddress(forceBindIP, port);
      serverTransport = tcpKeepAlive ?
          new TServerSocketKeepAlive(address) : new TServerSocket(address);

    } else {
      serverTransport = tcpKeepAlive ?
          new TServerSocketKeepAlive(port) : new TServerSocket(port);
    }

    TProcessor processor;
    TTransportFactory transFactory;

    IHMSHandler handler = (IHMSHandler) HiveMetaStore
        .newRetryingHMSHandler("new db based metaserver", conf, true);

    if (conf.getBoolVar(HiveConf.ConfVars.METASTORE_EXECUTE_SET_UGI)) {
      transFactory = useFramedTransport ?
          new ChainedTTransportFactory(new TFramedTransport.Factory(),
              new TUGIContainingTransport.Factory())
          : new TUGIContainingTransport.Factory();

      processor = new TUGIBasedProcessor<IHMSHandler>(handler);
      LOG.info("Starting DB backed MetaStore Server with SetUGI enabled");
    } else {
      transFactory = useFramedTransport ?
          new TFramedTransport.Factory() : new TTransportFactory();
      processor = new TSetIpAddressProcessor<IHMSHandler>(handler);
      LOG.info("Starting DB backed MetaStore Server");
    }

    TThreadPoolServer.Args args = new TThreadPoolServer.Args(serverTransport)
        .processor(processor)
        .transportFactory(transFactory)
        .protocolFactory(new TBinaryProtocol.Factory())
        .minWorkerThreads(minWorkerThreads)
        .maxWorkerThreads(maxWorkerThreads);

    final TServer tServer = new TThreadPoolServer(args);
    executorService.submit(new Runnable() {
      @Override
      public void run() {
        tServer.serve();
      }
    });
    return tServer;
  } catch (Throwable x) {
    throw new IOException(x);
  }
}
 

@Test
public void testExceptionInTask() throws Exception {
  //Set up mocks: db1.tb1, with tb1 returning a wrong dbname (db2)
  Database db1 = Mockito.mock(Database.class);
  Mockito.when(db1.getName()).thenReturn("db1");
  Mockito.when(db1.getLocationUri()).thenReturn("hdfs:///db1");

  Table tab1 = Mockito.mock(Table.class);
  //Return a wrong db name, so that this triggers an exception
  Mockito.when(tab1.getDbName()).thenReturn("db2");
  Mockito.when(tab1.getTableName()).thenReturn("tab1");

  IHMSHandler hmsHandler = Mockito.mock(IHMSHandler.class);
  Mockito.when(hmsHandler.get_all_databases()).thenReturn(Lists
      .newArrayList("db1"));
  Mockito.when(hmsHandler.get_database("db1")).thenReturn(db1);
  Mockito.when(hmsHandler.get_table_objects_by_name("db1",
      Lists.newArrayList("tab1")))
      .thenReturn(Lists.newArrayList(tab1));
  Mockito.when(hmsHandler.get_all_tables("db1")).thenReturn(Lists
      .newArrayList("tab1"));

  Configuration conf = new Configuration();
  conf.setInt(ServiceConstants.ServerConfig
      .SENTRY_HDFS_SYNC_METASTORE_CACHE_MAX_PART_PER_RPC, 1);
  conf.setInt(ServiceConstants.ServerConfig
      .SENTRY_HDFS_SYNC_METASTORE_CACHE_MAX_TABLES_PER_RPC, 1);
  conf.setInt(ServiceConstants.ServerConfig
      .SENTRY_HDFS_SYNC_METASTORE_CACHE_INIT_THREADS, 1);
  conf.setInt(ServiceConstants.ServerConfig
          .SENTRY_HDFS_SYNC_METASTORE_CACHE_RETRY_MAX_NUM, 2);

  try {
    MetastoreCacheInitializer cacheInitializer = new
        MetastoreCacheInitializer(hmsHandler, conf);
    cacheInitializer.createInitialUpdate();
    Assert.fail("Expected cacheInitializer to fail");
  } catch (Exception e) {
    Assert.assertTrue(e instanceof RuntimeException);
  }

}
 

@Test
public void testInitializer() throws Exception {

  Database db1 = Mockito.mock(Database.class);
  Mockito.when(db1.getName()).thenReturn("db1");
  Mockito.when(db1.getLocationUri()).thenReturn("hdfs:///db1");
  Database db2 = Mockito.mock(Database.class);
  Mockito.when(db2.getName()).thenReturn("db2");
  Mockito.when(db2.getLocationUri()).thenReturn("hdfs:///db2");
  Database db3 = Mockito.mock(Database.class);
  Mockito.when(db3.getName()).thenReturn("db3");
  Mockito.when(db3.getLocationUri()).thenReturn("hdfs:///db3");

  Table tab21 = Mockito.mock(Table.class);
  Mockito.when(tab21.getDbName()).thenReturn("db2");
  Mockito.when(tab21.getTableName()).thenReturn("tab21");
  StorageDescriptor sd21 = Mockito.mock(StorageDescriptor.class);
  Mockito.when(sd21.getLocation()).thenReturn("hdfs:///db2/tab21");
  Mockito.when(tab21.getSd()).thenReturn(sd21);

  Table tab31 = Mockito.mock(Table.class);
  Mockito.when(tab31.getDbName()).thenReturn("db3");
  Mockito.when(tab31.getTableName()).thenReturn("tab31");
  StorageDescriptor sd31 = Mockito.mock(StorageDescriptor.class);
  Mockito.when(sd31.getLocation()).thenReturn("hdfs:///db3/tab31");
  Mockito.when(tab31.getSd()).thenReturn(sd31);

  Partition part311 = Mockito.mock(Partition.class);
  StorageDescriptor sd311 = Mockito.mock(StorageDescriptor.class);
  Mockito.when(sd311.getLocation()).thenReturn("hdfs:///db3/tab31/part311");
  Mockito.when(part311.getSd()).thenReturn(sd311);

  Partition part312 = Mockito.mock(Partition.class);
  StorageDescriptor sd312 = Mockito.mock(StorageDescriptor.class);
  Mockito.when(sd312.getLocation()).thenReturn("hdfs:///db3/tab31/part312");
  Mockito.when(part312.getSd()).thenReturn(sd312);

  IHMSHandler hmsHandler = Mockito.mock(IHMSHandler.class);
  Mockito.when(hmsHandler.get_all_databases()).thenReturn(Lists
          .newArrayList("db1", "db2", "db3"));
  Mockito.when(hmsHandler.get_database("db1")).thenReturn(db1);
  Mockito.when(hmsHandler.get_all_tables("db1")).thenReturn(new
          ArrayList<String>());

  Mockito.when(hmsHandler.get_database("db2")).thenReturn(db2);
  Mockito.when(hmsHandler.get_all_tables("db2")).thenReturn(Lists
          .newArrayList("tab21"));
  Mockito.when(hmsHandler.get_table_objects_by_name("db2",
          Lists.newArrayList("tab21")))
          .thenReturn(Lists.newArrayList(tab21));
  Mockito.when(hmsHandler.get_partition_names("db2", "tab21", (short) -1))
          .thenReturn(new ArrayList<String>());

  Mockito.when(hmsHandler.get_database("db3")).thenReturn(db3);
  Mockito.when(hmsHandler.get_all_tables("db3")).thenReturn(Lists
          .newArrayList("tab31"));
  Mockito.when(hmsHandler.get_table_objects_by_name("db3",
          Lists.newArrayList("tab31")))
          .thenReturn(Lists.newArrayList(tab31));
  Mockito.when(hmsHandler.get_partition_names("db3", "tab31", (short) -1))
          .thenReturn(Lists.newArrayList("part311", "part312"));

  Mockito.when(hmsHandler.get_partitions_by_names("db3", "tab31",
          Lists.newArrayList("part311")))
          .thenReturn(Lists.newArrayList(part311));
  Mockito.when(hmsHandler.get_partitions_by_names("db3", "tab31",
          Lists.newArrayList("part312")))
          .thenReturn(Lists.newArrayList(part312));

  Configuration conf = new Configuration();
  conf.setInt(ServiceConstants.ServerConfig
          .SENTRY_HDFS_SYNC_METASTORE_CACHE_MAX_PART_PER_RPC, 1);
  conf.setInt(ServiceConstants.ServerConfig
          .SENTRY_HDFS_SYNC_METASTORE_CACHE_MAX_TABLES_PER_RPC, 1);
  conf.setInt(ServiceConstants.ServerConfig
          .SENTRY_HDFS_SYNC_METASTORE_CACHE_INIT_THREADS, 1);

  MetastoreCacheInitializer cacheInitializer = new
          MetastoreCacheInitializer(hmsHandler, conf);
  UpdateableAuthzPaths update = cacheInitializer.createInitialUpdate();

  Assert.assertEquals(new HashSet<String>(Arrays.asList("db1")), update.findAuthzObjectExactMatches(new
          String[]{"db1"}));
  Assert.assertEquals(new HashSet<String>(Arrays.asList("db2")), update.findAuthzObjectExactMatches(new
          String[]{"db2"}));
  Assert.assertEquals(new HashSet<String>(Arrays.asList("db2.tab21")), update.findAuthzObjectExactMatches(new
          String[]{"db2", "tab21"}));
  Assert.assertEquals(new HashSet<String>(Arrays.asList("db3")), update.findAuthzObjectExactMatches(new
          String[]{"db3"}));
  Assert.assertEquals(new HashSet<String>(Arrays.asList("db3.tab31")), update.findAuthzObjectExactMatches(new
          String[]{"db3", "tab31"}));
  Assert.assertEquals(new HashSet<String>(Arrays.asList("db3.tab31")), update.findAuthzObjectExactMatches(new
          String[]{"db3", "tab31", "part311"}));
  Assert.assertEquals(new HashSet<String>(Arrays.asList("db3.tab31")), update.findAuthzObjectExactMatches(new
          String[]{"db3", "tab31", "part312"}));
  cacheInitializer.close();

}
 

public TServer startMetaStore(String forceBindIP, int port, HiveConf conf) throws IOException {
  try {
    // Server will create new threads up to max as necessary. After an idle
    // period, it will destory threads to keep the number of threads in the
    // pool to min.
    int minWorkerThreads = conf.getIntVar(HiveConf.ConfVars.METASTORESERVERMINTHREADS);
    int maxWorkerThreads = conf.getIntVar(HiveConf.ConfVars.METASTORESERVERMAXTHREADS);
    boolean tcpKeepAlive = conf.getBoolVar(HiveConf.ConfVars.METASTORE_TCP_KEEP_ALIVE);
    boolean useFramedTransport = conf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_FRAMED_TRANSPORT);

    // don't support SASL yet
    // boolean useSasl = conf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL);

    TServerTransport serverTransport;
    if (forceBindIP != null) {
      InetSocketAddress address = new InetSocketAddress(forceBindIP, port);
      serverTransport = tcpKeepAlive ? new TServerSocketKeepAlive(address) : new TServerSocket(address);

    } else {
      serverTransport = tcpKeepAlive ? new TServerSocketKeepAlive(port) : new TServerSocket(port);
    }

    TProcessor processor;
    TTransportFactory transFactory;

    HiveMetaStore.HMSHandler baseHandler = new HiveMetaStore.HMSHandler("new db based metaserver", conf, false);
    IHMSHandler handler = RetryingHMSHandler.getProxy(conf, baseHandler, true);

    if (conf.getBoolVar(HiveConf.ConfVars.METASTORE_EXECUTE_SET_UGI)) {
      transFactory = useFramedTransport
          ? new ChainedTTransportFactory(new TFramedTransport.Factory(), new TUGIContainingTransport.Factory())
          : new TUGIContainingTransport.Factory();

      processor = new TUGIBasedProcessor<>(handler);
      LOG.info("Starting DB backed MetaStore Server with SetUGI enabled");
    } else {
      transFactory = useFramedTransport ? new TFramedTransport.Factory() : new TTransportFactory();
      processor = new TSetIpAddressProcessor<>(handler);
      LOG.info("Starting DB backed MetaStore Server");
    }

    TThreadPoolServer.Args args = new TThreadPoolServer.Args(serverTransport).processor(processor)
        .transportFactory(transFactory).protocolFactory(new TBinaryProtocol.Factory())
        .minWorkerThreads(minWorkerThreads).maxWorkerThreads(maxWorkerThreads);

    final TServer tServer = new TThreadPoolServer(args);
    executorService.submit(tServer::serve);
    return tServer;
  } catch (Throwable x) {
    throw new IOException(x);
  }
}
 

@Test
public void get_databaseNoExceptions() throws Exception {
  IHMSHandler handler = ExceptionWrappingHMSHandler.newProxyInstance(baseHandler);
  handler.get_database("bdp");
  verify(baseHandler).get_database("bdp");
}
 

private IHMSHandler newRetryingHMSHandler(IHMSHandler baseHandler, HiveConf hiveConf, boolean local)
  throws MetaException {
  return RetryingHMSHandler.getProxy(hiveConf, baseHandler, local);
}
 

public ExceptionWrappingHMSHandler(IHMSHandler baseHandler) {
  this.baseHandler = baseHandler;
}
 

public static IHMSHandler newProxyInstance(IHMSHandler baseHandler) {
  return (IHMSHandler) Proxy.newProxyInstance(ExceptionWrappingHMSHandler.class.getClassLoader(),
      new Class[] { IHMSHandler.class }, new ExceptionWrappingHMSHandler(baseHandler));
}
 

public IHMSHandler getMetastoreHandler() {
    return metastoreHandler;
}