org.apache.hadoop.hive.shims.ShimLoader Java Examples

private TServerSocket createServerSocket(boolean useSSL, int port) throws IOException, TTransportException {
  TServerSocket serverSocket = null;
  // enable SSL support for HMS
  List<String> sslVersionBlacklist = new ArrayList<>(Arrays.asList(hiveConf.getVar(ConfVars.HIVE_SSL_PROTOCOL_BLACKLIST).split(",")));
  if (!useSSL) {
    serverSocket = HiveAuthUtils.getServerSocket(null, port);
  } else {
    String keyStorePath = hiveConf.getVar(ConfVars.HIVE_METASTORE_SSL_KEYSTORE_PATH).trim();
    if (keyStorePath.isEmpty()) {
      throw new IllegalArgumentException(
          ConfVars.HIVE_METASTORE_SSL_KEYSTORE_PASSWORD.varname + " Not configured for SSL connection");
    }
    String keyStorePassword = ShimLoader
        .getHadoopShims()
        .getPassword(hiveConf, HiveConf.ConfVars.HIVE_METASTORE_SSL_KEYSTORE_PASSWORD.varname);
    serverSocket = HiveAuthUtils.getServerSSLSocket(null, port, keyStorePath, keyStorePassword, sslVersionBlacklist);
  }
  return serverSocket;
}
 

public static List<InputSplit> getSplits(JobContext jobContext)
        throws IOException, InterruptedException {
    List<OrcSplit> splits =
            OrcInputFormat.generateSplitsInfo(ShimLoader.getHadoopShims()
                    .getConfiguration(jobContext));
    List<InputSplit> result = new ArrayList<InputSplit>(splits.size());
    // Filter Out Splits based on paths...
    for(OrcSplit split: splits) {

        result.add(new OrcNewSplit(split));
    }



    return result;
}
 

public void invokeOutputCommitterForLocalMode(Job job) throws IOException {
  if (ConfigurationHelper.isLocalJobTracker(job.getConfiguration()) && isHadoop1()) {
    // HCatalog 0.11- do have special class HCatHadoopShims, however this
    // class got merged into Hive Shim layer in 0.12+. Following method will
    // try to find correct implementation via reflection.
    HadoopShims shims = ShimLoader.getHadoopShims();
    HCatHadoopShims hcatShims = shims.getHCatShim();
    try {
      hcatShims.commitJob(new HCatOutputFormat(), job);
    } catch (Exception e) {
      throw new RuntimeException("Can't explicitly commit job", e);
    }
  }
}
 

@Override
public void run(ApplicationArguments args) throws Exception {
  if (isRunning()) {
    throw new RuntimeException("Can't run more than one instance");
  }

  final boolean isCliVerbose = waggleDanceConfiguration.isVerbose();

  try {
    String msg = "Starting WaggleDance on port " + waggleDanceConfiguration.getPort();
    LOG.info(msg);
    if (waggleDanceConfiguration.isVerbose()) {
      System.err.println(msg);
    }

    // Add shutdown hook.
    Runtime.getRuntime().addShutdownHook(new Thread(() -> {
      String shutdownMsg = "Shutting down WaggleDance.";
      LOG.info(shutdownMsg);
      if (isCliVerbose) {
        System.err.println(shutdownMsg);
      }
    }));

    AtomicBoolean startedServing = new AtomicBoolean();
    startWaggleDance(ShimLoader.getHadoopThriftAuthBridge(), startLock, startCondition, startedServing);
  } catch (Throwable t) {
    // Catch the exception, log it and rethrow it.
    LOG.error("WaggleDance Thrift Server threw an exception...", t);
    throw new Exception(t);
  }
}
 

/**
 * Create a generic Hive RecordReader than can iterate over all chunks in a CombinedFileSplit.
 */
@Override
public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException {
  if (!(split instanceof CombineHiveInputSplit)) {
    return super.getRecordReader(split, job, reporter);
  }

  CombineHiveInputSplit hsplit = (CombineHiveInputSplit) split;

  String inputFormatClassName = null;
  Class<?> inputFormatClass;
  try {
    inputFormatClassName = hsplit.inputFormatClassName();
    inputFormatClass = job.getClassByName(inputFormatClassName);
  } catch (Exception e) {
    throw new IOException("cannot find class " + inputFormatClassName);
  }

  pushProjectionsAndFilters(job, inputFormatClass, hsplit.getPath(0));

  if (inputFormatClass.getName().equals(HoodieParquetInputFormat.class.getName())) {
    return ShimLoader.getHadoopShims().getCombineFileInputFormat().getRecordReader(job, (CombineFileSplit) split,
        reporter, CombineHiveRecordReader.class);
  } else if (inputFormatClass.getName().equals(HoodieParquetRealtimeInputFormat.class.getName())) {
    HoodieCombineFileInputFormatShim shims = new HoodieCombineFileInputFormatShim();
    IOContextMap.get(job).setInputPath(((CombineHiveInputSplit) split).getPath(0));
    return shims.getRecordReader(job, ((CombineHiveInputSplit) split).getInputSplitShim(),
        reporter, CombineHiveRecordReader.class);
  } else {
    throw new HoodieException("Unexpected input format : " + inputFormatClassName);
  }
}
 

private void startMetastore() throws Exception {
  Callable<Void> metastoreService = new Callable<Void>() {
    public Void call() throws Exception {
      try {
        HiveMetaStore.startMetaStore(getMetastorePort(conf),
            ShimLoader.getHadoopThriftAuthBridge(), conf);
      } catch (Throwable e) {
        throw new Exception("Error starting metastore", e);
      }
      return null;
    }
  };
  metaStoreExecutor.submit(metastoreService);
}
 

public static List<InputSplit> getSplits(JobContext jobContext)
        throws IOException, InterruptedException {
    Configuration conf = ShimLoader.getHadoopShims().getConfiguration(jobContext);
    List<OrcSplit> splits = OrcInputFormat.generateSplitsInfo(conf, createContext(conf, -1));
    List<InputSplit> result = new ArrayList<InputSplit>(splits.size());
    for(OrcSplit split: splits) {
        result.add(new OrcNewSplit(split));
    }
    return result;
}
 

public CombineHiveInputSplit() throws IOException {
  this(ShimLoader.getHadoopShims().getCombineFileInputFormat().getInputSplitShim());
}
 

public HoodieCombineRealtimeHiveSplit() throws IOException {
  super(ShimLoader.getHadoopShims().getCombineFileInputFormat().getInputSplitShim());
}
 

public void validateInput(JobConf job) throws IOException {
  ShimLoader.getHadoopShims().inputFormatValidateInput(format, job);
}
 

@Override
public void checkOutputSpecs(final FileSystem ignored, final JobConf job) throws IOException {
  realOutputFormat.checkOutputSpecs(ShimLoader.getHadoopShims().getHCatShim().createJobContext(job, null));
}
 

protected static void startHiveMetaStore() throws Exception {
  final int port = MetaStoreUtils.findFreePort();

  hiveConf.set(METASTOREURIS.varname, "thrift://localhost:" + port);

  MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), hiveConf);
}