Java Code Examples for org.mortbay.log.Log#warn()

/**
 * Notify the URL just once. Use best effort.
 */
protected boolean notifyURLOnce() {
  boolean success = false;
  try {
    Log.info("Job end notification trying " + urlToNotify);
    HttpURLConnection conn =
      (HttpURLConnection) urlToNotify.openConnection(proxyToUse);
    conn.setConnectTimeout(timeout);
    conn.setReadTimeout(timeout);
    conn.setAllowUserInteraction(false);
    if(conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
      Log.warn("Job end notification to " + urlToNotify +" failed with code: "
      + conn.getResponseCode() + " and message \"" + conn.getResponseMessage()
      +"\"");
    }
    else {
      success = true;
      Log.info("Job end notification to " + urlToNotify + " succeeded");
    }
  } catch(IOException ioe) {
    Log.warn("Job end notification to " + urlToNotify + " failed", ioe);
  }
  return success;
}
 

/**
 * Notify the URL just once. Use best effort.
 */
protected boolean notifyURLOnce() {
  boolean success = false;
  try {
    Log.info("Job end notification trying " + urlToNotify);
    HttpURLConnection conn =
      (HttpURLConnection) urlToNotify.openConnection(proxyToUse);
    conn.setConnectTimeout(timeout);
    conn.setReadTimeout(timeout);
    conn.setAllowUserInteraction(false);
    if(conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
      Log.warn("Job end notification to " + urlToNotify +" failed with code: "
      + conn.getResponseCode() + " and message \"" + conn.getResponseMessage()
      +"\"");
    }
    else {
      success = true;
      Log.info("Job end notification to " + urlToNotify + " succeeded");
    }
  } catch(IOException ioe) {
    Log.warn("Job end notification to " + urlToNotify + " failed", ioe);
  }
  return success;
}
 

/**
 * Builds the reverseKey to fetch the pcaps in the reverse traffic
 * (destination to source).
 * 
 * @param key
 *          indicates hbase rowKey (partial or full) in the format
 *          "srcAddr-dstAddr-protocol-srcPort-dstPort-fragment"
 * @return String indicates the key in the format
 *         "dstAddr-srcAddr-protocol-dstPort-srcPort"
 */
public static String reverseKey(String key) {
  Assert.hasText(key, "key must not be null or empty");
  String delimeter = HBaseConfigConstants.PCAP_KEY_DELIMETER;
  String regex = "\\" + delimeter;
  StringBuffer sb = new StringBuffer();
  try {
    String[] tokens = key.split(regex);
    Assert
        .isTrue(
            (tokens.length == 5 || tokens.length == 6 || tokens.length == 7),
            "key is not in the format : 'srcAddr-dstAddr-protocol-srcPort-dstPort-{ipId-fragment identifier}'");
    sb.append(tokens[1]).append(delimeter).append(tokens[0])
        .append(delimeter).append(tokens[2]).append(delimeter)
        .append(tokens[4]).append(delimeter).append(tokens[3]);
  } catch (Exception e) {
    Log.warn("Failed to reverse the key. Reverse scan won't be performed.", e);
  }
  return sb.toString();
}
 

/**
 * Builds the reverseKey to fetch the pcaps in the reverse traffic
 * (destination to source).
 * 
 * @param key
 *          indicates hbase rowKey (partial or full) in the format
 *          "srcAddr-dstAddr-protocol-srcPort-dstPort-fragment"
 * @return String indicates the key in the format
 *         "dstAddr-srcAddr-protocol-dstPort-srcPort"
 */
public static String reverseKey(String key) {
  Assert.hasText(key, "key must not be null or empty");
  String delimeter = HBaseConfigConstants.PCAP_KEY_DELIMETER;
  String regex = "\\" + delimeter;
  StringBuffer sb = new StringBuffer();
  try {
    String[] tokens = key.split(regex);
    Assert
        .isTrue(
            (tokens.length == 5 || tokens.length == 6 || tokens.length == 7),
            "key is not in the format : 'srcAddr-dstAddr-protocol-srcPort-dstPort-{ipId-fragment identifier}'");
    sb.append(tokens[1]).append(delimeter).append(tokens[0])
        .append(delimeter).append(tokens[2]).append(delimeter)
        .append(tokens[4]).append(delimeter).append(tokens[3]);
  } catch (Exception e) {
    Log.warn("Failed to reverse the key. Reverse scan won't be performed.", e);
  }
  return sb.toString();
}
 

@Override
public Row deserialize(byte[] message) throws IOException {
    try {
        JsonNode root = objectMapper.readTree(message);

        Row row = new Row(fieldNames.length);
        for (int i = 0; i < fieldNames.length; i++) {

            //数据源本来的字段名
            String columnName = columnNames[i];

            JsonNode node = root.get(columnName);

            if (node == null) {
                if (failOnMissingField) {
                    throw new IllegalStateException("Failed to find field with name '"
                            + fieldNames[i] + "'.");
                } else {
                    row.setField(i, null);
                }
            } else {
                // Read the value as specified type
                try {
                    Object value = objectMapper.treeToValue(node, fieldTypes[i].getTypeClass());
                    row.setField(i, value);
                }catch (Exception e){
                    Log.warn("Failed to deserialize JSON object.[" + new String(message, "UTF-8") + "]", e);
                    row.setField(i, null);
                }

            }
        }

        return row;
    } catch (Throwable t) {
        Log.warn("Failed to deserialize JSON object.[" + new String(message, "UTF-8") + "]", t);
        return null;
    }
}
 

/**
 * Notify a server of the completion of a submitted job. The user must have
 * configured MRJobConfig.MR_JOB_END_NOTIFICATION_URL
 * @param jobReport JobReport used to read JobId and JobStatus
 * @throws InterruptedException
 */
public void notify(JobReport jobReport)
  throws InterruptedException {
  // Do we need job-end notification?
  if (userUrl == null) {
    Log.info("Job end notification URL not set, skipping.");
    return;
  }

  //Do string replacements for jobId and jobStatus
  if (userUrl.contains(JOB_ID)) {
    userUrl = userUrl.replace(JOB_ID, jobReport.getJobId().toString());
  }
  if (userUrl.contains(JOB_STATUS)) {
    userUrl = userUrl.replace(JOB_STATUS, jobReport.getJobState().toString());
  }

  // Create the URL, ensure sanity
  try {
    urlToNotify = new URL(userUrl);
  } catch (MalformedURLException mue) {
    Log.warn("Job end notification couldn't parse " + userUrl, mue);
    return;
  }

  // Send notification
  boolean success = false;
  while (numTries-- > 0 && !success) {
    Log.info("Job end notification attempts left " + numTries);
    success = notifyURLOnce();
    if (!success) {
      Thread.sleep(waitInterval);
    }
  }
  if (!success) {
    Log.warn("Job end notification failed to notify : " + urlToNotify);
  } else {
    Log.info("Job end notification succeeded for " + jobReport.getJobId());
  }
}
 

@Override
public QuotaCounts cleanSubtree(BlockStoragePolicySuite bsps,
    final int snapshot, int prior, final BlocksMapUpdateInfo collectedBlocks,
    final List<INode> removedINodes) {
  // since WithName node resides in deleted list acting as a snapshot copy,
  // the parameter snapshot must be non-null
  Preconditions.checkArgument(snapshot != Snapshot.CURRENT_STATE_ID);
  // if prior is NO_SNAPSHOT_ID, we need to check snapshot belonging to the
  // previous WithName instance
  if (prior == Snapshot.NO_SNAPSHOT_ID) {
    prior = getPriorSnapshot(this);
  }
  
  if (prior != Snapshot.NO_SNAPSHOT_ID
      && Snapshot.ID_INTEGER_COMPARATOR.compare(snapshot, prior) <= 0) {
    return new QuotaCounts.Builder().build();
  }

  QuotaCounts counts = getReferredINode().cleanSubtree(bsps, snapshot, prior,
      collectedBlocks, removedINodes);
  INodeReference ref = getReferredINode().getParentReference();
  if (ref != null) {
    try {
      ref.addSpaceConsumed(counts.negation(), true);
    } catch (QuotaExceededException e) {
      Log.warn("Should not have QuotaExceededException");
    }
  }
  
  if (snapshot < lastSnapshotId) {
    // for a WithName node, when we compute its quota usage, we only count
    // in all the nodes existing at the time of the corresponding rename op.
    // Thus if we are deleting a snapshot before/at the snapshot associated 
    // with lastSnapshotId, we do not need to update the quota upwards.
    counts = new QuotaCounts.Builder().build();
  }
  return counts;
}
 

/**
 * Notify a server of the completion of a submitted job. The user must have
 * configured MRJobConfig.MR_JOB_END_NOTIFICATION_URL
 * @param jobReport JobReport used to read JobId and JobStatus
 * @throws InterruptedException
 */
public void notify(JobReport jobReport)
  throws InterruptedException {
  // Do we need job-end notification?
  if (userUrl == null) {
    Log.info("Job end notification URL not set, skipping.");
    return;
  }

  //Do string replacements for jobId and jobStatus
  if (userUrl.contains(JOB_ID)) {
    userUrl = userUrl.replace(JOB_ID, jobReport.getJobId().toString());
  }
  if (userUrl.contains(JOB_STATUS)) {
    userUrl = userUrl.replace(JOB_STATUS, jobReport.getJobState().toString());
  }

  // Create the URL, ensure sanity
  try {
    urlToNotify = new URL(userUrl);
  } catch (MalformedURLException mue) {
    Log.warn("Job end notification couldn't parse " + userUrl, mue);
    return;
  }

  // Send notification
  boolean success = false;
  while (numTries-- > 0 && !success) {
    Log.info("Job end notification attempts left " + numTries);
    success = notifyURLOnce();
    if (!success) {
      Thread.sleep(waitInterval);
    }
  }
  if (!success) {
    Log.warn("Job end notification failed to notify : " + urlToNotify);
  } else {
    Log.info("Job end notification succeeded for " + jobReport.getJobId());
  }
}
 

@Override
public QuotaCounts cleanSubtree(BlockStoragePolicySuite bsps,
    final int snapshot, int prior, final BlocksMapUpdateInfo collectedBlocks,
    final List<INode> removedINodes) {
  // since WithName node resides in deleted list acting as a snapshot copy,
  // the parameter snapshot must be non-null
  Preconditions.checkArgument(snapshot != Snapshot.CURRENT_STATE_ID);
  // if prior is NO_SNAPSHOT_ID, we need to check snapshot belonging to the
  // previous WithName instance
  if (prior == Snapshot.NO_SNAPSHOT_ID) {
    prior = getPriorSnapshot(this);
  }
  
  if (prior != Snapshot.NO_SNAPSHOT_ID
      && Snapshot.ID_INTEGER_COMPARATOR.compare(snapshot, prior) <= 0) {
    return new QuotaCounts.Builder().build();
  }

  QuotaCounts counts = getReferredINode().cleanSubtree(bsps, snapshot, prior,
      collectedBlocks, removedINodes);
  INodeReference ref = getReferredINode().getParentReference();
  if (ref != null) {
    try {
      ref.addSpaceConsumed(counts.negation(), true);
    } catch (QuotaExceededException e) {
      Log.warn("Should not have QuotaExceededException");
    }
  }
  
  if (snapshot < lastSnapshotId) {
    // for a WithName node, when we compute its quota usage, we only count
    // in all the nodes existing at the time of the corresponding rename op.
    // Thus if we are deleting a snapshot before/at the snapshot associated 
    // with lastSnapshotId, we do not need to update the quota upwards.
    counts = new QuotaCounts.Builder().build();
  }
  return counts;
}
 

public boolean dispatch(Runnable job)
{
    try
    {
        _executor.execute(job);
        return true;
    }
    catch(RejectedExecutionException e)
    {
        Log.warn(e);
        return false;
    }
}
 

protected static Schema staticSchemaGen(String s) {
    try {
        if (s.equals("")) {
            Log.warn("No Schema present in SchemaTuple generated class");
            return new Schema();
        }
        return (Schema) ObjectSerializer.deserialize(s);
    } catch (IOException e) {
        throw new RuntimeException("Unable to deserialize serialized Schema: " + s, e);
    }
}
 

public static boolean validate(final URL file) {
  try (Scanner scanner =
      new Scanner(file.openStream(), StringUtils.getGeoWaveCharset().toString())) {
    if (scanner.hasNextLine()) {
      final String line = scanner.nextLine();
      return line.split(",").length == 4;
    }
  } catch (final Exception e) {
    Log.warn("Error validating file: " + file.getPath(), e);
    return false;
  }
  return false;
}
 

public void write(File file) throws Exception {
  super.write(file);
  if (!usingSentryService()) {
    return;
  }

  // remove existing metadata
  for (TSentryRole tRole : sentryClient.listRoles(StaticUserGroup.ADMIN1)) {
    sentryClient.dropRole(StaticUserGroup.ADMIN1, tRole.getRoleName());
  }

  // create roles and add privileges
  for (Entry<String, Collection<String>> roleEntry : rolesToPermissions
      .asMap().entrySet()) {
    sentryClient.createRole(StaticUserGroup.ADMIN1, roleEntry.getKey());
    for (String privilege : roleEntry.getValue()) {
      addPrivilege(roleEntry.getKey(), privilege);
    }
  }

  // grant roles to groups
  for (Entry<String, Collection<String>> groupEntry : groupsToRoles.asMap()
      .entrySet()) {
    for (String roleNames : groupEntry.getValue()) {
      for (String roleName : roleNames.split(",")) {
        try {
          sentryClient
              .grantRoleToGroup(StaticUserGroup.ADMIN1, groupEntry.getKey(), roleName);
        } catch (SentryUserException e) {
          Log.warn("Error granting role " + roleName + " to group "
              + groupEntry.getKey());
        }
      }
    }
  }
}
 

/**
 * Parse the URL that needs to be notified of the end of the job, along
 * with the number of retries in case of failure, the amount of time to
 * wait between retries and proxy settings
 * @param conf the configuration 
 */
public void setConf(Configuration conf) {
  this.conf = conf;
  
  numTries = Math.min(
    conf.getInt(MRJobConfig.MR_JOB_END_RETRY_ATTEMPTS, 0) + 1
    , conf.getInt(MRJobConfig.MR_JOB_END_NOTIFICATION_MAX_ATTEMPTS, 1)
  );
  waitInterval = Math.min(
  conf.getInt(MRJobConfig.MR_JOB_END_RETRY_INTERVAL, 5000)
  , conf.getInt(MRJobConfig.MR_JOB_END_NOTIFICATION_MAX_RETRY_INTERVAL, 5000)
  );
  waitInterval = (waitInterval < 0) ? 5000 : waitInterval;

  timeout = conf.getInt(JobContext.MR_JOB_END_NOTIFICATION_TIMEOUT,
      JobContext.DEFAULT_MR_JOB_END_NOTIFICATION_TIMEOUT);

  userUrl = conf.get(MRJobConfig.MR_JOB_END_NOTIFICATION_URL);

  proxyConf = conf.get(MRJobConfig.MR_JOB_END_NOTIFICATION_PROXY);

  //Configure the proxy to use if its set. It should be set like
  //proxyType@proxyHostname:port
  if(proxyConf != null && !proxyConf.equals("") &&
       proxyConf.lastIndexOf(":") != -1) {
    int typeIndex = proxyConf.indexOf("@");
    Proxy.Type proxyType = Proxy.Type.HTTP;
    if(typeIndex != -1 &&
      proxyConf.substring(0, typeIndex).compareToIgnoreCase("socks") == 0) {
      proxyType = Proxy.Type.SOCKS;
    }
    String hostname = proxyConf.substring(typeIndex + 1,
      proxyConf.lastIndexOf(":"));
    String portConf = proxyConf.substring(proxyConf.lastIndexOf(":") + 1);
    try {
      int port = Integer.parseInt(portConf);
      proxyToUse = new Proxy(proxyType,
        new InetSocketAddress(hostname, port));
      Log.info("Job end notification using proxy type \"" + proxyType + 
      "\" hostname \"" + hostname + "\" and port \"" + port + "\"");
    } catch(NumberFormatException nfe) {
      Log.warn("Job end notification couldn't parse configured proxy's port "
        + portConf + ". Not going to use a proxy");
    }
  }

}
 

/**
 * Parse the URL that needs to be notified of the end of the job, along
 * with the number of retries in case of failure, the amount of time to
 * wait between retries and proxy settings
 * @param conf the configuration 
 */
public void setConf(Configuration conf) {
  this.conf = conf;
  
  numTries = Math.min(
    conf.getInt(MRJobConfig.MR_JOB_END_RETRY_ATTEMPTS, 0) + 1
    , conf.getInt(MRJobConfig.MR_JOB_END_NOTIFICATION_MAX_ATTEMPTS, 1)
  );
  waitInterval = Math.min(
  conf.getInt(MRJobConfig.MR_JOB_END_RETRY_INTERVAL, 5000)
  , conf.getInt(MRJobConfig.MR_JOB_END_NOTIFICATION_MAX_RETRY_INTERVAL, 5000)
  );
  waitInterval = (waitInterval < 0) ? 5000 : waitInterval;

  timeout = conf.getInt(JobContext.MR_JOB_END_NOTIFICATION_TIMEOUT,
      JobContext.DEFAULT_MR_JOB_END_NOTIFICATION_TIMEOUT);

  userUrl = conf.get(MRJobConfig.MR_JOB_END_NOTIFICATION_URL);

  proxyConf = conf.get(MRJobConfig.MR_JOB_END_NOTIFICATION_PROXY);

  //Configure the proxy to use if its set. It should be set like
  //proxyType@proxyHostname:port
  if(proxyConf != null && !proxyConf.equals("") &&
       proxyConf.lastIndexOf(":") != -1) {
    int typeIndex = proxyConf.indexOf("@");
    Proxy.Type proxyType = Proxy.Type.HTTP;
    if(typeIndex != -1 &&
      proxyConf.substring(0, typeIndex).compareToIgnoreCase("socks") == 0) {
      proxyType = Proxy.Type.SOCKS;
    }
    String hostname = proxyConf.substring(typeIndex + 1,
      proxyConf.lastIndexOf(":"));
    String portConf = proxyConf.substring(proxyConf.lastIndexOf(":") + 1);
    try {
      int port = Integer.parseInt(portConf);
      proxyToUse = new Proxy(proxyType,
        new InetSocketAddress(hostname, port));
      Log.info("Job end notification using proxy type \"" + proxyType + 
      "\" hostname \"" + hostname + "\" and port \"" + port + "\"");
    } catch(NumberFormatException nfe) {
      Log.warn("Job end notification couldn't parse configured proxy's port "
        + portConf + ". Not going to use a proxy");
    }
  }

}