Java Code Examples for org.codehaus.jettison.json.JSONObject#optJSONArray()

/**
 * A utility method that tells whether a class is considered a bean.<br/>
 * For simplicity we exclude classes that have any type-args.
 *
 * @param className name of the class
 * @return true if it is a bean false otherwise.
 */
public boolean isInstantiableBean(String className) throws JSONException
{
  JSONObject classDesc = describeClass(className);
  if (classDesc.has("typeArgs")) {
    //any type with generics is not considered a bean
    return false;
  }
  JSONArray classProps = classDesc.optJSONArray("properties");
  if (classProps == null || classProps.length() == 0) {
    //no properties then cannot be a bean
    return false;
  }
  for (int p = 0; p < classProps.length(); p++) {
    JSONObject propDesc = classProps.getJSONObject(p);
    if (propDesc.optBoolean("canGet", false)) {
      return true;
    }
  }
  return false;
}
 

/**
 * Processes the conditional info from simple_components_build_info.json into
 * a structure mapping annotation types to component names to block names to
 * values.
 *
 * @param compJson Parsed component data from JSON
 * @param type The name of the type being processed
 * @param targetInfo Name of the annotation target being processed (e.g.,
 *                   permissions). Any of: PERMISSIONS_TARGET,
 *                   BROADCAST_RECEIVERS_TARGET
 */
private void processConditionalInfo(JSONObject compJson, String type, String targetInfo) {
  // Strip off the package name since SCM and BKY use unqualified names
  type = type.substring(type.lastIndexOf('.') + 1);

  JSONObject conditionals = compJson.optJSONObject(ComponentDescriptorConstants.CONDITIONALS_TARGET);
  if (conditionals != null) {
    JSONObject jsonBlockMap = conditionals.optJSONObject(targetInfo);
    if (jsonBlockMap != null) {
      if (!this.conditionals.containsKey(targetInfo)) {
        this.conditionals.put(targetInfo, new HashMap<String, Map<String, Set<String>>>());
      }
      Map<String, Set<String>> blockMap = new HashMap<>();
      this.conditionals.get(targetInfo).put(type, blockMap);
      for (String key : (List<String>) Lists.newArrayList(jsonBlockMap.keys())) {
        JSONArray data = jsonBlockMap.optJSONArray(key);
        HashSet<String> result = new HashSet<>();
        for (int i = 0; i < data.length(); i++) {
          result.add(data.optString(i));
        }
        blockMap.put(key, result);
      }
    }
  }
}
 

/**
 * A utility method that tells whether a class is considered a bean.<br/>
 * For simplicity we exclude classes that have any type-args.
 *
 * @param className name of the class
 * @return true if it is a bean false otherwise.
 */
public boolean isInstantiableBean(String className) throws JSONException
{
  JSONObject classDesc = describeClass(className);
  if (classDesc.has("typeArgs")) {
    //any type with generics is not considered a bean
    return false;
  }
  JSONArray classProps = classDesc.optJSONArray("properties");
  if (classProps == null || classProps.length() == 0) {
    //no properties then cannot be a bean
    return false;
  }
  for (int p = 0; p < classProps.length(); p++) {
    JSONObject propDesc = classProps.getJSONObject(p);
    if (propDesc.optBoolean("canGet", false)) {
      return true;
    }
  }
  return false;
}
 

@Test
public void testAdditionalPortInfo() throws Exception
{
  String[] classFilePath = getClassFileInClasspath();
  OperatorDiscoverer operatorDiscoverer = new OperatorDiscoverer(classFilePath);
  operatorDiscoverer.buildTypeGraph();
  JSONObject operator = operatorDiscoverer.describeOperator(SubSubClassGeneric.class.getName());

  JSONObject portClassHierarchy = new JSONObject();
  JSONObject portsWithSchemaClasses = new JSONObject();
  operatorDiscoverer.buildAdditionalPortInfo(operator, portClassHierarchy, portsWithSchemaClasses);

  JSONArray stringTypeArray = portClassHierarchy.optJSONArray("java.lang.String");
  Assert.assertNotNull("string hierarchy", stringTypeArray);

  Assert.assertEquals("number of immediate ancestors", 4, stringTypeArray.length());

  Assert.assertEquals("number of port types with schema", 0, portsWithSchemaClasses.length());
}
 

@Override
public VertexStatus getVertexStatus(String vertexName, Set<StatusGetOpts> statusOptions)
    throws IOException, TezException {
  final String url = String.format(
      "%s/%s?primaryFilter=%s:%s&secondaryFilter=vertexName:%s&fields=%s", baseUri,
      ATSConstants.TEZ_VERTEX_ID, ATSConstants.TEZ_DAG_ID, dagId, vertexName, FILTER_BY_FIELDS);

  try {
    VertexStatusProto.Builder statusBuilder;
    final JSONObject jsonRoot = getJsonRootEntity(url);
    JSONArray entitiesNode = jsonRoot.optJSONArray(ATSConstants.ENTITIES);
    if (entitiesNode == null || entitiesNode.length() != 1) {
      throw new TezException("Failed to get vertex status YARN Timeline");
    }
    JSONObject vertexNode = entitiesNode.getJSONObject(0);

    statusBuilder = parseVertexStatus(vertexNode, statusOptions);
    if (statusBuilder == null) {
      throw new TezException("Failed to parse vertex status from YARN Timeline");
    }

    return new VertexStatus(statusBuilder);
  } catch (JSONException je) {
    throw new TezException("Failed to parse VertexStatus json from YARN Timeline", je);
  }
}
 

private TezCountersProto.Builder parseDagCounters(JSONObject countersNode)
    throws JSONException {
  if (countersNode == null) {
    return null;
  }

  TezCountersProto.Builder countersProto = TezCountersProto.newBuilder();
  final JSONArray counterGroupNodes = countersNode.optJSONArray(ATSConstants.COUNTER_GROUPS);
  if (counterGroupNodes != null) {
    final int numCounterGroups = counterGroupNodes.length();

    for (int i = 0; i < numCounterGroups; i++) {
      TezCounterGroupProto.Builder counterGroupBuilder =
          parseCounterGroup(counterGroupNodes.optJSONObject(i));
      if (counterGroupBuilder != null) {
        countersProto.addCounterGroups(counterGroupBuilder);
      }
    }
  }

  return countersProto;
}
 

private void parseDAGPlan(JSONObject dagPlan) throws JSONException {
  int version = dagPlan.optInt(Constants.VERSION, 1);
  parseEdges(dagPlan.optJSONArray(Constants.EDGES));

  JSONArray verticesInfo = dagPlan.optJSONArray(Constants.VERTICES);
  parseBasicVertexInfo(verticesInfo);

  if (version > 1) {
    parseDAGContext(dagPlan.optJSONObject(Constants.DAG_CONTEXT));
  }
}
 

private TezCounterGroupProto.Builder parseCounterGroup(JSONObject counterGroupNode)
    throws JSONException {

  if (counterGroupNode == null) {
    return null;
  }

  TezCounterGroupProto.Builder counterGroup = TezCounterGroupProto.newBuilder();

  final String groupName = counterGroupNode.optString(ATSConstants.COUNTER_GROUP_NAME);
  final String groupDisplayName = counterGroupNode.optString(
      ATSConstants.COUNTER_GROUP_DISPLAY_NAME, groupName);
  final JSONArray counterNodes = counterGroupNode.optJSONArray(ATSConstants.COUNTERS);
  final int numCounters = counterNodes.length();

  List<TezCounterProto> counters = new ArrayList<TezCounterProto>(numCounters);

  for (int i = 0; i < numCounters; i++) {
    final JSONObject counterNode = counterNodes.getJSONObject(i);
    final String counterName = counterNode.getString(ATSConstants.COUNTER_NAME);
    final String counterDisplayName = counterNode.optString(ATSConstants.COUNTER_DISPLAY_NAME,
        counterName);
    final long counterValue = counterNode.getLong(ATSConstants.COUNTER_VALUE);

    counters.add(
        TezCounterProto.newBuilder()
            .setName(counterName)
            .setDisplayName(counterDisplayName)
            .setValue(counterValue)
            .build());
  }

  return counterGroup.setName(groupName)
      .setDisplayName(groupDisplayName)
      .addAllCounters(counters);
}
 

@VisibleForTesting
protected Map<String, VertexTaskStats> parseTaskStatsForVertexes()
    throws TezException, JSONException {

  if (vertexTaskStatsCache == null) {
    final String url = String.format("%s/%s?primaryFilter=%s:%s&fields=%s", baseUri,
        ATSConstants.TEZ_VERTEX_ID, ATSConstants.TEZ_DAG_ID, dagId, FILTER_BY_FIELDS);

    final JSONObject jsonRoot = getJsonRootEntity(url);
    final JSONArray vertexNodes = jsonRoot.optJSONArray(ATSConstants.ENTITIES);

    if (vertexNodes != null) {
      final int numVertexNodes = vertexNodes.length();
      Map<String, VertexTaskStats> vertexTaskStatsMap =
          new HashMap<String, VertexTaskStats>(numVertexNodes);
      for (int i = 0; i < numVertexNodes; i++) {
        final JSONObject vertexNode = vertexNodes.getJSONObject(i);
        final JSONObject otherInfoNode = vertexNode.getJSONObject(ATSConstants.OTHER_INFO);
        final String vertexName = otherInfoNode.getString(ATSConstants.VERTEX_NAME);
        final VertexTaskStats vertexTaskStats =
            new VertexTaskStats(otherInfoNode.optInt(ATSConstants.NUM_TASKS),
                otherInfoNode.optInt(ATSConstants.NUM_COMPLETED_TASKS),
                otherInfoNode.optInt(ATSConstants.NUM_SUCCEEDED_TASKS),
                otherInfoNode.optInt(ATSConstants.NUM_KILLED_TASKS),
                otherInfoNode.optInt(ATSConstants.NUM_FAILED_TASKS));
        vertexTaskStatsMap.put(vertexName, vertexTaskStats);
      }
      vertexTaskStatsCache = vertexTaskStatsMap;
    }
  }
  return vertexTaskStatsCache;
}
 

/**
 * Parse tez counters from json
 *
 * @param jsonObject
 * @return TezCounters
 * @throws JSONException
 */
public static TezCounters parseTezCountersFromJSON(JSONObject jsonObject)
    throws JSONException {
  TezCounters counters = new TezCounters();

  if (jsonObject == null) {
    return counters; //empty counters.
  }

  final JSONArray counterGroupNodes = jsonObject.optJSONArray(Constants.COUNTER_GROUPS);
  if (counterGroupNodes != null) {
    for (int i = 0; i < counterGroupNodes.length(); i++) {
      JSONObject counterGroupNode = counterGroupNodes.optJSONObject(i);
      final String groupName = counterGroupNode.optString(Constants.COUNTER_GROUP_NAME);
      final String groupDisplayName = counterGroupNode.optString(
          Constants.COUNTER_GROUP_DISPLAY_NAME, groupName);

      CounterGroup group = counters.addGroup(groupName, groupDisplayName);

      final JSONArray counterNodes = counterGroupNode.optJSONArray(Constants.COUNTERS);

      //Parse counter nodes
      for (int j = 0; j < counterNodes.length(); j++) {
        JSONObject counterNode = counterNodes.optJSONObject(j);
        final String counterName = counterNode.getString(Constants.COUNTER_NAME);
        final String counterDisplayName =
            counterNode.optString(Constants.COUNTER_DISPLAY_NAME, counterName);
        final long counterValue = counterNode.getLong(Constants.COUNTER_VALUE);
        addCounter(group, counterName, counterDisplayName, counterValue);
      }
    }
  }
  return counters;
}
 

public static List<DataDependencyEvent> parseDataEventDependencyFromJSON(JSONObject jsonObject) 
    throws JSONException {
  List<DataDependencyEvent> events = Lists.newArrayList();
  JSONArray fields = jsonObject.optJSONArray(Constants.LAST_DATA_EVENTS);
  for (int i=0; i<fields.length(); i++) {
    JSONObject eventMap = fields.getJSONObject(i);
    events.add(new DataDependencyEvent(
        StringInterner.weakIntern(eventMap.optString(EntityTypes.TEZ_TASK_ATTEMPT_ID.name())),
        eventMap.optLong(Constants.TIMESTAMP)));
  }
  return events;
}
 

/**
 * Read zip file contents. Every file can contain "dag", "vertices", "tasks", "task_attempts"
 *
 * @param atsFile
 * @throws IOException
 * @throws JSONException
 */
private void parseATSZipFile(File atsFile)
    throws IOException, JSONException, TezException, InterruptedException {
  final ZipFile atsZipFile = new ZipFile(atsFile);
  try {
    Enumeration<? extends ZipEntry> zipEntries = atsZipFile.entries();
    while (zipEntries.hasMoreElements()) {
      ZipEntry zipEntry = zipEntries.nextElement();
      LOG.debug("Processing " + zipEntry.getName());
      InputStream inputStream = atsZipFile.getInputStream(zipEntry);
      JSONObject jsonObject = readJson(inputStream);

      //This json can contain dag, vertices, tasks, task_attempts
      JSONObject dagJson = jsonObject.optJSONObject(Constants.DAG);
      if (dagJson != null) {
        //TODO: support for multiple dags per ATS file later.
        dagInfo = DagInfo.create(dagJson);
      }

      //Process vertex
      JSONArray vertexJson = jsonObject.optJSONArray(Constants.VERTICES);
      if (vertexJson != null) {
        processVertices(vertexJson);
      }

      //Process task
      JSONArray taskJson = jsonObject.optJSONArray(Constants.TASKS);
      if (taskJson != null) {
        processTasks(taskJson);
      }

      //Process task attempts
      JSONArray attemptsJson = jsonObject.optJSONArray(Constants.TASK_ATTEMPTS);
      if (attemptsJson != null) {
        processAttempts(attemptsJson);
      }

      //Process application (mainly versionInfo)
      JSONObject tezAppJson = jsonObject.optJSONObject(Constants.APPLICATION);
      if (tezAppJson != null) {
        processApplication(tezAppJson);
      }
    }
  } finally {
    atsZipFile.close();
  }
}
 

private void parseBasicVertexInfo(JSONArray verticesInfo) throws JSONException {
  if (verticesInfo == null) {
    LOG.info("No vertices available.");
    return;
  }

  //Parse basic information available in DAG for vertex and edges
  for (int i = 0; i < verticesInfo.length(); i++) {
    BasicVertexInfo basicVertexInfo = new BasicVertexInfo();

    JSONObject vJson = verticesInfo.getJSONObject(i);
    basicVertexInfo.vertexName =
        vJson.optString(Constants.VERTEX_NAME);
    JSONArray inEdges = vJson.optJSONArray(Constants.IN_EDGE_IDS);
    if (inEdges != null) {
      String[] inEdgeIds = new String[inEdges.length()];
      for (int j = 0; j < inEdges.length(); j++) {
        inEdgeIds[j] = inEdges.get(j).toString();
      }
      basicVertexInfo.inEdgeIds = inEdgeIds;
    }

    JSONArray outEdges = vJson.optJSONArray(Constants.OUT_EDGE_IDS);
    if (outEdges != null) {
      String[] outEdgeIds = new String[outEdges.length()];
      for (int j = 0; j < outEdges.length(); j++) {
        outEdgeIds[j] = outEdges.get(j).toString();
      }
      basicVertexInfo.outEdgeIds = outEdgeIds;
    }

    JSONArray addInputsJson =
        vJson.optJSONArray(Constants.ADDITIONAL_INPUTS);
    basicVertexInfo.additionalInputs = parseAdditionalDetailsForVertex(addInputsJson);

    JSONArray addOutputsJson =
        vJson.optJSONArray(Constants.ADDITIONAL_OUTPUTS);
    basicVertexInfo.additionalOutputs = parseAdditionalDetailsForVertex(addOutputsJson);

    basicVertexInfoMap.put(basicVertexInfo.vertexName, basicVertexInfo);
  }
}
 

DagInfo(JSONObject jsonObject) throws JSONException {
  super(jsonObject);

  vertexNameMap = Maps.newHashMap();
  vertexNameIDMapping = new DualHashBidiMap<>();
  edgeInfoMap = Maps.newHashMap();
  basicVertexInfoMap = Maps.newHashMap();
  containerMapping = LinkedHashMultimap.create();

  Preconditions.checkArgument(jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase
      (Constants.TEZ_DAG_ID));

  dagId = StringInterner.weakIntern(jsonObject.getString(Constants.ENTITY));

  //Parse additional Info
  JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO);

  long sTime = otherInfoNode.optLong(Constants.START_TIME);
  long eTime= otherInfoNode.optLong(Constants.FINISH_TIME);
  userName = otherInfoNode.optString(Constants.USER);
  if (eTime < sTime) {
    LOG.warn("DAG has got wrong start/end values. "
        + "startTime=" + sTime + ", endTime=" + eTime + ". Will check "
        + "timestamps in DAG started/finished events");

    // Check if events DAG_STARTED, DAG_FINISHED can be made use of
    for(Event event : eventList) {
      switch (HistoryEventType.valueOf(event.getType())) {
      case DAG_STARTED:
        sTime = event.getAbsoluteTime();
        break;
      case DAG_FINISHED:
        eTime = event.getAbsoluteTime();
        break;
      default:
        break;
      }
    }

    if (eTime < sTime) {
      LOG.warn("DAG has got wrong start/end values in events as well. "
          + "startTime=" + sTime + ", endTime=" + eTime);
    }
  }
  startTime = sTime;
  endTime = eTime;

  //TODO: Not getting populated correctly for lots of jobs.  Verify
  submitTime = otherInfoNode.optLong(Constants.START_REQUESTED_TIME);
  diagnostics = otherInfoNode.optString(Constants.DIAGNOSTICS);
  failedTasks = otherInfoNode.optInt(Constants.NUM_FAILED_TASKS);
  JSONObject dagPlan = otherInfoNode.optJSONObject(Constants.DAG_PLAN);
  name = StringInterner.weakIntern((dagPlan != null) ? (dagPlan.optString(Constants.DAG_NAME)) : null);
  if (dagPlan != null) {
    JSONArray vertices = dagPlan.optJSONArray(Constants.VERTICES);
    if (vertices != null) {
      numVertices = vertices.length();
    } else {
      numVertices = 0;
    }
    parseDAGPlan(dagPlan);
  } else {
    numVertices = 0;
  }
  status = StringInterner.weakIntern(otherInfoNode.optString(Constants.STATUS));

  //parse name id mapping
  JSONObject vertexIDMappingJson = otherInfoNode.optJSONObject(Constants.VERTEX_NAME_ID_MAPPING);
  if (vertexIDMappingJson != null) {
    //get vertex name
    for (Map.Entry<String, BasicVertexInfo> entry : basicVertexInfoMap.entrySet()) {
      String vertexId = vertexIDMappingJson.optString(entry.getKey());
      //vertexName --> vertexId
      vertexNameIDMapping.put(entry.getKey(), vertexId);
    }
  }
}
 

protected void readEventsFromSource(String dagId, JSONObjectSource source,
    Map<String, JSONObject> vertexJsonMap, Map<String, JSONObject> taskJsonMap,
    Map<String, JSONObject> attemptJsonMap) throws JSONException, TezException, IOException{
  JSONObject dagJson = null;
  TezDAGID tezDAGID = TezDAGID.fromString(dagId);
  String userName = null;

  while (source.hasNext()) {
    JSONObject jsonObject = source.next();

    String entity = jsonObject.getString(Constants.ENTITY);
    String entityType = jsonObject.getString(Constants.ENTITY_TYPE);
    switch (entityType) {
    case Constants.TEZ_DAG_ID:
      if (!dagId.equals(entity)) {
        LOG.warn(dagId + " is not matching with " + entity);
        continue;
      }
      // Club all DAG related information together (DAG_INIT, DAG_FINISH etc). Each of them
      // would have a set of entities in otherinfo (e.g vertex mapping, dagPlan, start/finish
      // time etc).
      if (dagJson == null) {
        dagJson = jsonObject;
      } else if (dagJson.optJSONObject(ATSConstants.OTHER_INFO)
          .optJSONObject(ATSConstants.DAG_PLAN) == null) {
        // if DAG_PLAN is not filled already, let's try to fetch it from other
        dagJson.getJSONObject(ATSConstants.OTHER_INFO).put(ATSConstants.DAG_PLAN, jsonObject
            .getJSONObject(ATSConstants.OTHER_INFO).getJSONObject(ATSConstants.DAG_PLAN));
      }
      JSONArray relatedEntities = dagJson.optJSONArray(Constants
          .RELATED_ENTITIES);
      //UserName is present in related entities
      // {"entity":"userXYZ","entitytype":"user"}
      if (relatedEntities != null) {
        for (int i = 0; i < relatedEntities.length(); i++) {
          JSONObject subEntity = relatedEntities.getJSONObject(i);
          String subEntityType = subEntity.optString(Constants.ENTITY_TYPE);
          if (subEntityType != null && subEntityType.equals(Constants.USER)) {
            userName = subEntity.getString(Constants.ENTITY);
            break;
          }
        }
      }
      populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO),
          dagJson.getJSONObject(Constants.OTHER_INFO));
      break;
    case Constants.TEZ_VERTEX_ID:
      String vertexName = entity;
      TezVertexID tezVertexID = TezVertexID.fromString(vertexName);
      if (!tezDAGID.equals(tezVertexID.getDAGId())) {
        LOG.warn("{} does not belong to {} ('{}' != '{}')}", vertexName, tezDAGID, tezDAGID, tezVertexID.getDAGId());
        continue;
      }
      if (!vertexJsonMap.containsKey(vertexName)) {
        vertexJsonMap.put(vertexName, jsonObject);
      }
      populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), vertexName, vertexJsonMap);
      break;
    case Constants.TEZ_TASK_ID:
      String taskName = entity;
      TezTaskID tezTaskID = TezTaskID.fromString(taskName);
      if (!tezDAGID.equals(tezTaskID.getVertexID().getDAGId())) {
        LOG.warn("{} does not belong to {} ('{}' != '{}')}", taskName, tezDAGID, tezDAGID,
            tezTaskID.getVertexID().getDAGId());
        continue;
      }
      if (!taskJsonMap.containsKey(taskName)) {
        taskJsonMap.put(taskName, jsonObject);
      }
      populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), taskName, taskJsonMap);
      break;
    case Constants.TEZ_TASK_ATTEMPT_ID:
      String taskAttemptName = entity;
      TezTaskAttemptID tezAttemptId = TezTaskAttemptID.fromString(taskAttemptName);
      if (!tezDAGID.equals(tezAttemptId.getTaskID().getVertexID().getDAGId())) {
        LOG.warn("{} does not belong to {} ('{}' != '{}')}", taskAttemptName, tezDAGID, tezDAGID,
            tezAttemptId.getTaskID().getVertexID().getDAGId());
        continue;
      }
      if (!attemptJsonMap.containsKey(taskAttemptName)) {
        attemptJsonMap.put(taskAttemptName, jsonObject);
      }
      populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), taskAttemptName, attemptJsonMap);
      break;
    default:
      break;
    }
  }
  source.close();
  if (dagJson != null) {
    this.dagInfo = DagInfo.create(dagJson);
    setUserName(userName);
  } else {
    LOG.error("Dag is not yet parsed. Looks like partial file.");
    throw new TezException(
        "Please provide a valid/complete history log file containing " + dagId);
  }
}
 

public static PropertiesConfiguration convertToProperties(JSONObject json) throws JSONException
{
  PropertiesConfiguration props = new PropertiesConfiguration();
  JSONArray allOperators = json.getJSONArray("operators");
  JSONArray allStreams = json.getJSONArray("streams");

  for (int j = 0; j < allOperators.length(); j++) {
    JSONObject operatorDetail = allOperators.getJSONObject(j);
    String operatorName = operatorDetail.getString("name");
    String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorName;
    props.setProperty(operatorKey + ".classname", operatorDetail.getString("class"));
    JSONObject properties = operatorDetail.optJSONObject("properties");
    if (properties != null) {
      Iterator<String> iter2 = properties.keys();
      while (iter2.hasNext()) {
        String propertyName = iter2.next();
        if (!propertyName.equals("name") && !propertyName.equals("class") && properties.opt(propertyName) != null) {
          JSONArray list = properties.optJSONArray(propertyName);
          String value = "";
          if (list != null) {
            for (int i = 0; i < list.length(); i++) {
              if (i != 0) {
                value += ",";
              }
              value += list.get(i).toString();
            }
            props.setProperty(operatorKey + "." + propertyName, value);
          } else {
            props.setProperty(operatorKey + "." + propertyName, properties.get(propertyName));
          }
        }
      }
    }
  }

  for (int j = 0; j < allStreams.length(); j++) {
    JSONObject streamDetail = allStreams.getJSONObject(j);
    String streamName = streamDetail.getString("name");
    String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamName;
    JSONObject sourceDetail = streamDetail.getJSONObject("source");
    JSONArray sinksList = streamDetail.getJSONArray("sinks");

    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, sourceDetail.getString("operatorName") + "." + sourceDetail.getString("portName"));
    String sinksValue = "";
    for (int i = 0; i < sinksList.length(); i++) {
      if (!sinksValue.isEmpty()) {
        sinksValue += ",";
      }
      sinksValue += sinksList.getJSONObject(i).getString("operatorName") + "." + sinksList.getJSONObject(i).getString("portName");
    }
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
    String locality = streamDetail.optString("locality", null);
    if (locality != null) {
      props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, Locality.valueOf(locality));
    }
  }

  // TBD: Attributes

  return props;
}
 

private String authenticateADFS(HttpClientAdapter httpClientAdapter, String responseBodyAsString, String authorizeUrl) throws IOException {
    URI location;

    if (responseBodyAsString.contains("login.microsoftonline.com")) {
        LOGGER.info("Already authenticated through Basic or NTLM");
    } else {
        // parse form to get target url, authenticate as userid
        PostRequest logonMethod = new PostRequest(extract("method=\"post\" action=\"([^\"]+)\"", responseBodyAsString));
        logonMethod.setRequestHeader("Content-Type", "application/x-www-form-urlencoded");

        logonMethod.setParameter("UserName", userid);
        logonMethod.setParameter("Password", password);
        logonMethod.setParameter("AuthMethod", "FormsAuthentication");

        httpClientAdapter.executePostRequest(logonMethod);
        location = logonMethod.getRedirectLocation();
        if (location == null) {
            throw new DavMailAuthenticationException("EXCEPTION_AUTHENTICATION_FAILED");
        }

        GetRequest redirectMethod = new GetRequest(location);
        responseBodyAsString = httpClientAdapter.executeGetRequest(redirectMethod);
    }

    if (!responseBodyAsString.contains("login.microsoftonline.com")) {
        throw new DavMailAuthenticationException("EXCEPTION_AUTHENTICATION_FAILED");
    }
    String targetUrl = extract("action=\"([^\"]+)\"", responseBodyAsString);
    String wa = extract("name=\"wa\" value=\"([^\"]+)\"", responseBodyAsString);
    String wresult = extract("name=\"wresult\" value=\"([^\"]+)\"", responseBodyAsString);
    // decode wresult
    wresult = wresult.replaceAll(""", "\"");
    wresult = wresult.replaceAll("&lt;", "<");
    wresult = wresult.replaceAll("&gt;", ">");
    String wctx = extract("name=\"wctx\" value=\"([^\"]+)\"", responseBodyAsString);
    wctx = wctx.replaceAll("&amp;", "&");

    PostRequest targetMethod = new PostRequest(targetUrl);
    targetMethod.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
    targetMethod.setParameter("wa", wa);
    targetMethod.setParameter("wresult", wresult);
    targetMethod.setParameter("wctx", wctx);

    responseBodyAsString = httpClientAdapter.executePostRequest(targetMethod);
    location = targetMethod.getRedirectLocation();

    LOGGER.debug(targetMethod.getURI().toString());
    LOGGER.debug(targetMethod.getReasonPhrase());
    LOGGER.debug(responseBodyAsString);

    if (targetMethod.getStatusCode() == HttpStatus.SC_OK) {
        JSONObject config = extractConfig(responseBodyAsString);
        if (config.optJSONArray("arrScopes") != null || config.optJSONArray("urlPostRedirect") != null) {
            LOGGER.debug("Authentication successful but user consent or validation needed, please open the following url in a browser");
            LOGGER.debug(authorizeUrl);
            throw new DavMailAuthenticationException("EXCEPTION_AUTHENTICATION_FAILED");
        }
    } else if (targetMethod.getStatusCode() != HttpStatus.SC_MOVED_TEMPORARILY || location == null) {
        throw new IOException("Unknown ADFS authentication failure");
    }

    if ("device.login.microsoftonline.com".equals(location.getHost())) {
        location = processDeviceLogin(httpClientAdapter, location);
    }
    String query = location.getQuery();
    if (query == null) {
        // failover for null query with non https URI like urn:ietf:wg:oauth:2.0:oob?code=...
        query = location.getSchemeSpecificPart();
    }

    if (query.contains("code=") && query.contains("&session_state=")) {
        String code = query.substring(query.indexOf("code=") + 5, query.indexOf("&session_state="));
        LOGGER.debug("Authentication Code: " + code);
        return code;
    }
    throw new IOException("Unknown ADFS authentication failure");
}
 

public static PropertiesConfiguration convertToProperties(JSONObject json) throws JSONException
{
  PropertiesConfiguration props = new PropertiesConfiguration();
  JSONArray allOperators = json.getJSONArray("operators");
  JSONArray allStreams = json.getJSONArray("streams");

  for (int j = 0; j < allOperators.length(); j++) {
    JSONObject operatorDetail = allOperators.getJSONObject(j);
    String operatorName = operatorDetail.getString("name");
    String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorName;
    props.setProperty(operatorKey + ".classname", operatorDetail.getString("class"));
    JSONObject properties = operatorDetail.optJSONObject("properties");
    if (properties != null) {
      Iterator<String> iter2 = properties.keys();
      while (iter2.hasNext()) {
        String propertyName = iter2.next();
        if (!propertyName.equals("name") && !propertyName.equals("class") && properties.opt(propertyName) != null) {
          JSONArray list = properties.optJSONArray(propertyName);
          String value = "";
          if (list != null) {
            for (int i = 0; i < list.length(); i++) {
              if (i != 0) {
                value += ",";
              }
              value += list.get(i).toString();
            }
            props.setProperty(operatorKey + "." + propertyName, value);
          } else {
            props.setProperty(operatorKey + "." + propertyName, properties.get(propertyName));
          }
        }
      }
    }
  }

  for (int j = 0; j < allStreams.length(); j++) {
    JSONObject streamDetail = allStreams.getJSONObject(j);
    String streamName = streamDetail.getString("name");
    String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamName;
    JSONObject sourceDetail = streamDetail.getJSONObject("source");
    JSONArray sinksList = streamDetail.getJSONArray("sinks");

    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, sourceDetail.getString("operatorName") + "." + sourceDetail.getString("portName"));
    String sinksValue = "";
    for (int i = 0; i < sinksList.length(); i++) {
      if (!sinksValue.isEmpty()) {
        sinksValue += ",";
      }
      sinksValue += sinksList.getJSONObject(i).getString("operatorName") + "." + sinksList.getJSONObject(i).getString("portName");
    }
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
    String locality = streamDetail.optString("locality", null);
    if (locality != null) {
      props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, Locality.valueOf(locality));
    }
  }

  // TBD: Attributes

  return props;
}