Java Code Examples for net.htmlparser.jericho.Source#getAllElements()

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {

    List<Element> sourceElements = source.getAllElements();
    sourceElements.stream()
            .filter(element -> SupportedElements.contains(element.getName()))
            .filter(unsafeSubResource(msg.getRequestHeader().getHostName()))
            .forEach(
                    element -> {
                        newAlert()
                                .setRisk(Alert.RISK_MEDIUM)
                                .setConfidence(Alert.CONFIDENCE_HIGH)
                                .setDescription(getString("desc"))
                                .setSolution(getString("soln"))
                                .setReference(getString("refs"))
                                .setEvidence(element.toString())
                                .setCweId(16) // CWE CATEGORY: Configuration
                                .setWascId(15) // Application Misconfiguration
                                .raise();
                    });
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    if (msg.getResponseBody().length() > 0 && msg.getResponseHeader().isHtml()) {
        List<Element> sourceElements = source.getAllElements(HTMLElementName.SCRIPT);
        if (sourceElements != null) {
            for (Element sourceElement : sourceElements) {
                String src = sourceElement.getAttributeValue("src");
                if (src != null
                        && isScriptFromOtherDomain(
                                msg.getRequestHeader().getHostName(), src, msg)) {
                    String integrity = sourceElement.getAttributeValue("integrity");
                    if (integrity == null || integrity.trim().length() == 0) {
                        /*
                         * If it has an integrity value assume its fine
                         * We dont check the integrity value is valid because
                         * 1. pscan rules cant make new requests and
                         * 2. the browser will check it anyway
                         */
                        this.raiseAlert(msg, id, src, sourceElement.toString());
                    }
                }
            }
        }
    }
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    if (msg.getResponseHeader().getStatusCode() != HttpStatusCode.OK
            || isHttps(msg)
            || !isResponseHTML(msg, source)) {
        return;
    }

    List<Element> formElements = source.getAllElements(HTMLElementName.FORM);
    for (Element formElement : formElements) {
        String formAction = formElement.getAttributeValue("action");
        if (formAction != null && formAction.trim().toLowerCase().startsWith("https://")) {
            raiseAlert(msg, id, formElement);
        }
    }
}
 

/**
 * Checks the source of the response for HSTS being set via a META tag which is explicitly not
 * supported per the spec (rfc6797).
 *
 * @param source the source of the response to be analyzed.
 * @return returns a string if HSTS was set via META (for use as alert evidence) otherwise
 *     return {@code null}.
 * @see <a href="https://tools.ietf.org/html/rfc6797#section-8.5">RFC 6797 Section 8.5</a>
 */
private String getMetaHSTSEvidence(Source source) {
    List<Element> metaElements = source.getAllElements(HTMLElementName.META);
    String httpEquiv;

    if (metaElements != null) {
        for (Element metaElement : metaElements) {
            httpEquiv = metaElement.getAttributeValue("http-equiv");
            if (STS_HEADER.equalsIgnoreCase(httpEquiv)) {
                return httpEquiv; // This is a META which attempts to define HSTS return it's
                // value
            }
        }
    }
    return null;
}
 

protected void loadFirstResource ( final List<ClassResource> resources, final List<String> xmlContent, final List<String> xmlNames,
                                   final List<ClassResource> xmlFiles ) throws IOException
{
    final ClassResource rf = resources.get ( 0 );
    final Source xmlSource = new Source ( ReflectUtils.getClassSafely ( rf.getClassName () ).getResource ( rf.getPath () ) );
    xmlSource.setLogger ( null );
    xmlSource.fullSequentialParse ();

    final Element baseClassTag = xmlSource.getFirstElement ( SkinInfoConverter.CLASS_NODE );
    final String baseClass = baseClassTag != null ? baseClassTag.getContent ().toString () : null;

    for ( final Element includeTag : xmlSource.getAllElements ( SkinInfoConverter.INCLUDE_NODE ) )
    {
        final String includeClass = includeTag.getAttributeValue ( SkinInfoConverter.NEAR_CLASS_ATTRIBUTE );
        final String finalClass = includeClass != null ? includeClass : baseClass;
        final String src = includeTag.getContent ().toString ();
        resources.add ( new ClassResource ( finalClass, src ) );
    }

    xmlContent.add ( xmlSource.toString () );
    xmlNames.add ( new File ( rf.getPath () ).getName () );
    xmlFiles.add ( rf );

    resources.remove ( 0 );
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    if (msg.getResponseHeader().getStatusCode() != 200 || !isResponseHTML(msg, source)) {
        return;
    }

    List<Element> htmlElements = source.getAllElements();
    if (htmlElements.size() == 0) {
        return;
    }

    Set<HtmlParameter> params = new TreeSet<HtmlParameter>(msg.getFormParams());
    params.addAll(msg.getUrlParams());
    if (params.size() == 0) {
        return;
    }

    checkHtmlElements(msg, id, params, htmlElements);
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    if (msg.getResponseHeader().getStatusCode() != HttpStatusCode.OK
            || !isHttps(msg)
            || !isResponseHTML(msg, source)) {
        return;
    }

    List<Element> formElements = source.getAllElements(HTMLElementName.FORM);
    for (Element formElement : formElements) {
        String formAction = formElement.getAttributeValue("action");
        if (formAction != null && formAction.trim().toLowerCase().startsWith("http://")) {
            raiseAlert(msg, id, formElement);
        }
    }
}
 

private void checkMetaContentCharset(
        HttpMessage msg, int id, Source source, Set<HtmlParameter> params) {
    List<Element> metaElements = source.getAllElements(HTMLElementName.META);
    if (metaElements == null || metaElements.size() == 0) {
        return;
    }

    for (Element metaElement : metaElements) {
        String httpEquiv = metaElement.getAttributeValue("http-equiv");
        String bodyContentType = metaElement.getAttributeValue("content");

        // If META element defines HTTP-EQUIV and CONTENT attributes,
        // compare charset values
        if (httpEquiv == null
                || bodyContentType == null
                || !httpEquiv.equalsIgnoreCase("content-type")) {
            continue;
        }

        String bodyContentCharset = getBodyContentCharset(bodyContentType);
        if (bodyContentCharset == null) {
            continue;
        }
        for (HtmlParameter param : params) {
            if (bodyContentCharset.equalsIgnoreCase(param.getValue())) {
                raiseAlert(msg, id, "META", "Content-Type", param, bodyContentCharset);
            }
        }
    }
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    if (msg.getResponseHeader().getStatusCode() != HttpStatusCode.OK) {
        return;
    }

    if (!isResponseHTML(msg)) {
        return;
    }

    Set<HtmlParameter> params = new TreeSet<HtmlParameter>(msg.getFormParams());
    params.addAll(msg.getUrlParams());
    if (params.size() == 0) {
        return;
    }

    List<Element> htmlElements = source.getAllElements();
    for (Element htmlElement : htmlElements) {
        Attributes attributes = htmlElement.getAttributes();
        if (attributes == null) {
            continue;
        }

        for (Attribute attribute : attributes) {
            if (Arrays.binarySearch(JAVASCRIPT_EVENTS, attribute.getName().toLowerCase())
                    >= 0) {
                for (HtmlParameter param : params) {
                    if (param.getValue() != null && param.getValue().length() > 0) {
                        checkJavascriptEvent(msg, id, htmlElement, attribute, param);
                    }
                }
            }
        }
    }
}
 

private void checkMetaElementsMatches(Source source) {
    List<Element> metaElements = source.getAllElements(HTMLElementName.META);
    for (Element metaElement : metaElements) {
        for (Map<String, AppPattern> sp : currentApp.getMetas()) {
            for (Map.Entry<String, AppPattern> entry : sp.entrySet()) {
                String name = metaElement.getAttributeValue("name");
                String content = metaElement.getAttributeValue("content");
                if (name != null && content != null && name.equals(entry.getKey())) {
                    AppPattern p = entry.getValue();
                    addIfMatches(p, content);
                }
            }
        }
    }
}
 

public Attributes getParam(String body, String paramName) {

        Attributes param = null;
        Source src = new Source(body);
        List<Element> formElements = src.getAllElements(HTMLElementName.FORM);

        if (formElements != null && formElements.size() > 0) {
            // Loop through all of the FORM tags
            logger.debug("Found " + formElements.size() + " forms");

            for (Element formElement : formElements) {
                List<Element> elements = formElement.getAllElements();

                if (elements != null && elements.size() > 0) {
                    // Loop through all of the elements
                    logger.debug("Found " + elements.size() + " inputs");
                    for (Element element : elements) {
                        Attributes atts = element.getAttributes();
                        try {
                            //  Get attr name
                            Attribute name = atts.get("name");
                            if (name != null) {
                                if (name.getValue().equals(paramName)) {
                                    param = atts;
                                }
                            }
                        } catch (Exception e) {
                            logger.debug("Couldnt get name attribute of parameter", e);
                        }
                    }
                }
            }
        }

        return param;
    }
 

/**
 * Checks the source of the response for XFO being set via a META tag which is explicitly not
 * supported per the spec (rfc7034).
 *
 * @param source the source of the response to be analyzed.
 * @return returns a string if XFO was set via META (for use as alert evidence) otherwise return
 *     {@code null}.
 * @see <a href="https://tools.ietf.org/html/rfc7034#section-4">RFC 7034 Section 4</a>
 */
private String getMetaXFOEvidence(Source source) {
    List<Element> metaElements = source.getAllElements(HTMLElementName.META);
    String httpEquiv;

    if (metaElements != null) {
        for (Element metaElement : metaElements) {
            httpEquiv = metaElement.getAttributeValue("http-equiv");
            if (HttpHeader.X_FRAME_OPTION.equalsIgnoreCase(httpEquiv)) {
                return metaElement.toString();
            }
        }
    }
    return null;
}
 

public String getTokenValue(HttpMessage tokenMsg, String tokenName) {
    Source source = new Source(tokenMsg.getResponseBody().toString());
    List<Element> formElements = source.getAllElements(HTMLElementName.FORM);

    if (formElements != null && formElements.size() > 0) {
        // Loop through all of the FORM tags

        for (Element formElement : formElements) {
            List<Element> inputElements = formElement.getAllElements(HTMLElementName.INPUT);

            if (inputElements != null && inputElements.size() > 0) {
                // Loop through all of the INPUT elements
                for (Element inputElement : inputElements) {
                    String id = inputElement.getAttributeValue("ID");
                    if (id != null && id.equalsIgnoreCase(tokenName)) {
                        return inputElement.getAttributeValue("VALUE");
                    }
                    String name = inputElement.getAttributeValue("NAME");
                    if (name != null && name.equalsIgnoreCase(tokenName)) {
                        return inputElement.getAttributeValue("VALUE");
                    }
                }
            }
        }
    }
    return null;
}
 

public Vector<String> getFormInputFields(HttpMessage tokenMsg) {
    Source source = new Source(tokenMsg.getResponseBody().toString());
    List<Element> formElements = source.getAllElements(HTMLElementName.FORM);
    Vector<String> fifs = new Vector<>();

    if (formElements != null && formElements.size() > 0) {
        // Loop through all of the FORM tags

        for (Element formElement : formElements) {
            List<Element> inputElements = formElement.getAllElements(HTMLElementName.INPUT);

            if (inputElements != null && inputElements.size() > 0) {
                // Loop through all of the INPUT elements
                for (Element inputElement : inputElements) {
                    String id = inputElement.getAttributeValue("ID");
                    if (id != null && id.length() > 0) {
                        fifs.add(id);
                    } else {
                        String name = inputElement.getAttributeValue("NAME");
                        if (name != null && name.length() > 0) {
                            fifs.add(name);
                        }
                    }
                }
            }
        }
    }
    return fifs;
}
 

private void checkScriptElementsMatches(Source source) {
    for (Element scriptElement : source.getAllElements(HTMLElementName.SCRIPT)) {
        for (AppPattern appPattern : currentApp.getScript()) {
            String src = scriptElement.getAttributeValue("src");
            if (src != null && !src.isEmpty()) {
                addIfMatches(appPattern, src);
            }
        }
    }
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    if (!AlertThreshold.LOW.equals(this.getAlertThreshold())
            || !getHelper().getTechSet().includes(Tech.JSP_SERVLET)) {
        return;
    }

    List<Element> formElements = source.getAllElements(HTMLElementName.FORM);

    if (formElements != null && formElements.size() > 0) {
        // Loop through all of the FORM tags
        logger.debug("Found " + formElements.size() + " forms");

        // check for 'target' param

        for (Element formElement : formElements) {
            boolean actionMissingOrEmpty =
                    StringUtils.isEmpty(formElement.getAttributeValue("action"));

            if (actionMissingOrEmpty) {
                newAlert()
                        .setRisk(Alert.RISK_MEDIUM)
                        .setConfidence(Alert.CONFIDENCE_LOW)
                        .setDescription(getDescription())
                        .setSolution(getSolution())
                        .setReference(getReference())
                        .setEvidence(
                                formElement
                                        .getFirstStartTag()
                                        .toString()) // evidence - just include the first <form
                        // ..>
                        // element
                        .setCweId(20) // CWE Id 20 - Improper Input Validation
                        .setWascId(20) // WASC Id 20 - Improper Input Handling
                        .raise();
                // Only raise one alert per page
                return;
            }
        }
    }
}
 

@Override
public boolean onHttpResponseReceive(HttpMessage msg) {
    if (options.isEnabled() && msg.getResponseHeader().isHtml()) {
        try {
            String html = msg.getResponseBody().toString();

            Source document = new Source(html);
            List<Element> heads = document.getAllElements("head");
            Element head = heads.isEmpty() ? null : heads.get(0);

            if (head != null && msg.getHistoryRef() != null) {
                String host = msg.getRequestHeader().getHeader("host");
                String frontEndApiUrl =
                        API.getInstance().getCallBackUrl(this.api, "https://" + host);

                int historyReferenceId = msg.getHistoryRef().getHistoryId();

                StringBuilder injectedContentBuilder =
                        new StringBuilder(200)
                                .append("<script src='")
                                .append(frontEndApiUrl)
                                .append("?action=getFile")
                                .append("&filename=front-end-scanner.js")
                                .append("&historyReferenceId=")
                                .append(historyReferenceId)
                                .append("'></script>");

                String injectedContent = injectedContentBuilder.toString();

                OutputDocument newResponseBody = new OutputDocument(document);
                int insertPosition = head.getChildElements().get(0).getBegin();
                newResponseBody.insert(insertPosition, injectedContent);

                msg.getResponseBody().setBody(newResponseBody.toString());

                int newLength = msg.getResponseBody().length();
                msg.getResponseHeader().setContentLength(newLength);
            } else {
                LOGGER.debug("<head></head> is missing in the response");
            }
        } catch (Exception e) {
            LOGGER.error(e.getMessage(), e);
        }
    }
    return true;
}
 

private void revealFields(HttpMessage msg) {
    boolean changed = false;
    String response = msg.getResponseBody().toString();
    Source src = new Source(response);
    OutputDocument outputDocument = new OutputDocument(src);

    List<Element> formElements = src.getAllElements(HTMLElementName.FORM);

    if (formElements != null && formElements.size() > 0) {
        // Loop through all of the FORM tags
        logger.debug("Found " + formElements.size() + " forms");

        for (Element formElement : formElements) {
            List<Element> elements = formElement.getAllElements();

            if (elements != null && elements.size() > 0) {
                // Loop through all of the elements
                logger.debug("Found " + elements.size() + " inputs");
                for (Element element : elements) {
                    Attributes atts = element.getAttributes();

                    if (atts != null && atts.size() > 0) {
                        Iterator<Attribute> iter = atts.iterator();
                        while (iter.hasNext()) {
                            Attribute att = iter.next();
                            if (ATT_DISABLED.equalsIgnoreCase(att.getName())
                                    || ATT_READONLY.equalsIgnoreCase(att.getName())
                                    || (ATT_TYPE.equalsIgnoreCase(att.getName())
                                            && TYPE_HIDDEN.equalsIgnoreCase(att.getValue()))) {
                                logger.debug(
                                        "Removing "
                                                + att.getName()
                                                + ": "
                                                + response.substring(
                                                        att.getBegin(), att.getEnd()));
                                outputDocument.remove(att);
                                changed = true;
                            }
                        }
                    }
                }
            }
        }
    }
    if (changed) {
        msg.setResponseBody(outputDocument.toString());
    }
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    if (!msg.getRequestHeader().isSecure()) {
        // If SSL/TLS isn't used then this check isn't relevant
        return;
    }

    if (msg.getResponseBody().length() == 0 || !msg.getResponseHeader().isHtml()) {
        // No point attempting to parse non-HTML content, it will not be correctly interpreted.
        return;
    }

    List<MixedContent> list = new ArrayList<>();
    boolean incScript = false;
    List<Element> sourceElements = source.getAllElements();
    for (Element sourceElement : sourceElements) {
        if (addAttsContainingHttpContent(sourceElement, "src", list)) {
            if (HTMLElementName.SCRIPT.equals(sourceElement.getName())) {
                // Considered to be more serious
                incScript = true;
            }
        }
        addAttsContainingHttpContent(sourceElement, "background", list);
        addAttsContainingHttpContent(sourceElement, "classid", list);
        addAttsContainingHttpContent(sourceElement, "codebase", list);
        addAttsContainingHttpContent(sourceElement, "data", list);
        addAttsContainingHttpContent(sourceElement, "icon", list);
        addAttsContainingHttpContent(sourceElement, "usemap", list);

        switch (this.getAlertThreshold()) {
            case LOW:
            case MEDIUM:
                // These are a bit more debatable, so dont do them on the HIGH setting
                addAttsContainingHttpContent(sourceElement, "action", list);
                addAttsContainingHttpContent(sourceElement, "formaction", list);
                break;
            default:
                // No other checks
        }
    }

    final int numberOfMixedElements = list.size();
    if (numberOfMixedElements > 0) {
        StringBuilder sb = new StringBuilder(numberOfMixedElements * 40);
        for (MixedContent mc : list) {
            sb.append("tag=");
            sb.append(mc.getTag());
            sb.append(' ');
            sb.append(mc.getAtt());
            sb.append('=');
            sb.append(mc.getValue());
            sb.append('\n');
        }

        this.raiseAlert(msg, id, list.get(0).getValue(), sb.toString(), incScript);
    }
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    if (msg.getResponseBody().length() > 0 && msg.getResponseHeader().isText()) {
        List<Element> sourceElements = source.getAllElements(HTMLElementName.INPUT);
        if (sourceElements != null) {
            for (Element sourceElement : sourceElements) {

                // Find ones where id="javax.faces.ViewState"
                //
                // TODO: Other possible field names include:
                // jsf_state_64
                // jsf_sequence
                // jsf_tree
                // jsf_tree_64
                // jsf_viewid
                // jsf_state
                String src = sourceElement.getAttributeValue("id");
                if (src != null && src.toLowerCase().contains("javax.faces.viewstate")) {
                    // Get the ViewState value
                    String val = sourceElement.getAttributeValue("value");
                    // Server-side ViewState usually comes down as an ID
                    // value like
                    // _id16683
                    // Ignoring these for now. Underscore is not a valid
                    // Base64 character
                    // so it's safe to ignore this.
                    if (val != null && val.startsWith("_")) {
                        return;
                    }

                    if (isViewStateStoredOnServer(val)) {
                        return;
                    }

                    // If the ViewState is not secured cryptographic
                    // protections then raise an alert.
                    if (!isViewStateSecure(val, msg.getRequestBody().getCharset())) {
                        raiseAlert(msg, id, src);
                    }
                }
            }
        }
    }
}