net.htmlparser.jericho.Source Java Examples

/**
 * scans the HTTP response for signatures that might indicate the Heartbleed OpenSSL
 * vulnerability
 *
 * @param msg
 * @param id
 * @param source unused
 */
@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    // get the body contents as a String, so we can match against it
    String responseHeaders = msg.getResponseHeader().getHeadersAsString();

    Matcher matcher = openSSLversionPattern.matcher(responseHeaders);
    while (matcher.find()) {
        String fullVersionString = matcher.group(1); // get the full string e.g. OpenSSL/1.0.1e
        String versionNumber = matcher.group(2); // get the version e.g. 1.0.1e

        // if the version matches any of the known vulnerable versions, raise an alert.
        for (String openSSLvulnerableVersion : openSSLvulnerableVersions) {
            if (versionNumber.equalsIgnoreCase(openSSLvulnerableVersion)) {
                raiseAlert(msg, id, fullVersionString);
                return;
            }
        }
    }
}
 

/**
 * Checks the source of the response for HSTS being set via a META tag which is explicitly not
 * supported per the spec (rfc6797).
 *
 * @param source the source of the response to be analyzed.
 * @return returns a string if HSTS was set via META (for use as alert evidence) otherwise
 *     return {@code null}.
 * @see <a href="https://tools.ietf.org/html/rfc6797#section-8.5">RFC 6797 Section 8.5</a>
 */
private String getMetaHSTSEvidence(Source source) {
    List<Element> metaElements = source.getAllElements(HTMLElementName.META);
    String httpEquiv;

    if (metaElements != null) {
        for (Element metaElement : metaElements) {
            httpEquiv = metaElement.getAttributeValue("http-equiv");
            if (STS_HEADER.equalsIgnoreCase(httpEquiv)) {
                return httpEquiv; // This is a META which attempts to define HSTS return it's
                // value
            }
        }
    }
    return null;
}
 

public void setHTMLText(String htmlText) {
    try {
        this.text = htmlText;
        if (formatt) {
            Source segment = new Source(text);
            segment.fullSequentialParse();
            SourceFormatter formatter = new SourceFormatter(
                    segment);
            htmlText = formatter.toString();
        }

        editorPane.read(new StringReader(htmlText), null);
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 

@Override
public void scan(ScriptsPassiveScanner scriptsPassiveScanner, HttpMessage msg, Source source)
        throws ScriptException {
    logger.debug("Zest PassiveScan script: " + this.script.getName());
    this.sps = scriptsPassiveScanner;
    this.msg = msg;

    try {
        // Create the previous request so the script has something to run against
        ZestRequest req = ZestZapUtils.toZestRequest(msg, false, true, extension.getParam());
        req.setResponse(ZestZapUtils.toZestResponse(msg));

        this.run(script.getZestScript(), req, null);

    } catch (Exception e) {
        throw new ScriptException(e);
    }
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    long start = System.currentTimeMillis();

    if (isXPoweredByHeaderExist(msg)) {
        List<String> xpbHeaders = getXPoweredByHeaders(msg);
        raiseAlert(msg, id, xpbHeaders);
        if (logger.isDebugEnabled()) {
            logger.debug(
                    "\tScan of record "
                            + id
                            + " took "
                            + (System.currentTimeMillis() - start)
                            + " ms");
        }
    }
}
 

protected void locateView ()
{
    if ( !locate )
    {
        return;
    }

    final WebSyntaxArea syntaxArea = editors.get ( editorTabs.getSelectedIndex () );
    final String xml = syntaxArea.getText ();

    final Source xmlSource = new Source ( xml );
    xmlSource.setLogger ( null );
    xmlSource.fullSequentialParse ();

    final StartTag tag = xmlSource.getPreviousStartTag ( syntaxArea.getCaretPosition (), "style" );
    if ( tag != null )
    {
        // todo Won't work with new scheme, have to go all the way up and gather all style identifiers
        final String type = tag.getAttributeValue ( ComponentStyleConverter.COMPONENT_TYPE_ATTRIBUTE );
        final String id = tag.getAttributeValue ( ComponentStyleConverter.STYLE_ID_ATTRIBUTE );
        locateView ( previewPanel, type, id );
    }
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    if (msg.getResponseHeader().getStatusCode() != HttpStatusCode.OK
            || isHttps(msg)
            || !isResponseHTML(msg, source)) {
        return;
    }

    List<Element> formElements = source.getAllElements(HTMLElementName.FORM);
    for (Element formElement : formElements) {
        String formAction = formElement.getAttributeValue("action");
        if (formAction != null && formAction.trim().toLowerCase().startsWith("https://")) {
            raiseAlert(msg, id, formElement);
        }
    }
}
 

@Override
public void actionPerformed(ActionEvent e) {
    if (formatt == ((Boolean) getValue(SELECTED_KEY)))
        return;
    formatt = (Boolean) getValue(SELECTED_KEY);
    putValue(SELECTED_KEY, formatt);
    try {
        String text = HTMLView.this.text;
        if (formatt) {
            SourceFormatter formatter = new SourceFormatter(new Source(
                    text));
            text = formatter.toString();
        }

        editorPane.read(new StringReader(text), null);
    } catch (IOException ex) {
        ex.printStackTrace();
    }
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    if (msg.getResponseHeader().getStatusCode() != 200 || !isResponseHTML(msg, source)) {
        return;
    }

    List<Element> htmlElements = source.getAllElements();
    if (htmlElements.size() == 0) {
        return;
    }

    Set<HtmlParameter> params = new TreeSet<HtmlParameter>(msg.getFormParams());
    params.addAll(msg.getUrlParams());
    if (params.size() == 0) {
        return;
    }

    checkHtmlElements(msg, id, params, htmlElements);
}
 

private void checkXmlEncodingCharset(
        HttpMessage msg, int id, Source source, Set<HtmlParameter> params) {
    List<StartTag> xmlDeclarationTags = source.getAllStartTags(StartTagType.XML_DECLARATION);
    if (xmlDeclarationTags.size() == 0) {
        return;
    }

    StartTag xmlDeclarationTag = xmlDeclarationTags.get(0);
    String encoding = xmlDeclarationTag.getAttributeValue("encoding");

    if (encoding == null || encoding.equals("")) {
        return;
    }

    for (HtmlParameter param : params) {
        if (encoding.equalsIgnoreCase(param.getValue())) {
            raiseAlert(msg, id, "\\?xml", "encoding", param, encoding);
        }
    }
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {

    List<Element> sourceElements = source.getAllElements();
    sourceElements.stream()
            .filter(element -> SupportedElements.contains(element.getName()))
            .filter(unsafeSubResource(msg.getRequestHeader().getHostName()))
            .forEach(
                    element -> {
                        newAlert()
                                .setRisk(Alert.RISK_MEDIUM)
                                .setConfidence(Alert.CONFIDENCE_HIGH)
                                .setDescription(getString("desc"))
                                .setSolution(getString("soln"))
                                .setReference(getString("refs"))
                                .setEvidence(element.toString())
                                .setCweId(16) // CWE CATEGORY: Configuration
                                .setWascId(15) // Application Misconfiguration
                                .raise();
                    });
}
 

/**
 * {@inheritDoc}
 */
public String getContent(String eventResource) {
    String reference = getReferenceFromEventResource(eventResource);
    EntityProviderManager entityProviderManager = ComponentManager.get(EntityProviderManager.class);
    EntityReference er= new EntityReference("/sam_publisheditem/"+reference);
    PublishedItemEntityProviderImpl qhp= (PublishedItemEntityProviderImpl)entityProviderManager.getProviderByPrefix(er.getPrefix());
    try {
        PublishedItemFacade item = (PublishedItemFacade)qhp.getEntity(er);
        String content = qhp.content(item);
        //We will filter the HTML here just before send to the index
        Source parseContent = new Source(content);
        return parseContent.getTextExtractor().toString();
    } catch (Exception e) {
        throw new RuntimeException(" Failed to get item content ", e);
    }
}
 

@Override
public boolean parseResource(HttpMessage message, Source source, int depth) {
    try {
        Converter converter =
                new SwaggerConverter(
                        null,
                        message.getRequestHeader().getURI().toString(),
                        message.getResponseBody().toString(),
                        this.getValueGenerator());
        requestor.run(converter.getRequestModels());
    } catch (Exception e) {
        log.debug(e.getMessage(), e);
        return false;
    }

    return true;
}
 

/**
 * {@inheritDoc}
 */
public String getContent(String eventResource) {
    String reference = getReferenceFromEventResource(eventResource);
    EntityProviderManager entityProviderManager = ComponentManager.get(EntityProviderManager.class);
    EntityReference er= new EntityReference("/sam_publisheditem/"+reference);
    PublishedItemEntityProviderImpl qhp= (PublishedItemEntityProviderImpl)entityProviderManager.getProviderByPrefix(er.getPrefix());
    try {
        PublishedItemFacade item = (PublishedItemFacade)qhp.getEntity(er);
        String content = qhp.content(item);
        //We will filter the HTML here just before send to the index
        Source parseContent = new Source(content);
        return parseContent.getTextExtractor().toString();
    } catch (Exception e) {
        throw new RuntimeException(" Failed to get item content ", e);
    }
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    List<HttpCookie> cookies =
            msg.getResponseHeader().getHttpCookies(msg.getRequestHeader().getHostName());

    // name of a host from which the response has been sent from
    String host = msg.getRequestHeader().getHostName();

    // find all loosely scoped cookies
    List<HttpCookie> looselyScopedCookies = new LinkedList<HttpCookie>();
    for (HttpCookie cookie : cookies) {
        if (isLooselyScopedCookie(cookie, host)) {
            looselyScopedCookies.add(cookie);
        }
    }

    // raise alert if have found any loosely scoped cookies
    if (looselyScopedCookies.size() > 0) {
        raiseAlert(msg, id, host, looselyScopedCookies);
    }
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    String responseBody = msg.getResponseBody().toString();
    List<Candidate> candidates = getNumberSequences(responseBody);
    for (Candidate candidate : candidates) {
        for (CreditCard cc : CreditCard.values()) {
            Matcher matcher = cc.matcher(candidate.getCandidate());
            while (matcher.find()) {
                String evidence = matcher.group();
                if (PiiUtils.isValidLuhn(evidence) && !isSci(candidate.getContainingString())) {
                    BinRecord binRec = BinList.getSingleton().get(evidence);
                    raiseAlert(msg, evidence, cc.name, binRec);
                }
            }
        }
    }
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    if (msg.getResponseHeader().getStatusCode() != HttpStatusCode.OK
            || !isHttps(msg)
            || !isResponseHTML(msg, source)) {
        return;
    }

    List<Element> formElements = source.getAllElements(HTMLElementName.FORM);
    for (Element formElement : formElements) {
        String formAction = formElement.getAttributeValue("action");
        if (formAction != null && formAction.trim().toLowerCase().startsWith("http://")) {
            raiseAlert(msg, id, formElement);
        }
    }
}
 

protected void loadFirstResource ( final List<ClassResource> resources, final List<String> xmlContent, final List<String> xmlNames,
                                   final List<ClassResource> xmlFiles ) throws IOException
{
    final ClassResource rf = resources.get ( 0 );
    final Source xmlSource = new Source ( ReflectUtils.getClassSafely ( rf.getClassName () ).getResource ( rf.getPath () ) );
    xmlSource.setLogger ( null );
    xmlSource.fullSequentialParse ();

    final Element baseClassTag = xmlSource.getFirstElement ( SkinInfoConverter.CLASS_NODE );
    final String baseClass = baseClassTag != null ? baseClassTag.getContent ().toString () : null;

    for ( final Element includeTag : xmlSource.getAllElements ( SkinInfoConverter.INCLUDE_NODE ) )
    {
        final String includeClass = includeTag.getAttributeValue ( SkinInfoConverter.NEAR_CLASS_ATTRIBUTE );
        final String finalClass = includeClass != null ? includeClass : baseClass;
        final String src = includeTag.getContent ().toString ();
        resources.add ( new ClassResource ( finalClass, src ) );
    }

    xmlContent.add ( xmlSource.toString () );
    xmlNames.add ( new File ( rf.getPath () ).getName () );
    xmlFiles.add ( rf );

    resources.remove ( 0 );
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    if (msg.getResponseHeader().getStatusCode() == HttpStatusCode.MOVED_PERMANENTLY
            || msg.getResponseHeader().getStatusCode() == HttpStatusCode.FOUND) {
        if (msg.getResponseHeader().getHeader(HttpResponseHeader.LOCATION) != null) {
            Set<HtmlParameter> params = new TreeSet<>(msg.getUrlParams());
            params.addAll(msg.getFormParams());

            if (!params.isEmpty()) {
                checkUserControllableLocationHeaderValue(
                        msg,
                        id,
                        params,
                        msg.getResponseHeader().getHeader(HttpResponseHeader.LOCATION));
            }
        }
    }
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    long start = System.currentTimeMillis();

    List<String> xbsOption = msg.getResponseHeader().getHeaderValues("X-Backend-Server");
    if (!xbsOption.isEmpty()) { // Header Found
        // It is set so lets check it. Should only be one but it's a vector so iterate to be
        // sure.
        for (String xbsDirective : xbsOption) {
            newAlert()
                    .setRisk(Alert.RISK_LOW)
                    .setConfidence(Alert.CONFIDENCE_MEDIUM)
                    .setDescription(getDescription())
                    .setSolution(getSolution())
                    .setReference(getReference())
                    .setEvidence(xbsDirective)
                    .setCweId(200)
                    .setWascId(13)
                    .raise();
        }
    }
    if (logger.isDebugEnabled()) {
        logger.debug(
                "\tScan of record "
                        + id
                        + " took "
                        + (System.currentTimeMillis() - start)
                        + " ms");
    }
}
 

private void checkAppMatches(HttpMessage msg, Source source) {
    checkUrlMatches(msg);
    checkHeadersMatches(msg);
    if (!msg.getResponseHeader().isText()) {
        return; // Don't check body if not text'ish
    }
    checkBodyMatches(msg);
    checkMetaElementsMatches(source);
    checkScriptElementsMatches(source);
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    if (msg.getResponseHeader().getStatusCode() != HttpStatusCode.OK) {
        return;
    }

    String responseBody = msg.getRequestBody().toString();
    if (responseBody == null) {
        return;
    }

    Set<HtmlParameter> params = new TreeSet<>(msg.getFormParams());
    params.addAll(msg.getUrlParams());
    if (params.size() == 0) {
        return;
    }

    if (!isResponseHTML(msg, source) && !isResponseXML(source)) {
        return;
    }

    if (isResponseHTML(msg, source)) {
        checkMetaContentCharset(msg, id, source, params);
    } else if (isResponseXML(source)) {
        checkXmlEncodingCharset(msg, id, source, params);
    }

    checkContentTypeCharset(msg, id, params);
}
 

private void checkScriptElementsMatches(Source source) {
    for (Element scriptElement : source.getAllElements(HTMLElementName.SCRIPT)) {
        for (AppPattern appPattern : currentApp.getScript()) {
            String src = scriptElement.getAttributeValue("src");
            if (src != null && !src.isEmpty()) {
                addIfMatches(appPattern, src);
            }
        }
    }
}
 

private boolean isResponseHTML(HttpMessage message, Source source) {
    String contentType = message.getResponseHeader().getHeader(HttpHeader.CONTENT_TYPE);
    if (contentType == null) {
        return false;
    }

    return contentType.indexOf("text/html") != -1
            || contentType.indexOf("application/xhtml+xml") != -1
            || contentType.indexOf("application/xhtml") != -1;
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    String host = msg.getRequestHeader().getHostName();

    String txtBody = msg.getResponseBody().toString();
    Matcher matcher = patternPrivateIP.matcher(txtBody);
    StringBuilder sbTxtFound = new StringBuilder();
    String firstOne = null;

    while (matcher.find()) {
        if (getAlertThreshold() != AlertThreshold.LOW
                && matcher.group(1).equalsIgnoreCase(host)) {
            continue;
        }

        if (firstOne == null) {
            firstOne = matcher.group();
        }
        sbTxtFound.append(matcher.group()).append("\n");
    }

    if (sbTxtFound.length() != 0) {
        newAlert()
                .setRisk(getRisk())
                .setConfidence(Alert.CONFIDENCE_MEDIUM)
                .setDescription(getDescription())
                .setOtherInfo(sbTxtFound.toString())
                .setSolution(getSolution())
                .setReference(getReference())
                .setEvidence(firstOne)
                .setCweId(200)
                .setWascId(13)
                .raise();
    }
}
 

public Vector<String> getFormInputFields(HttpMessage tokenMsg) {
    Source source = new Source(tokenMsg.getResponseBody().toString());
    List<Element> formElements = source.getAllElements(HTMLElementName.FORM);
    Vector<String> fifs = new Vector<>();

    if (formElements != null && formElements.size() > 0) {
        // Loop through all of the FORM tags

        for (Element formElement : formElements) {
            List<Element> inputElements = formElement.getAllElements(HTMLElementName.INPUT);

            if (inputElements != null && inputElements.size() > 0) {
                // Loop through all of the INPUT elements
                for (Element inputElement : inputElements) {
                    String id = inputElement.getAttributeValue("ID");
                    if (id != null && id.length() > 0) {
                        fifs.add(id);
                    } else {
                        String name = inputElement.getAttributeValue("NAME");
                        if (name != null && name.length() > 0) {
                            fifs.add(name);
                        }
                    }
                }
            }
        }
    }
    return fifs;
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    // At medium or high exclude javascript responses
    if (!AlertThreshold.LOW.equals(this.getAlertThreshold())
            && msg.getResponseHeader().isJavaScript()) {
        return;
    }
    if (msg.getResponseBody().length() > 0 && msg.getResponseHeader().isText()) {
        String parameter;
        if ((parameter = doesResponseContainsDebugErrorMessage(msg.getResponseBody()))
                != null) {
            this.raiseAlert(msg, id, parameter);
        }
    }
}
 

private boolean isResponseHTML(HttpMessage message, Source source) {
    String contentType = message.getResponseHeader().getHeader(HttpHeader.CONTENT_TYPE);
    if (contentType == null) {
        return false;
    }

    return contentType.indexOf("text/html") != -1
            || contentType.indexOf("application/xhtml+xml") != -1
            || contentType.indexOf("application/xhtml") != -1;
}
 

@Override
public void scanHttpResponseReceive(HttpMessage msg, int id, Source source) {
    if (isWsdl(msg)) {
        HttpResponseHeader header = msg.getResponseHeader();
        String contentType = header.getHeader(HttpHeader.CONTENT_TYPE).trim();
        raiseAlert(msg, id, contentType);
    }
}
 

@Override
public boolean isFalsePositive(HttpMessage msg) {
    if (msg.getResponseBody().length() == 0) {
        // No content
        return true;
    }
    if (msg.getResponseHeader().isXml()) {
        // Pretty unlikely to be an htaccess file
        return true;
    }
    if (msg.getResponseHeader().isJson()) {
        // Pretty unlikely to be an htaccess file
        return true;
    }
    if (msg.getResponseHeader().isHtml()) {
        // Double check it does really look like HTML
        try {
            Source src = new Source(msg.getResponseBody().toString());
            if (src.getFirstElement(HTMLElementName.HTML) != null) {
                // Yep, it really looks like HTML
                return true;
            }
        } catch (Exception e) {
            // Ignore exceptions - they indicate its probably not really HTML
        }
    }

    return false;
}