Java Code Examples for it.unimi.dsi.lang.MutableString#toLowerCase()

/**
 * It normalizes the anchor text:
 * 1. ascii normalization
 * 2. delete brackets at the end (i.e. in titles)
 * 3. delete a pattern at the beginning of the text, see anchorStart
 * 4. delete all dots '.'
 * 5. replace all punctuations with whitespaces, except for {@link WikipediaAnchorParser#SPECIAL_PUNCTS}
 * 6. if the original contained any of {@link WikipediaAnchorParser#SPECIAL_PUNCTS},
 * 	  it returns 2 anchors (with those puncts replaced by whitespace, and with those puncts deleted)
 *    otherwise, the normalization at 5.
 * If the normalization process doesn't produce a valid anchor, an empty array is returned
 * @param original
 * @param anchorStart A pattern that identifies some common articles or preposition to be deleted if they occur at the beginning of the anchor
 * @return
 */
public static CharSequence[] parseAnchor(CharSequence original, Pattern anchorStart)
{

	MutableString anchor = Chars.toNormalizedASCII(original);
	anchor.squeezeSpace();
	anchor.trim();

	if (anchor.length() < MIN_ANCHOR_LEN || !contaisText(anchor)) return Chars.EMPTY_STRINGS;

	anchor.loose();
	anchor.toLowerCase();

	Matcher m = P_FINAL_BRACKETS.matcher(anchor);
	if (m.find()) anchor.delete(m.start(), m.end());
	anchor.trim();
	if (anchor.length() < MIN_ANCHOR_LEN) return Chars.EMPTY_STRINGS;

	if (anchorStart != null)
	{
		Matcher m2 = anchorStart.matcher(anchor);
		if (m2.find()) anchor.delete(m2.start(), m2.end());
		anchor.trim();
		if (anchor.length() < MIN_ANCHOR_LEN) return Chars.EMPTY_STRINGS;
	}

	anchor = removeDots(anchor);
	if (anchor.length() < MIN_ANCHOR_LEN) return Chars.EMPTY_STRINGS;

	anchor = removePunctuations(anchor, SPECIAL_PUNCTS, false);
	if (anchor.length() < MIN_ANCHOR_LEN) return Chars.EMPTY_STRINGS;

	if (anchor.indexOfAnyOf(SPECIAL_PUNCTS_CHARS, 0)<0){
		if (!contaisText(anchor)) return Chars.EMPTY_STRINGS;
		else return new CharSequence[]{anchor};
	} else {
		MutableString anchorNoPuncts = new MutableString(anchor);
		anchor.replace(SPECIAL_PUNCTS_CHARS, SPECIAL_PUNCTS_CHAR_MAP);
		//			Chars.trimMultispace(anchor);
		anchor.squeezeSpace().trim();
		if (anchor.length() < MIN_ANCHOR_LEN || !contaisText(anchor)) return Chars.EMPTY_STRINGS;

		anchorNoPuncts.delete(SPECIAL_PUNCTS_CHARS);
		anchorNoPuncts.squeezeSpace().trim();
		if (anchorNoPuncts.length() < MIN_ANCHOR_LEN || !contaisText(anchorNoPuncts)) return new CharSequence[]{anchor};
		else return new CharSequence[]{anchor, anchorNoPuncts};
	}
}
 

public boolean startElement( final Element element, final Map<Attribute,MutableString> attrMap ) {
	Object s;

	// TODO: what about IMG?
	
	if ( element == Element.A || element == Element.AREA || element == Element.LINK ) {
		s = attrMap.get( Attribute.HREF );
		if ( s != null )
			urls.add( s.toString() );
	}

	// IFRAME or FRAME + SRC
	if ( element == Element.IFRAME || element == Element.FRAME || element == Element.EMBED ) {
		s = attrMap.get( Attribute.SRC );
		if ( s != null )
			urls.add( s.toString() );
	}

	// BASE + HREF (change context!)
	if ( element == Element.BASE && base == null ) {
		s = attrMap.get( Attribute.HREF );
		if ( s != null )
			base = s.toString();
	}

	// META REFRESH/LOCATION
	if ( element == Element.META ) {
		final MutableString equiv = attrMap.get( Attribute.HTTP_EQUIV );
		final MutableString content = attrMap.get( Attribute.CONTENT );
		if ( equiv != null && content != null ) {
			equiv.toLowerCase();

			// http-equiv="refresh" content="0;URL=http://foo.bar/..."
			if ( equiv.equals( "refresh" ) && ( metaRefresh == null ) ) {

				final int pos = URLEQUAL_PATTERN.search( content );
				if ( pos != -1 )
					metaRefresh = content.substring( pos + URLEQUAL_PATTERN.length() ).toString();
			}

			// http-equiv="location" content="http://foo.bar/..."
			if ( equiv.equals( "location" ) && ( metaLocation == null ) )
				metaLocation = attrMap.get( Attribute.CONTENT ).toString();
		}
	}

	return true;
}