org.apache.nutch.metadata.Metadata#add

Source File: TestParseData.java From nutch-htmlunit with Apache License 2.0

6 votes

public void testParseData() throws Exception {

    String title = "The Foo Page";

    Outlink[] outlinks = new Outlink[] {
      new Outlink("http://foo.com/", "Foo"),
      new Outlink("http://bar.com/", "Bar")
    };

    Metadata metaData = new Metadata();
    metaData.add("Language", "en/us");
    metaData.add("Charset", "UTF-8");

    ParseData r = new ParseData(ParseStatus.STATUS_SUCCESS, title, outlinks, metaData);
                        
    WritableTestUtils.testWritable(r, null);
  }

Source File: TestParseData.java From anthelion with Apache License 2.0

6 votes

public void testParseData() throws Exception {

    String title = "The Foo Page";

    Outlink[] outlinks = new Outlink[] {
      new Outlink("http://foo.com/", "Foo"),
      new Outlink("http://bar.com/", "Bar")
    };

    Metadata metaData = new Metadata();
    metaData.add("Language", "en/us");
    metaData.add("Charset", "UTF-8");

    ParseData r = new ParseData(ParseStatus.STATUS_SUCCESS, title, outlinks, metaData);
                        
    WritableTestUtils.testWritable(r, null);
  }

Source File: RelTagParser.java From nutch-htmlunit with Apache License 2.0

6 votes

/**
 * Scan the HTML document looking at possible rel-tags
 */
public ParseResult filter(Content content, ParseResult parseResult,
  HTMLMetaTags metaTags, DocumentFragment doc) {
  
  // get parse obj
  Parse parse = parseResult.get(content.getUrl());
  // Trying to find the document's rel-tags
  Parser parser = new Parser(doc);
  Set<?> tags = parser.getRelTags();
  Iterator<?> iter = tags.iterator();
  Metadata metadata = parse.getData().getParseMeta();
  while (iter.hasNext())
    metadata.add(REL_TAG, (String) iter.next());

  return parseResult;
}

Source File: RelTagParser.java From anthelion with Apache License 2.0

6 votes

/**
 * Scan the HTML document looking at possible rel-tags
 */
public ParseResult filter(Content content, ParseResult parseResult,
  HTMLMetaTags metaTags, DocumentFragment doc) {
  
  // get parse obj
  Parse parse = parseResult.get(content.getUrl());
  // Trying to find the document's rel-tags
  Parser parser = new Parser(doc);
  Set tags = parser.getRelTags();
  Iterator iter = tags.iterator();
  Metadata metadata = parse.getData().getParseMeta();
  while (iter.hasNext()) {
    metadata.add(REL_TAG, (String) iter.next());
  }
  return parseResult;
}

Source File: TestMoreIndexingFilter.java From nutch-htmlunit with Apache License 2.0

5 votes

public void testContentDispositionTitle() throws IndexingException {
  Configuration conf = NutchConfiguration.create();

  Metadata metadata = new Metadata();
  metadata.add(Response.CONTENT_DISPOSITION, "filename=filename.ext");
  MoreIndexingFilter filter = new MoreIndexingFilter();
  filter.setConf(conf);

  NutchDocument doc = filter.filter(new NutchDocument(), new ParseImpl("text", new ParseData(
    new ParseStatus(), "title", new Outlink[0], metadata)), new Text(
      "http://www.example.com/"), new CrawlDatum(), new Inlinks());

  assertEquals("content-disposition not detected", "filename.ext", doc.getFieldValue("title"));
}

Source File: TestIndexingFilters.java From nutch-htmlunit with Apache License 2.0

5 votes

/**
 * Test behaviour when reset the index filter order will not take effect
 *
 * @throws IndexingException
 */
public void testFilterCacheIndexingFilter() throws IndexingException{
  Configuration conf = NutchConfiguration.create();
  conf.addResource("nutch-default.xml");
  conf.addResource("crawl-tests.xml");

  String class1 = "org.apache.nutch.indexer.basic.BasicIndexingFilter";
  conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1);

  IndexingFilters filters1 = new IndexingFilters(conf);
  NutchDocument fdoc1 = filters1.filter(new NutchDocument(),new ParseImpl("text",new ParseData(
    new ParseStatus(),"title",new Outlink[0],new Metadata())),new Text("http://www.example.com/"),
    new CrawlDatum(),new Inlinks());

  // add another index filter
  String class2 = "org.apache.nutch.indexer.metadata.MetadataIndexer";
  // set content metadata
  Metadata md = new Metadata();
  md.add("example","data");
  // set content metadata property defined in MetadataIndexer
  conf.set("index.content.md","example");
  // add MetadataIndxer filter
  conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1 + " " + class2);
  IndexingFilters filters2 = new IndexingFilters(conf);
  NutchDocument fdoc2 = filters2.filter(new NutchDocument(),new ParseImpl("text",new ParseData(
    new ParseStatus(),"title",new Outlink[0],md)),new Text("http://www.example.com/"),
    new CrawlDatum(),new Inlinks());
  assertEquals(fdoc1.getFieldNames().size(),fdoc2.getFieldNames().size());
}

Source File: FeedParser.java From nutch-htmlunit with Apache License 2.0

5 votes

private void mergeMetadata(Metadata first, Metadata second) {
  for (String name : second.names()) {
    String[] values = second.getValues(name);
    for (String value : values) {
      first.add(name, value);
    }
  }
}

Source File: TestBasicIndexingFilter.java From nutch-htmlunit with Apache License 2.0

5 votes

public void testBasicIndexingFilter() throws Exception { 
  Configuration conf = NutchConfiguration.create();
  conf.setInt("indexer.max.title.length", 10);
  conf.setBoolean("indexer.add.domain", true);
  conf.setInt("indexer.max.content.length", 20);

  BasicIndexingFilter filter = new BasicIndexingFilter();
  filter.setConf(conf);
  assertNotNull(filter);

  NutchDocument doc = new NutchDocument();

  String title = "The Foo Page";
  Outlink[] outlinks = new Outlink[] { new Outlink("http://foo.com/", "Foo") };
  Metadata metaData = new Metadata();
  metaData.add("Language", "en/us");
  ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, title, outlinks, metaData);
  ParseImpl parse = new ParseImpl("this is a sample foo bar page. hope you enjoy it.", parseData);

  CrawlDatum crawlDatum = new CrawlDatum();
  crawlDatum.setFetchTime(100L);

  Inlinks inlinks = new Inlinks();

  try {
    filter.filter(doc, parse, new Text("http://nutch.apache.org/index.html"), crawlDatum, inlinks);
  } catch(Exception e){
    e.printStackTrace();
    fail(e.getMessage());
  }
  assertNotNull(doc);
  assertEquals("test title, expect \"The Foo Pa\"", "The Foo Pa", doc.getField("title").getValues().get(0));
  assertEquals("test domain, expect \"apache.org\"", "apache.org", doc.getField("domain").getValues().get(0));
  assertEquals("test host, expect \"nutch.apache.org\"", "nutch.apache.org", doc.getField("host").getValues().get(0));
  assertEquals("test url, expect \"http://nutch.apache.org/index.html\"", "http://nutch.apache.org/index.html", 
    doc.getField("url").getValues().get(0));
  assertEquals("test content", "this is a sample foo", doc.getField("content").getValues().get(0));
  assertEquals("test fetch time", new Date(100L), (Date)doc.getField("tstamp").getValues().get(0));
}

Source File: TestMoreIndexingFilter.java From nutch-htmlunit with Apache License 2.0

5 votes

private void assertContentType(Configuration conf, String source, String expected) throws IndexingException {
  Metadata metadata = new Metadata();
  metadata.add(Response.CONTENT_TYPE, source);
  MoreIndexingFilter filter = new MoreIndexingFilter();
  filter.setConf(conf);
  NutchDocument doc = filter.filter(new NutchDocument(), new ParseImpl("text", new ParseData(
      new ParseStatus(), "title", new Outlink[0], metadata)), new Text(
      "http://www.example.com/"), new CrawlDatum(), new Inlinks());
  assertEquals("mime type not detected", expected, doc.getFieldValue("type"));
}

Source File: FeedParser.java From anthelion with Apache License 2.0

5 votes

private void mergeMetadata(Metadata first, Metadata second) {
  for (String name : second.names()) {
    String[] values = second.getValues(name);
    for (String value : values) {
      first.add(name, value);
    }
  }
}

Source File: TestMoreIndexingFilter.java From anthelion with Apache License 2.0

5 votes

private void assertContentType(Configuration conf, String source, String expected) throws IndexingException {
  Metadata metadata = new Metadata();
  metadata.add(Response.CONTENT_TYPE, source);
  MoreIndexingFilter filter = new MoreIndexingFilter();
  filter.setConf(conf);
  NutchDocument doc = filter.filter(new NutchDocument(), new ParseImpl("text", new ParseData(
      new ParseStatus(), "title", new Outlink[0], metadata)), new Text(
      "http://www.example.com/"), new CrawlDatum(), new Inlinks());
  assertEquals("mime type not detected", expected, doc.getFieldValue("type"));
}

Source File: TestMoreIndexingFilter.java From anthelion with Apache License 2.0

5 votes

public void testContentDispositionTitle() throws IndexingException {
  Configuration conf = NutchConfiguration.create();

  Metadata metadata = new Metadata();
  metadata.add(Response.CONTENT_DISPOSITION, "filename=filename.ext");
  MoreIndexingFilter filter = new MoreIndexingFilter();
  filter.setConf(conf);

  NutchDocument doc = filter.filter(new NutchDocument(), new ParseImpl("text", new ParseData(
    new ParseStatus(), "title", new Outlink[0], metadata)), new Text(
      "http://www.example.com/"), new CrawlDatum(), new Inlinks());

  assertEquals("content-disposition not detected", "filename.ext", doc.getFieldValue("title"));
}

Source File: CCParseFilter.java From anthelion with Apache License 2.0

4 votes

/** Scan the document adding attributes to metadata.*/
public static void walk(Node doc, URL base, Metadata metadata, Configuration conf)
  throws ParseException {

  // walk the DOM tree, scanning for license data
  Walker walker = new Walker(base);
  walker.walk(doc);

  // interpret results of walk
  String licenseUrl = null;
  String licenseLocation = null;
  if (walker.rdfLicense != null) {            // 1st choice: subject in RDF
    licenseLocation = "rdf";
    licenseUrl = walker.rdfLicense;
  } else if (walker.relLicense != null) {     // 2nd: anchor w/ rel=license
    licenseLocation = "rel";
    licenseUrl = walker.relLicense.toString();
  } else if (walker.anchorLicense != null) {  // 3rd: anchor w/ CC license
    licenseLocation = "a";
    licenseUrl = walker.anchorLicense.toString();
  } else if (conf.getBoolean("creativecommons.exclude.unlicensed", false)) {
    throw new ParseException("No CC license.  Excluding.");
  }

  // add license to metadata
  if (licenseUrl != null) {
    if (LOG.isInfoEnabled()) {
      LOG.info("CC: found "+licenseUrl+" in "+licenseLocation+" of "+base);
    }
    metadata.add(CreativeCommons.LICENSE_URL, licenseUrl);
    metadata.add(CreativeCommons.LICENSE_LOCATION, licenseLocation);
  }

  if (walker.workType != null) {
    if (LOG.isInfoEnabled()) {
      LOG.info("CC: found "+walker.workType+" in "+base);
    }
    metadata.add(CreativeCommons.WORK_TYPE, walker.workType);
  }

}

Source File: TestHTMLLanguageParser.java From anthelion with Apache License 2.0

4 votes

private Content getContent(String text) {
  Metadata meta = new Metadata();
  meta.add("Content-Type", "text/html");
  return new Content(URL, BASE, text.getBytes(), "text/html", meta, NutchConfiguration.create());
}

Source File: TestHTMLLanguageParser.java From nutch-htmlunit with Apache License 2.0

4 votes

private Content getContent(String text) {
  Metadata meta = new Metadata();
  meta.add("Content-Type", "text/html");
  return new Content(URL, BASE, text.getBytes(), "text/html", meta, NutchConfiguration.create());
}

Source File: CCParseFilter.java From nutch-htmlunit with Apache License 2.0

4 votes

/** Scan the document adding attributes to metadata.*/
public static void walk(Node doc, URL base, Metadata metadata, Configuration conf)
  throws ParseException {

  // walk the DOM tree, scanning for license data
  Walker walker = new Walker(base);
  walker.walk(doc);

  // interpret results of walk
  String licenseUrl = null;
  String licenseLocation = null;
  if (walker.rdfLicense != null) {            // 1st choice: subject in RDF
    licenseLocation = "rdf";
    licenseUrl = walker.rdfLicense;
  } else if (walker.relLicense != null) {     // 2nd: anchor w/ rel=license
    licenseLocation = "rel";
    licenseUrl = walker.relLicense.toString();
  } else if (walker.anchorLicense != null) {  // 3rd: anchor w/ CC license
    licenseLocation = "a";
    licenseUrl = walker.anchorLicense.toString();
  } else if (conf.getBoolean("creativecommons.exclude.unlicensed", false)) {
    throw new ParseException("No CC license.  Excluding.");
  }

  // add license to metadata
  if (licenseUrl != null) {
    if (LOG.isInfoEnabled()) {
      LOG.info("CC: found "+licenseUrl+" in "+licenseLocation+" of "+base);
    }
    metadata.add(CreativeCommons.LICENSE_URL, licenseUrl);
    metadata.add(CreativeCommons.LICENSE_LOCATION, licenseLocation);
  }

  if (walker.workType != null) {
    if (LOG.isInfoEnabled()) {
      LOG.info("CC: found "+walker.workType+" in "+base);
    }
    metadata.add(CreativeCommons.WORK_TYPE, walker.workType);
  }

}

Source File: S2jhHtmlParseFilter.java From nutch-htmlunit with Apache License 2.0

4 votes

@Override
public ParseResult filterInternal(Content content, ParseResult parseResult, HTMLMetaTags metaTags,
        DocumentFragment doc) {
    String url = content.getUrl();
    Parse parse = parseResult.get(new Text(url));
    //String text = parse.getText();
    ParseData parseData = parse.getData();

    //后续的index索引之用元数据
    Metadata parseMeta = parseData.getParseMeta();
    //数据库需要采集记录的数据信息集合
    //一个属性一行数据
    List<CrawlData> crawlDatas = Lists.newArrayList();

    String sku = null;
    Matcher matcher = this.skuMatchPattern.matcher(url);
    if (matcher.find()) {
        sku = matcher.group(1);
    }
    if (StringUtils.isBlank(sku)) {
        LOG.warn("SKU not parsed for url: " + url);
        return parseResult;
    }

    //数据库记录
    crawlDatas.add(new CrawlData(url, "sku").setTextValue(sku));
    //元数据数据
    parseMeta.add("sku", sku);

    String price = getXPathValue(doc, "//SPAN[@class='tm-price']");
    crawlDatas.add(new CrawlData(url, "price").setTextValue(price));
    parseMeta.add("price", price);

    LOG.info(" - SKU：{}, Parse Meta: {}", sku, parseMeta);

    NodeList nodes = selectNodeList(doc,
            "//DIV[@id='description']/DIV[@class='content ke-post']//IMG[@data-ks-lazyload]");
    LOG.info("Product description content image list: ");
    for (int i = 0; i < nodes.getLength(); i++) {
        Node node = nodes.item(i);
        String imgUrl = node.getAttributes().getNamedItem("data-ks-lazyload").getTextContent();
        LOG.info(" - {}", imgUrl);
    }

    saveCrawlData(url, crawlDatas);

    return parseResult;
}

Java Code Examples for org.apache.nutch.metadata.Metadata#add()