Java Code Examples for us.codecraft.webmagic.Task#getUUID()

The following examples show how to use us.codecraft.webmagic.Task#getUUID() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: FilePipeline.java From webmagic with Apache License 2.0

6 votes

@Override
public void process(ResultItems resultItems, Task task) {
    String path = this.path + PATH_SEPERATOR + task.getUUID() + PATH_SEPERATOR;
    try {
        PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(getFile(path + DigestUtils.md5Hex(resultItems.getRequest().getUrl()) + ".html")),"UTF-8"));
        printWriter.println("url:\t" + resultItems.getRequest().getUrl());
        for (Map.Entry<String, Object> entry : resultItems.getAll().entrySet()) {
            if (entry.getValue() instanceof Iterable) {
                Iterable value = (Iterable) entry.getValue();
                printWriter.println(entry.getKey() + ":");
                for (Object o : value) {
                    printWriter.println(o);
                }
            } else {
                printWriter.println(entry.getKey() + ":\t" + entry.getValue());
            }
        }
        printWriter.close();
    } catch (IOException e) {
        logger.warn("write file error", e);
    }
}

Example 2

Source File: JsonFilePageModelPipeline.java From webmagic with Apache License 2.0

6 votes

@Override
public void process(Object o, Task task) {
    String path = this.path + PATH_SEPERATOR + task.getUUID() + PATH_SEPERATOR;
    try {
        String filename;
        if (o instanceof HasKey) {
            filename = path + ((HasKey) o).key() + ".json";
        } else {
            filename = path + DigestUtils.md5Hex(ToStringBuilder.reflectionToString(o)) + ".json";
        }
        PrintWriter printWriter = new PrintWriter(new FileWriter(getFile(filename)));
        printWriter.write(JSON.toJSONString(o));
        printWriter.close();
    } catch (IOException e) {
        logger.warn("write file error", e);
    }
}

Example 3

Source File: FilePageModelPipeline.java From webmagic with Apache License 2.0

6 votes

@Override
public void process(Object o, Task task) {
    String path = this.path + PATH_SEPERATOR + task.getUUID() + PATH_SEPERATOR;
    try {
        String filename;
        if (o instanceof HasKey) {
            filename = path + ((HasKey) o).key() + ".html";
        } else {
            filename = path + DigestUtils.md5Hex(ToStringBuilder.reflectionToString(o)) + ".html";
        }
        PrintWriter printWriter = new PrintWriter(new FileWriter(getFile(filename)));
        printWriter.write(ToStringBuilder.reflectionToString(o));
        printWriter.close();
    } catch (IOException e) {
        logger.warn("write file error", e);
    }
}

Example 4

Source File: RedisScheduler.java From webmagic with Apache License 2.0

6 votes

@Override
public synchronized Request poll(Task task) {
    Jedis jedis = pool.getResource();
    try {
        String url = jedis.lpop(getQueueKey(task));
        if (url == null) {
            return null;
        }
        String key = ITEM_PREFIX + task.getUUID();
        String field = DigestUtils.shaHex(url);
        byte[] bytes = jedis.hget(key.getBytes(), field.getBytes());
        if (bytes != null) {
            Request o = JSON.parseObject(new String(bytes), Request.class);
            return o;
        }
        Request request = new Request(url);
        return request;
    } finally {
        pool.returnResource(jedis);
    }
}

Example 5

Source File: JsonFilePipeline.java From webmagic with Apache License 2.0

5 votes

@Override
public void process(ResultItems resultItems, Task task) {
    String path = this.path + PATH_SEPERATOR + task.getUUID() + PATH_SEPERATOR;
    try {
        PrintWriter printWriter = new PrintWriter(new FileWriter(getFile(path + DigestUtils.md5Hex(resultItems.getRequest().getUrl()) + ".json")));
        printWriter.write(JSON.toJSONString(resultItems.getAll()));
        printWriter.close();
    } catch (IOException e) {
        logger.warn("write file error", e);
    }
}

Example 6

Source File: RedisConstant.java From tom-crawler with Apache License 2.0

4 votes

public static String getQueueKey(Task task) {
    return QUEUE_PREFIX + task.getUUID();
}

Example 7

Source File: RedisConstant.java From tom-crawler with Apache License 2.0

4 votes

public static String getItemKey(Task task) {
    return ITEM_PREFIX + task.getUUID();
}

Example 8

Source File: RedisConstant.java From tom-crawler with Apache License 2.0

4 votes

public static String getRunKey(Task task) {
    return RUN_PREFIX + task.getUUID();
}

Example 9

Source File: RedisConstant.java From tom-crawler with Apache License 2.0

4 votes

public static String getBitKey(Task task) {
    return BIT_PREFIX + task.getUUID();
}

Example 10

Source File: RedisScheduler.java From webmagic with Apache License 2.0

4 votes

protected String getSetKey(Task task) {
    return SET_PREFIX + task.getUUID();
}

Example 11

Source File: RedisScheduler.java From webmagic with Apache License 2.0

4 votes

protected String getQueueKey(Task task) {
    return QUEUE_PREFIX + task.getUUID();
}

Example 12

Source File: RedisScheduler.java From webmagic with Apache License 2.0

4 votes

protected String getItemKey(Task task) {
    return ITEM_PREFIX + task.getUUID();
}

Example 13

Source File: RedisPriorityScheduler.java From webmagic with Apache License 2.0

4 votes

private String getZsetPlusPriorityKey(Task task)
{
    return ZSET_PREFIX + task.getUUID() + PLUS_PRIORITY_SUFFIX;
}

Example 14

Source File: RedisPriorityScheduler.java From webmagic with Apache License 2.0

4 votes

private String getQueueNoPriorityKey(Task task)
{
    return QUEUE_PREFIX + task.getUUID() + NO_PRIORITY_SUFFIX;
}

Example 15

Source File: RedisPriorityScheduler.java From webmagic with Apache License 2.0

4 votes

private String getZsetMinusPriorityKey(Task task)
{
    return ZSET_PREFIX + task.getUUID() + MINUS_PRIORITY_SUFFIX;
}