Java Code Examples for com.safframework.tony.common.utils.Preconditions#isBlank()

/**
 * 方便在 pipeline 中往队列中发起爬取任务(进行深度爬取)
 * @param spider
 * @param originalRequest 原始的request，新的request可以继承原始request的header信息
 * @param url
 */
public void push(Spider spider, Request originalRequest, String url) {

    if (spider==null || originalRequest==null || Preconditions.isBlank(url)) {
        return;
    }

    Request request = new Request(url,spider.getName());     // 根据spider的名称来创建request
    Map<String,String> header = originalRequest.getHeader(); // 从原始request中获取header
    if (Preconditions.isNotBlank(header)) {                  // 将原始request的header复制到新的request

        header.forEach((key,value)->{

            request.header(key,value);
        });
    }

    spider.getQueue().push(request);
}
 

@Around("methodAnnotatedWithTrace() || constructorAnnotatedTrace()")
public Object traceMethod(final ProceedingJoinPoint joinPoint) throws Throwable {
    MethodSignature methodSignature = (MethodSignature) joinPoint.getSignature();

    Trace trace = methodSignature.getMethod().getAnnotation(Trace.class);
    if (trace!=null && !trace.enable()) {
        return joinPoint.proceed();
    }

    String className = methodSignature.getDeclaringType().getSimpleName();
    String methodName = methodSignature.getName();
    final StopWatch stopWatch = new StopWatch();
    stopWatch.start();
    Object result = joinPoint.proceed();
    stopWatch.stop();

    if (Preconditions.isBlank(className)) {
        className = "Anonymous class";
    }

    L.i(className, buildLogMessage(methodName, stopWatch.getElapsedTime()));

    return result;
}
 

@Override
public synchronized Request poll(String spiderName) {
    StatefulRedisConnection<String, String> connection = redisClient.connect();
    RedisCommands<String, String> commands = connection.sync();
    try {
        String url = getRequest(commands, spiderName);
        if (Preconditions.isBlank(url)) {
            return null;
        }

        return getExtrasInItem(commands, url, spiderName);
    } finally {
        connection.close();
    }
}
 

public EtcdWatchManager(String etcdStr, String etcdPath) {

        if (Preconditions.isNotBlank(etcdStr)) {
            client = Client.builder().endpoints(etcdStr).build();
        }

        if (Preconditions.isBlank(etcdPath)) {
            this.path = Constant.DEFAULT_REGISTRY_PATH;
        } else {
            this.path = etcdPath;
        }

        vertx = Vertx.vertx();
    }
 

public ZooKeeperWatchManager(String zkStr, String zkPath) {

        if (Preconditions.isNotBlank(zkStr)) {
            log.info("zkStr: {}", zkStr);

            RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
            client = CuratorFrameworkFactory.newClient(zkStr, retryPolicy);
            // 在start方法之后书写具体的操作
            client.start();

            try {
                if (Preconditions.isBlank(zkPath)) {
                    this.path = Constant.DEFAULT_REGISTRY_PATH;
                } else {
                    this.path = zkPath;
                }

                Stat stat = client.checkExists().forPath(zkPath);

                if (stat==null) {
                    client.create().creatingParentContainersIfNeeded().withMode(CreateMode.PERSISTENT).forPath(zkPath);
                }

                znodes = client.getChildren().usingWatcher(this).forPath(zkPath);
            } catch (Exception e) {
                e.printStackTrace();
            }

            if (Preconditions.isNotBlank(znodes)) {

                znodes.forEach(node->{

                    stateMap.put(node, SpiderEngineState.ONLINE);
                });
            }

            vertx = Vertx.vertx();
        }
    }
 

@Override
public String select(Element element) {
    List<Element> elements = selectElements(element);
    if (Preconditions.isBlank(elements)) {
        return null;
    }
    return getValue(elements.get(0));
}
 

private void compileRegex(String regexStr) {
    if (Preconditions.isBlank(regexStr)) {
        throw new IllegalArgumentException("regex must not be empty");
    }
    try {
        this.regex = Pattern.compile(regexStr, Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
        this.regexStr = regexStr;
    } catch (PatternSyntaxException e) {
        throw new IllegalArgumentException("invalid regex "+regexStr, e);
    }
}
 

/**
 * 方便在 pipeline 中往队列中发起爬取任务(进行深度爬取)
 * @param spider
 * @param url
 */
public void push(Spider spider, String url) {

    if (spider==null || Preconditions.isBlank(url)) {
        return;
    }

    Request request = new Request(url,spider.getName());     // 根据spider的名称来创建request
    spider.getQueue().push(request);
}
 

private static LinkedHashMap<String, List<String>> parseQueryString(String query) {
    LinkedHashMap<String, List<String>> params = new LinkedHashMap<String, List<String>>();
    if (Preconditions.isBlank(query)) {
        return params;
    }
    String[] items = query.split("&");
    for (String item : items) {
        String name = substringBefore(item, "=");
        String value = substringAfter(item, "=");
        List<String> values = getOrCreate(params, name);
        values.add(value);
    }
    return params;
}
 

public Proxy(String ip,int port,String scheme) {
    this.ip = ip;
    this.port = port;
    if (Preconditions.isBlank(scheme)) {
        this.scheme = "http";
    } else {
        this.scheme = scheme;
    }
}
 

@Override
public boolean saveProxyResource(ProxyResource proxyResource) {
    boolean result = false;
    if(Preconditions.isBlank(proxyResource.getResId())) {            //insert
        proxyResource.setResId(commonDao.getNextSequence(Constant.COL_NAME_PROXY_RESOURCE).getSequence());
        proxyResource.setAddTime(new Date().getTime());
        proxyResource.setModTime(new Date().getTime());
        mongoTemplate.save(proxyResource, Constant.COL_NAME_PROXY_RESOURCE);

        result = Preconditions.isNotBlank(proxyResource.getId());
    } else {                                                        //update
        Query query = new Query().addCriteria(Criteria.where("resId").is(proxyResource.getResId()));
        Update update = new Update();
        update.set("webName", proxyResource.getWebName());
        update.set("webUrl", proxyResource.getWebUrl());
        update.set("pageCount", proxyResource.getPageCount());
        update.set("prefix", proxyResource.getPrefix());
        update.set("suffix", proxyResource.getSuffix());
        update.set("parser", proxyResource.getParser());
        update.set("modTime", new Date().getTime());

        WriteResult writeResult = mongoTemplate.updateFirst(query, update, Constant.COL_NAME_PROXY_RESOURCE);

        result = writeResult!=null && writeResult.getN() > 0;
    }
    return result;
}
 

/**
 * 将下载的proxy放入代理池
 * @param page
 */
private List<Proxy> handle(Page page){

    if (page == null || Preconditions.isBlank(page.getHtml())){
        return new ArrayList<Proxy>();
    }

    List<Proxy> result = new ArrayList<>();

    ProxyListPageParser parser = ProxyListPageParserFactory.getProxyListPageParser(ProxyPool.proxyMap.get(url));
    if (parser!=null) {

        List<Proxy> proxyList = parser.parse(page.getHtml());
        if(Preconditions.isNotBlank(proxyList)) {

            for(Proxy p : proxyList){

                if (!ProxyPool.proxyList.contains(p)) {
                    result.add(p);
                }
            }
        }

    }

    return result;
}
 

private static boolean isEmpty(String line) {
    return Preconditions.isBlank(line) || N.equals(line) || T.equals(line);
}
 

/**
 * 每隔几个小时跑一次任务
 */
@Scheduled(cron="${cronJob.schedule}")
public void cronJob() {

    //1.检查job的状态
    checkRunningStat();

    log.info("Job Start...");

    //2.获取目标网页的Url
    ProxyPool.proxyMap = proxyDao.getProxyMap();

    //3.如果数据库里没取到，用默认内置的
    if(Preconditions.isBlank(ProxyPool.proxyMap)) {
        log.info("Job proxyDao.getProxyMap() is empty");
        ProxyPool.proxyMap = Constant.proxyMap;
    }

    //4.每次跑job先清空缓存中的内容
    if (cacheManager.getCache("proxys")!=null) {

        cacheManager.getCache("proxys").clear();
    }

    //5.创建一个日志对象，用于存储job的每次工作记录
    JobLog jobLog = new JobLog();
    jobLog.setJobName("ScheduleJobs.cronJob");
    jobLog.setStartTime(JodaUtils.formatDateTime(new Date()));

    //6.跑任务之前先清空proxyList中上一次job留下的proxy数据，
    ProxyPool.proxyList.clear();

    //7.从数据库中选取10个代理作为种子代理，遇到http 503时使用代理来抓数据
    ProxyPool.addProxyList(getProxyList(proxyDao.takeRandomTenProxy()));
    log.info("Job ProxyPool.proxyList size = "+ProxyPool.proxyList.size());
    //8.正式开始，爬代理数据
    proxyManager.start();

    //9.爬完以后，把数据转换为ProxyData并存到数据库
    CopyOnWriteArrayList<ProxyData> list = getProxyDataList(ProxyPool.proxyList);
    log.info("Job ProxyData list size = "+list.size());
    if (Preconditions.isNotBlank(list)) {

        // 10. list的数量<=15时，不删除数据库里的老数据
        if (list.size()>15) {
            proxyDao.deleteAll();
            log.info("Job after deleteAll");
        }

        //11. 然后再进行插入新的proxy
        for (ProxyData p:list) {
            proxyDao.saveProxy(p);
        }
        log.info("Job save count = "+list.size());

        jobLog.setResultDesc(String.format("success save count = %s", list.size()));
        jobLog.setEndTime(JodaUtils.formatDateTime(new Date()));
        commonDao.saveJobLog(jobLog);

    } else {
        log.info("Job proxyList is empty...");
    }

    //12. 设置job状态为停止
    stop();

    log.info("Job End...");
}