Java Code Examples for com.safframework.tony.common.utils.Preconditions#isNotBlank()

@Override
public void process(ResultItems resultItems) {
    StringBuilder sb = new StringBuilder();

    for (Map.Entry<String, Object> entry : resultItems.getAll().entrySet()) {
        sb.append(entry.getValue()).append(",");
    }

    String[] ss = sb.toString().split(",");

    if (Preconditions.isNotBlank(ss)) {

        List<String> dataList = Arrays.asList(ss);
        SpiderUtils.exportCsv(csvFile,dataList, Charset.forName("GBK"));
    }
}
 

/**
 * 方便在 pipeline 中往队列中发起爬取任务(进行深度爬取)
 * @param spider
 * @param originalRequest 原始的request，新的request可以继承原始request的header信息
 * @param url
 */
public void push(Spider spider, Request originalRequest, String url) {

    if (spider==null || originalRequest==null || Preconditions.isBlank(url)) {
        return;
    }

    Request request = new Request(url,spider.getName());     // 根据spider的名称来创建request
    Map<String,String> header = originalRequest.getHeader(); // 从原始request中获取header
    if (Preconditions.isNotBlank(header)) {                  // 将原始request的header复制到新的request

        header.forEach((key,value)->{

            request.header(key,value);
        });
    }

    spider.getQueue().push(request);
}
 

@Override
    public void onBindViewHolder(final SpitalConvAdapter.ViewHolder holder, int position) {

        if (position == 0) {
            holder.image.setImageBitmap(mBitmap);
        } else {
            String filterName = mList.get(position);
            if (Preconditions.isNotBlank(filterName)) {
                CommonFilter filter = (CommonFilter)getFilter(filterName);
                RxImageData.bitmap(mBitmap)
//                        .placeHolder(R.drawable.test_spital_conv)
                        .addFilter(filter)
                        .into(holder.image);
            }
        }

        holder.text.setText(map.get(position));
    }
 

@Override
public void execute(JobExecutionContext context) throws JobExecutionException {
    log.info("定时任务开始");

    log.info("jobName="+context.getJobDetail().getKey().getName());

    JobDataMap dataMap = context.getJobDetail().getJobDataMap();
    Spider spider = (Spider) dataMap.get("spider");
    Request[] requests = (Request[]) dataMap.get("requests");

    if (spider!=null && Preconditions.isNotBlank(requests)) {

        log.info("spiderName="+spider.getName());

        if (spider.getSpiderStatus() == Spider.SPIDER_STATUS_INIT
                || spider.getSpiderStatus() == Spider.SPIDER_STATUS_STOPPED) {

            spider.run();
        } else if (spider.getSpiderStatus() == Spider.SPIDER_STATUS_PAUSE) {

            spider.resume();
        }

        Stream.of(requests).forEach(request -> spider.getQueue().pushToRunninSpider(request,spider));
    }
}
 

private Spider() {
    String queueType = SpiderConfig.getInstance().getQueueType();
    if (Preconditions.isNotBlank(queueType)) {
        switch (queueType) {
            case Constant.QUEUE_TYPE_DEFAULT:
                this.queue = new DefaultQueue();
                break;
            case Constant.QUEUE_TYPE_DISRUPTOR:
                this.queue = new DisruptorQueue();
                break;
            default:
                break;
        }
    }

    if (this.queue == null) {
        this.queue = new DefaultQueue();
    }

    initSpiderConfig();
}
 

/**
 * 导出csv
 *
 * @param file
 * @param dataList
 * @param charset
 * @return
 */
public static boolean exportCsv(File file, List<String> dataList, Charset charset) {

    boolean isSucess = false;

    FileOutputStream out = null;
    OutputStreamWriter osw = null;
    BufferedWriter bw = null;
    try {
        out = new FileOutputStream(file);
        osw = new OutputStreamWriter(out, charset);
        bw = new BufferedWriter(osw);
        if (Preconditions.isNotBlank(dataList)) {
            for (String data : dataList) {
                bw.append(data).append("\r");
            }
        }
        isSucess = true;
    } catch (Exception e) {
        isSucess = false;
    } finally {
        IOUtils.closeQuietly(bw,osw,out);
    }

    return isSucess;
}
 

/**
 * 下载多个网页的全部图片
 * @param urls
 */
public void downloadWebPageImages(List<String> urls) {

    if (Preconditions.isNotBlank(urls)) {

        isWebPage = true;

        pageParser = new PicParser();

        Flowable.fromIterable(urls)
                .parallel()
                .map(url->httpManager.createHttpWithGet(url))
                .map(response->parseHtmlToImages(response,(PicParser)pageParser))
                .sequential()
                .subscribe(list -> downloadPics(list),
                        throwable-> System.out.println(throwable.getMessage()));
    }
}
 

public Request ua(String userAgent) {

        this.userAgent = userAgent;
        if (Preconditions.isNotBlank(userAgent)) {
            header.put("User-Agent", userAgent);
        }

        return this;
    }
 

private boolean hasExtraRequestInfo(Request request) {

        if (request == null) {
            return false;
        }

        return Preconditions.isNotBlank(request.getHeader())
                || Preconditions.isNotBlank(request.getCharset())
                || Preconditions.isNotBlank(request.getExtras())
                || request.getPriority() > 0;
    }
 

public Spider url(Charset charset, String... urls) {
    checkIfRunning();

    if (Preconditions.isNotBlank(urls)) {

        Arrays.asList(urls)
                .stream()
                .forEach(url -> pushToQueue(url,charset));

        signalNewRequest();
    }

    return this;
}
 

@Override
public String get() {
    if (Preconditions.isNotBlank(all())) {
        return all().get(0);
    } else {
        return null;
    }
}
 

public static String getUserAgent() {

        if (Preconditions.isNotBlank(uas)) {

            int index=(int)(Math.random() * uas.size());
            return uas.get(index);
        } else {

            return Constant.defaultUAArray[new Random().nextInt(Constant.defaultUAArray.length)];
        }
    }
 

@Override
public List<ProxyData> findProxyByCond(QueryProxyDTO queryProxyDTO, boolean isGetAll) {
    Query query = new Query();
    if(Preconditions.isNotBlank(queryProxyDTO.getType()) && !"all".equals(queryProxyDTO.getType())) {
        query.addCriteria(Criteria.where("proxyType").is(queryProxyDTO.getType()));
    }
    if(Preconditions.isNotBlank(queryProxyDTO.getIp()) && !"all".equals(queryProxyDTO.getIp())) {
        query.addCriteria(Criteria.where("proxyAddress").regex(".*?"+queryProxyDTO.getIp()+".*"));
    }
    if(queryProxyDTO.getMinPort() != null) {
        query.addCriteria(Criteria.where("proxyPort").gte(queryProxyDTO.getMinPort()).lte(queryProxyDTO.getMaxPort()));
    }
    if(!isGetAll) {
        if(Preconditions.isNotBlank(queryProxyDTO.getSort())) {
            if("asc".equals(queryProxyDTO.getOrder())) {
                query.with(new Sort(Sort.Direction.ASC, queryProxyDTO.getSort()));
            } else {
                query.with(new Sort(Sort.Direction.DESC, queryProxyDTO.getSort()));
            }
        } else {
            query.with(new Sort(Sort.Direction.DESC, "lastSuccessfulTime"));
            query.with(new Sort(Sort.Direction.ASC, "proxyPort"));
        }
        int skip = (queryProxyDTO.getPage() - 1) * queryProxyDTO.getRows();
        query.skip(skip);
        query.limit(queryProxyDTO.getRows());
    }
    return mongoTemplate.find(query, ProxyData.class, Constant.COL_NAME_PROXY);
}
 

private WebClientOptions initWebClientOptions(Request request) {

        WebClientOptions options = new WebClientOptions();
        options.setKeepAlive(SpiderConfig.getInstance().isKeepAlive())
                .setReuseAddress(SpiderConfig.getInstance().isReuseAddress())
                .setFollowRedirects(SpiderConfig.getInstance().isFollowRedirects())
                .setConnectTimeout(SpiderConfig.getInstance().getConnectTimeout())
                .setIdleTimeout(SpiderConfig.getInstance().getIdleTimeout())
                .setMaxWaitQueueSize(SpiderConfig.getInstance().getMaxWaitQueueSize());

        if (Preconditions.isNotBlank(request.getUserAgent())) {
            options.setUserAgent(request.getUserAgent());
        }

        if (Preconditions.isNotBlank(request.getProxy())) {

            ProxyOptions proxyOptions = new ProxyOptions();
            proxyOptions.setHost(request.getProxy().getIp());
            proxyOptions.setPort(request.getProxy().getPort());
            options.setProxyOptions(proxyOptions);
        }

        if (Preconditions.isNotBlank(request.getHeader())) {

            header = request.getHeader();
        }

        return options;
    }
 

public EtcdWatchManager(String etcdStr, String etcdPath) {

        if (Preconditions.isNotBlank(etcdStr)) {
            client = Client.builder().endpoints(etcdStr).build();
        }

        if (Preconditions.isBlank(etcdPath)) {
            this.path = Constant.DEFAULT_REGISTRY_PATH;
        } else {
            this.path = etcdPath;
        }

        vertx = Vertx.vertx();
    }
 

/**
 * 需要在启动 SpiderEngine 之前，启动 ProxyPool
 */
public void startProxyPool(Map<String, Class> proxyMap) {

    if (Preconditions.isNotBlank(proxyMap)) {
        ProxyPool.proxyMap = proxyMap;
        ProxyManager proxyManager = ProxyManager.get();
        proxyManager.start();
    }
}
 

@WebLog
@Cacheable(value="proxys")
@RequestMapping(value="/proxys/{count}", method = RequestMethod.GET)
public ResultProxyDataDTO getProxyData(@PathVariable String count) {

    int code = 0;
    String message = "";
    List<ProxyDataDTO> data = new ArrayList<>();
    if(isNumeric(count)) {
        data = proxyDao.findLimitProxy(Integer.parseInt(count));

        if(Preconditions.isNotBlank(data)) {
            code = 200;
            message = "成功返回有效数据";
        } else {
            code = 404;
            message = "无法返回有效数据";
        }
    } else {
        code = 400;
        message = "参数格式无效，请输入数字";
    }

    ResultProxyDataDTO resultProxyDataDTO = new ResultProxyDataDTO();
    resultProxyDataDTO.setCode(code);
    resultProxyDataDTO.setMessage(message);
    resultProxyDataDTO.setData(data);

    return resultProxyDataDTO;
}
 

/**
 * 当前所监控的父的 zNode 下若是子 zNode 发生了变化：新增，删除，修改
 * <p>
 * 下述方法都会触发执行
 *
 * @param event
 */
@Override
public void process(WatchedEvent event) {

    List<String> newZodeInfos = null;
    try {
        newZodeInfos = client.getChildren().usingWatcher(this).forPath(path);
        //根据初始化容器的长度与最新的容器的长度进行比对，就可以推导出当前 SpiderEngine 集群的状态：新增，宕机/下线，变更...
        //哪个容器中元素多，就循环遍历哪个容器。
        if (Preconditions.isNotBlank(newZodeInfos)) {
            if (newZodeInfos.size()>znodes.size()){
                //明确显示新增了哪个 SpiderEngine 节点
                for (String nowZNode:newZodeInfos) {
                    if (!znodes.contains(nowZNode)){
                        log.info("新增 SpiderEngine 节点{}", nowZNode);
                        stateMap.put(nowZNode, SpiderEngineState.ONLINE);
                    }
                }
            }else if (newZodeInfos.size()<znodes.size()){
                // 宕机/下线
                // 明确显示哪个 SpiderEngine 节点宕机/下线了
                for (String initZNode : znodes) {
                    if (!newZodeInfos.contains(initZNode)) {
                        log.info("SpiderEngine 节点【{}】下线了！", initZNode);
                        stateMap.put(initZNode, SpiderEngineState.OFFLINE);

                        // 如果有下线的处理，则处理(例如发邮件、短信、重启等)
                        if (serverOfflineProcess!=null) {
                            serverOfflineProcess.process();
                        }
                    }
                }
            }else {
                // SpiderEngine 集群正常运行;
                // 宕机/下线了，当时马上重启了，总数未发生变化
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
    }

    znodes = newZodeInfos;
}
 

/**
 * 从 application.conf 中获取配置，并依据这些配置来初始化爬虫
 */
private void initSpiderConfig() {
    autoProxy = SpiderConfig.getInstance().isAutoProxy();
    initialDelay = SpiderConfig.getInstance().getInitialDelay();
    maxRetries = SpiderConfig.getInstance().getMaxRetries();
    retryDelayMillis = SpiderConfig.getInstance().getRetryDelayMillis();

    requestSleepTime = SpiderConfig.getInstance().getSleepTime();
    autoSleepTime = SpiderConfig.getInstance().isAutoSleepTime();
    downloadDelay = SpiderConfig.getInstance().getDownloadDelay();
    autoDownloadDelay = SpiderConfig.getInstance().isAutoDownloadDelay();
    domainDelay = SpiderConfig.getInstance().getDomainDelay();
    autoDomainDelay = SpiderConfig.getInstance().isAutoDomainDelay();

    pipelineDelay = SpiderConfig.getInstance().getPipelineDelay();
    autoPipelineDelay = SpiderConfig.getInstance().isAutoPipelineDelay();

    String downloaderType = SpiderConfig.getInstance().getDownloaderType();

    if (Preconditions.isNotBlank(downloaderType)) {
        switch (downloaderType) {
            case Constant.DOWNLOAD_TYPE_VERTX:
                this.downloader = new VertxDownloader();
                break;
            case Constant.DOWNLOAD_TYPE_URL_CONNECTION:
                this.downloader = new UrlConnectionDownloader();
                break;
            case Constant.DOWNLOAD_TYPE_FILE:
                this.downloader = new FileDownloader();
                break;
            default:
                break;
        }
    }

    if (SpiderConfig.getInstance().isUsePrintRequestPipeline()) {
        this.pipelines.add(new PrintRequestPipeline()); // 默认使用 PrintRequestPipeline
    }
    if (SpiderConfig.getInstance().isUseConsolePipeline()) {
        this.pipelines.add(new ConsolePipeline()); // 默认使用 ConsolePipeline
    }
}
 

public Spider url(List<String> urls) {
    checkIfRunning();

    if (Preconditions.isNotBlank(urls)) {

        urls.forEach(url -> pushToQueue(url,null));

        signalNewRequest();
    }

    return this;
}