利用Java实现网站聚合工具

public SendReq.ResBody doRequest(String url, String method, Map<String, Object> params) {
    String urlTrue = url;
    SendReq.ResBody resBody = SendReq.sendReq(urlTrue, method, params, defaultHeaders());
    return resBody;
}

public static List<String> getUrls(String htmlText) {
    Pattern pattern = Pattern.compile("(http|https):\\/\\/[A-Za-z0-9_\\-\\+.:?&@=\\/%#,;]*");
    Matcher matcher = pattern.matcher(htmlText);
    Set<String> ans = new HashSet<>();
    while (matcher.find()){
        ans.add(DomainUtils.getDomainWithCompleteDomain(matcher.group()));
    }
    return new ArrayList<>(ans);
}

public static String getTitle(String htmlText){
    Pattern pattern = Pattern.compile("(?<=title\\>).*(?=</title)");
    Matcher matcher = pattern.matcher(htmlText);
    Set<String> ans = new HashSet<>();
    while (matcher.find()){
        return matcher.group();
    }
    return "";
}

public static String getContent(String html) {
    String ans = "";
    try {
        html = StringEscapeUtils.unescapeHtml4(html);
        html = delHTMLTag(html);
        html = htmlTextFORMat(html);
        return html;
    } catch (Exception e) {
        e.printStackTrace();
    }
    return ans;
}

public static String delHTMLTag(String htmlStr) {
    String regEx_script = "<script[^>]*?>[\\s\\S]*?<\\/script>"; //定义script的正则表达式
    String regEx_style = "<style[^>]*?>[\\s\\S]*?<\\/style>"; //定义style的正则表达式
    String regEx_html = "<[^>]+>"; //定义HTML标签的正则表达式

    Pattern p_script = Pattern.compile(regEx_script, Pattern.CASE_INSENSITIVE);
    Matcher m_script = p_script.matcher(htmlStr);
    htmlStr = m_script.replaceAll(""); //过滤script标签

    Pattern p_style = Pattern.compile(regEx_style, Pattern.CASE_INSENSITIVE);
    Matcher m_style = p_style.matcher(htmlStr);
    htmlStr = m_style.replaceAll(""); //过滤style标签

    Pattern p_html = Pattern.compile(regEx_html, Pattern.CASE_INSENSITIVE);
    Matcher m_html = p_html.matcher(htmlStr);
    htmlStr = m_html.replaceAll(""); //过滤html标签

    return htmlStr.trim();
}

private static Pattern ignoreWords = Pattern.compile("[,.0-9_\\- ，、：。；;\\]\\[\\/！（）【】*？“”()+:|\"%~<>——]+");

public static Set<Word> separateWordAndReturnUnit(String text) {
    Segment segment = HanLP.newSegment().enableOffset(true);
    Set<Word> detectorUnits = new HashSet<>();
    Map<Integer, Word> detectorUnitMap = new HashMap<>();
    List<Term> terms = segment.seg(text);
    for (Term term : terms) {
        Matcher matcher = ignoreWords.matcher(term.word);
        if (!matcher.find() && term.word.length() > 1 && !term.word.contains("�")) {
            Integer hashCode = term.word.hashCode();
            Word detectorUnit = detectorUnitMap.get(hashCode);
            if (Objects.nonNull(detectorUnit)) {
                detectorUnit.setCount(detectorUnit.getCount() + 1);
            } else {
                detectorUnit = new Word();
                detectorUnit.setWord(term.word.trim());
                detectorUnit.setCount(1);
                detectorUnitMap.put(hashCode, detectorUnit);
                detectorUnits.add(detectorUnit);
            }
        }
    }
    return detectorUnits;
}

public static List<String> print2List(List<Word> tmp,int cnt){
    PriorityQueue<Word> words = new PriorityQueue<>();
    List<String> ans = new ArrayList<>();
    for (Word word : tmp) {
        words.add(word);
    }
    int count = 0;
    while (!words.isEmpty()) {
        Word word = words.poll();
        if (word.getCount()<50){
            ans.add(word.getWord() + " " + word.getCount());
            count ++;
            if (count >= cnt){
                break;
            }
        }
    }
    return ans;
}

public class Word implements Comparable{
    private String word;
    private Integer count = 0;

    ... ...

    @Override
    public int compareTo(Object o) {
        if (this.count >= ((Word)o).count){
            return -1;
        }else {
            return 1;
        }
    }
}

public void doTask() {
    String root = "http://" + this.domain + "/";
    Queue<String> urls = new LinkedList<>();
    urls.add(root);
    Set<String> tmpDomains = new HashSet<>();
    tmpDomains.add(DomainUtils.getDomainWithCompleteDomain(root));
    while (!urls.isEmpty()) {
        String url = urls.poll();
        SendReq.ResBody html = doRequest(url, "GET", new HashMap<>());
        System.out.println("当前的请求为 " + url + " 队列的大小为 " + urls.size() + " 结果为" + html.getCode());
        if (html.getCode().equals(0)) {
            ignoreSet.add(DomainUtils.getDomainWithCompleteDomain(url));
            try {
                GenerateFile.createFile2("moneyframework/generate/ignore", "demo.txt", ignoreSet.toString());
            } catch (IOException e) {
                e.printStackTrace();
            }
            continue;
        }

        OnePage onePage = new OnePage();
        onePage.setUrl(url);
        onePage.setDomain(DomainUtils.getDomainWithCompleteDomain(url));
        onePage.setCode(html.getCode());
        String title = HtmlUtil.getTitle(html.getResponce()).trim();
        if (!StringUtils.hasText(title) || title.length() > 100 || title.contains("�")) continue;
        onePage.setTitle(title);
        String content = HtmlUtil.getContent(html.getResponce());
        Set<Word> words = Nlp.separateWordAndReturnUnit(content);
        List<String> wordStr = Nlp.print2List(new ArrayList<>(words), 10);
        handleWord(wordStr, DomainUtils.getDomainWithCompleteDomain(url), title);
        onePage.setContent(wordStr.toString());
        if (html.getCode().equals(200)) {
            List<String> domains = HtmlUtil.getUrls(html.getResponce());
            for (String domain : domains) {
                int flag = 0;
                for (String i : ignoreSet) {
                    if (domain.endsWith(i)) {
                        flag = 1;
                        break;
                    }
                }
                if (flag == 1) continue;
                if (StringUtils.hasText(domain.trim())) {
                    if (!tmpDomains.contains(domain)) {
                        tmpDomains.add(domain);
                        urls.add("http://" + domain + "/");
                    }
                }
            }
        }
    }
}

@Service
public class Task {

    @PostConstruct
    public void init(){
        new Thread(new Runnable() {
            @Override
            public void run() {
                while (true){
                    try {
                        HttpClientCrawl clientCrawl = new HttpClientCrawl("http://www.mengwa.store/");
                        clientCrawl.doTask();
                    }catch (Exception e){
                        e.printStackTrace();
                    }
                }
            }
        }).start();
    }
}