springboot微服务Lucence实现Mysql全文检索功能

CREATE TABLE `t_content` (
  `id` varchar(32) NOT NULL,
  `title` varchar(255) DEFAULT NULL,
  `price` varchar(32) DEFAULT NULL,
  `descs` varchar(255) DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('001', '测试用于', '10', '测试用于');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('1', 'Java面向对象', '10', 'Java面向对象从入门到精通,简单上手');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('10', 'Java开发实战经典', '51', '本书是一本综合讲解Java核心技术的书籍，在书中使用大量的代码及案例进行知识点的分析与运用');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('11', 'Effective Java', '10', '本书介绍了在Java编程中57条极具实用价值的经验规则，这些经验规则涵盖了大多数开发人员每天所面临的问题的解决方案');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('12', '分布式 Java 应用：基础与实践', '15', '本书介绍了编写分布式Java应用涉及的众多知识点，分为了基于Java实现网络通信、rpc;基于SOA实现大型分布式Java应用');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('13', 'Http权威指南', '11', '超文本传输协议(Hypertext Transfer Protocol，HTTP)是在万维网上进行通信时所使用的协议方案');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('14', 'spring', '15', '这是啥，还需要学习吗？Java程序员必备书籍');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('15', '深入理解 Java 虚拟机', '18', '作为一位Java程序员，你是否也曾经想深入理解Java虚拟机，但是却被它的复杂和深奥拒之门外');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('16', 'SpringBoot实战', '11', '完成对于springboot的理解，是每个Java程序员必备的姿势');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('17', 'springMVC学习', '72', 'springmvc学习指南');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('18', 'Vue入门到放弃', '20', 'vue入门到放弃书籍信息');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('19', 'vue入门到精通', '20', 'vue入门到精通相关书籍信息');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('2', 'Java面向对象java', '10', 'Java面向对象从入门到精通,简单上手');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('20', 'vue之旅', '20', '由浅入深地全面介绍vue技术，包含大量案例与代码');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('21', 'vue实战', '20', '以实战为导向，系统讲解如何使用 ');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('22', 'vue入门与实践', '20', '现已得到苹果、微软、谷歌等主流厂商全面支持');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('23', 'vue.js应用测试', '20', 'Vue.js创始人尤雨溪鼎力推荐！Vue官方测试工具作者亲笔撰写，Vue.js应用测试完全学习指南');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('24', 'PHP和MySQL web开发', '20', '本书是利用php和MySQL构建数据库驱动的WEB应用程序的权威指南');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('25', 'Web高效编程与优化实践', '20', '从思想提升和内容修炼两个维度，围绕前端工程师必备的前端技术和编程基础');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('26', 'Vue.js 2.x实践指南', '20', '本书旨在让初学者能够快速上手vue技术栈，并能够利用所学知识独立动手进行项目开发');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('27', '初始vue', '20', '解开vue的面纱');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('28', '什么是vue', '20', '一步一步的了解vue相关信息');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('29', '深入浅出vue', '20', '深入浅出vue，慢慢掌握');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('3', 'Java面向编程', '15', 'Java面向对象编程书籍');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('30', '三天vue实战', '20', '三天掌握vue开发');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('31', '不知火舞', '20', '不知名的vue');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('32', '娜可露露', '20', '一招秒人');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('33', '宫本武藏', '20', '我就是一个超级兵');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('34', 'vue宫本vue', '20', '我就是一个超级兵');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('4', 'javascript入门', '18', 'JavaScript入门编程书籍');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('5', '深入理解Java编程', '13', '十三四天掌握Java基础');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('6', '从入门到放弃_Java', '20', '一门从入门到放弃的书籍');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('7', 'Head First Java', '30', '《Head First Java》是一本完整地面向对象(object-oriented，OO)程序设计和Java的学习指导用书');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('8', 'Java 核心技术：卷1 基础知识', '22', '全书共14章，包括Java基本的程序结构、对象与类、继承、接口与内部类、图形程序设计、事件处理、Swing用户界面组件');
INSERT INTO `bank1`.`t_content`(`id`, `title`, `price`, `descs`) VALUES ('9', 'Java 编程思想', '12', '本书赢得了全球程序员的广泛赞誉，即使是最晦涩的概念，在Bruce Eckel的文字亲和力和小而直接的编程示例面前也会化解于无形');

select * from t_content tc 
where 
tc.title like concat('%','深入', '%') or tc.descs like concat('%','深入', '%')

<dependencies>
 
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
 
        
        <!-- lucene核心库 -->
        <dependency>
            <groupId>org.apache.lucene</groupId>
            <artifactId>lucene-core</artifactId>
            <version>7.6.0</version>
        </dependency>
        <!-- Lucene的查询解析器 -->
        <dependency>
            <groupId>org.apache.lucene</groupId>
            <artifactId>lucene-queryparser</artifactId>
            <version>7.6.0</version>
        </dependency>
        <!-- lucene的默认分词器库,适用于英文分词 -->
        <dependency>
            <groupId>org.apache.lucene</groupId>
            <artifactId>lucene-analyzers-common</artifactId>
            <version>7.6.0</version>
        </dependency>
        <!-- lucene的高亮显示 -->
        <dependency>
            <groupId>org.apache.lucene</groupId>
            <artifactId>lucene-highlighter</artifactId>
            <version>7.6.0</version>
        </dependency>
        <!-- smartcn中文分词器 -->
        <dependency>
            <groupId>org.apache.lucene</groupId>
            <artifactId>lucene-analyzers-smartcn</artifactId>
            <version>7.6.0</version>
        </dependency>
        <!-- ik分词器 -->
        <dependency>
            <groupId>com.janeluo</groupId>
            <artifactId>ikanalyzer</artifactId>
            <version>2012_u6</version>
        </dependency>
 
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>
 
        <dependency>
            <groupId>org.mybatis.spring.boot</groupId>
            <artifactId>mybatis-spring-boot-starter</artifactId>
            <version>2.1.1</version>
        </dependency>
 
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>8.0.11</version>
        </dependency>
 
    </dependencies>

server:
  port: 8083
 
spring:
  datasource:
    driver-class-name: com.mysql.jdbc.Driver
    url: jdbc:mysql://IP:3306/bank1?autoReconnect=true&useUnicode=true&characterEncoding=utf8&serverTimezone=GMT%2B8&useSSL=false
    username: root
    password: root
 
mybatis:
  type-aliases-package: com.congge.entity
  mapper-locations: classpath:mybatis
    @GetMapping("/initDbDataToIndex")
    public String initDbDataToIndex() throws Exception{
        //查询数据库数据
        List<Content> dbList = contentService.queryAll();
        // 创建文档的集合
        Collection<Document> docs = new ArrayList<>();
        for (int i = 0; i < dbList.size(); i++) {
            Document document = new Document();
            //StringField会创建索引，但是不会被分词，TextField，即创建索引又会被分词。
            document.add(new StringField("id", (i + 1) + "", Field.Store.YES));
            document.add(new StringField("title", dbList.get(i).getTitle(), Field.Store.YES));
            document.add(new StringField("price", dbList.get(i).getPrice(), Field.Store.YES));
            document.add(new TextField("descs", dbList.get(i).getDescs(), Field.Store.YES));
            docs.add(document);
        }
        // 索引目录类,指定索引在硬盘中的位置
        Directory directory = FSDirectory.open(FileSystems.getDefault().getPath("D:\\Lucene\\indexDir"));
        Analyzer analyzer = new MyIKAnalyzer();
        IndexWriterConfig conf = new IndexWriterConfig(analyzer);
        // 设置打开方式：OpenMode.APPEND 会在索引库的基础上追加新索引。OpenMode.CREATE会先清空原来数据，再提交新的索引
        conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
        // 创建索引的写出工具类。参数：索引的目录和配置信息
        IndexWriter indexWriter = new IndexWriter(directory, conf);
        // 把文档集合交给IndexWriter
        indexWriter.aDDDocuments(docs);
        indexWriter.commit();
        indexWriter.close();
        return "initDbDataToIndex success";
    }

    
    @GetMapping("/query/keyword")
    public Object searchKeyWord(@RequestParam("text") String text) throws Exception {
        Directory directory = FSDirectory.open(FileSystems.getDefault().getPath(INDEX_PATH));
        IndexReader reader = DirectoryReader.open(directory);
        IndexSearcher searcher = new IndexSearcher(reader);
        QueryParser parser = new QueryParser("descs", new MyIKAnalyzer(true));
        Query query = parser.parse(text);
        TopDocs topDocs = searcher.search(query, Integer.MAX_VALUE);
        log.info("本次搜索共找到" + topDocs.totalHits + "条数据");
        ScoreDoc[] scoreDocs = topDocs.scoreDocs;
        List<Content> list = new ArrayList<>();
        for (ScoreDoc scoreDoc : scoreDocs) {
            int docID = scoreDoc.doc;
            Document doc = reader.document(docID);
            Content content = new Content();
            content.setId(doc.get("id"));
            content.setTitle(doc.get("title"));
            content.setDescs(doc.get("descs"));
            list.add(content);
        }
        return list;
    }

    
    @GetMapping("/updateIndex")
    public String updateIndex(String desc) throws Exception {
 
        Content content = new Content();
        content.setId("1102");
        content.setTitle(desc);
        content.setPrice("10");
        content.setDescs(desc);
        contentService.save(content);
 
        // 创建目录对象
        Directory directory = FSDirectory.open(FileSystems.getDefault().getPath("D:\\Lucene\\indexDir"));
        // 创建配置对象
        IndexWriterConfig conf = new IndexWriterConfig(new MyIKAnalyzer());
        // 创建索引写出工具
        IndexWriter writer = new IndexWriter(directory, conf);
        // 创建新的文档数据
        Document doc = new Document();
        doc.add(new StringField("id", "1102", Field.Store.YES));
 
        doc.add(new StringField("title", content.getTitle(), Field.Store.YES));
        doc.add(new StringField("price", content.getPrice(), Field.Store.YES));
        doc.add(new TextField("descs", content.getDescs(), Field.Store.YES));
 
        writer.updateDocument(new Term("id", "1102"), doc);
        writer.commit();
        writer.close();
        return "updateIndex success";
    }

    
    @GetMapping("/deleteIndex")
    public String deleteIndex(String id) throws Exception {
        contentService.deleteById(id);
        // 创建目录对象
        Directory directory = FSDirectory.open(FileSystems.getDefault().getPath("D:\\Lucene\\indexDir"));
        // 创建配置对象
        IndexWriterConfig conf = new IndexWriterConfig(new IKAnalyzer());
        // 创建索引写出工具
        IndexWriter writer = new IndexWriter(directory, conf);
        // 根据词条进行删除
        writer.deleteDocuments(new Term("id", id));
        writer.commit();
        writer.close();
        return "deleteIndex success";
    }

    
    @GetMapping("/query/page")
    public Object queryForPage(@RequestParam("text") String text,
                               @RequestParam(value = "page",defaultValue = "1") int page,
                               @RequestParam(value = "pageSize",defaultValue = "10") int pageSize) throws Exception {
        Map<String, Object> resMap = new HashMap<>();
        Directory directory = FSDirectory.open(FileSystems.getDefault().getPath(INDEX_PATH));
        IndexReader reader = DirectoryReader.open(directory);
        IndexSearcher searcher = new IndexSearcher(reader);
        QueryParser parser = new QueryParser("descs", new MyIKAnalyzer(true));
        Query query = parser.parse(text);
        TopDocs topDocs = IndexUtils.searchByPage(page, pageSize, searcher, query);
        log.info("本次搜索共找到" + topDocs.totalHits + "条数据");
        ScoreDoc[] scoreDocs = topDocs.scoreDocs;
        List<Content> list = new ArrayList<>();
        List<String> idList = new ArrayList<>();
        for (ScoreDoc scoreDoc : scoreDocs) {
            int docID = scoreDoc.doc;
            Document doc = reader.document(docID);
            idList.add(doc.get("id"));;
        }
        if(!CollectionUtils.isEmpty(idList)){
            list = contentService.queryContentByIdList(idList);
        }
        resMap.put("page", page);
        resMap.put("pageSize", pageSize);
        resMap.put("total", topDocs.totalHits);
        resMap.put("list", list);
        return resMap;
    }

    
    @GetMapping("/query/multi")
    public Object multiFieldQuery(String text) throws Exception {
        String[] str = {"title", "descs"};
        Directory directory = FSDirectory.open(FileSystems.getDefault().getPath(INDEX_PATH));
        // 索引读取工具
        IndexReader reader = DirectoryReader.open(directory);
        // 索引搜索工具
        IndexSearcher searcher = new IndexSearcher(reader);
        // 创建查询解析器,两个参数：默认要查询的字段的名称，分词器
        MultiFieldQueryParser parser = new MultiFieldQueryParser(str, new MyIKAnalyzer());
        // 创建查询对象
        Query query = parser.parse(text);
        // 获取前十条记录
        TopDocs topDocs = searcher.search(query, 100);
        // 获取得分文档对象（ScoreDoc）数组.SocreDoc中包含：文档的编号、文档的得分
        ScoreDoc[] scoreDocs = topDocs.scoreDocs;
        List<Content> list = new ArrayList<>();
        for (ScoreDoc scoreDoc : scoreDocs) {
            // 取出文档编号
            int docID = scoreDoc.doc;
            // 根据编号去找文档
            Document doc = reader.document(docID);
            Content content = new Content();
            content.setId(doc.get("id"));
            content.setTitle(doc.get("title"));
            content.setDescs(doc.get("descs"));
            list.add(content);
        }
        return list;
    }

    
    @GetMapping("/query/high-light")
    public Object queryHighLight(String text) throws Exception {
        String[] str = {"title", "descs"};
        Directory directory = FSDirectory.open(FileSystems.getDefault().getPath(INDEX_PATH));
        IndexReader reader = DirectoryReader.open(directory);
        IndexSearcher searcher = new IndexSearcher(reader);
        MultiFieldQueryParser parser = new MultiFieldQueryParser(str, new MyIKAnalyzer());
        Query query = parser.parse(text);
        TopDocs topDocs = searcher.search(query, Integer.MAX_VALUE);
        //高亮显示设置
        SimpleHTMLFORMatter simpleHTMLFormatter = new SimpleHTMLFormatter("<span style='color:red'>", "</span>");
        Highlighter highlighter = new Highlighter(simpleHTMLFormatter, new QueryScorer(query));
        //高亮后的段落范围在100字内
        Fragmenter fragmenter = new SimpleFragmenter(100);
        highlighter.setTextFragmenter(fragmenter);
        ScoreDoc[] scoreDocs = topDocs.scoreDocs;
        List<Content> list = new ArrayList<>();
        for (ScoreDoc scoreDoc : scoreDocs) {
            int docID = scoreDoc.doc;
            Document doc = reader.document(docID);
            Content content = new Content();
            String title = highlighter.getBestFragment(new MyIKAnalyzer(), "title", doc.get("title"));
            if (title == null) {
                title = content.getTitle();
            }
            String descs = highlighter.getBestFragment(new MyIKAnalyzer(), "descs", doc.get("descs"));
            if (descs == null) {
                descs = content.getDescs();
            }
            content.setDescs(descs);
            content.setTitle(title);
            list.add(content);
        }
        return list;
    }

public class IndexUtils {
 
    public static final String INDEX_PATH = "D:\\Lucene\\indexDir";
 
    private static Directory dir;
 
    public static TopDocs searchByPage(int page, int perPage, IndexSearcher searcher, Query query) throws IOException {
        TopDocs result = null;
        if (query == null) {
            System.out.println(" Query is null return null ");
            return null;
        }
        ScoreDoc before = null;
        if (page != 1) {
            TopDocs docsBefore = searcher.search(query, (page - 1) * perPage);
            ScoreDoc[] scoreDocs = docsBefore.scoreDocs;
            if (scoreDocs.length > 0) {
                before = scoreDocs[scoreDocs.length - 1];
            }
        }
        result = searcher.searchAfter(before, query, perPage);
        return result;
    }
 
    
    public static IndexWriter getWriter() throws Exception {
        //使用中文分词器
        SmartChineseAnalyzer analyzer = new SmartChineseAnalyzer();
        //将中文分词器配到写索引的配置中
        IndexWriterConfig config = new IndexWriterConfig(analyzer);
        //实例化写索引对象
        IndexWriter writer = new IndexWriter(dir, config);
        return writer;
    }
 
}