Elasticsearch在springcloud项目中同步数据的运用

一，ES的基本概念

1.什么是全文搜索引擎：

我们搜索时按结构化的拼音搜到读音，然后按其指向的页数，便可找到我们的非结构化数据——也即对字的解释。这种先建立索引，再对索引进行搜索的过程就叫全文检索(Full-text Search)。

代表就是lucence。Lucene是根据关健字来搜索的文本搜索工具，只能在某个网站内部搜索文本内容，不能跨网站搜索。

对lucence进行简化可以采用ES，ES是面向文档的（document oriented）的，对文档（注意不是成行成列的数据）进行索引，搜索，排序，过滤。

对比着传统mysql的概念理解

database对应indexes索引，

table对应types类型，

rows对应doucment文档，

column对应field字段。

创建，删除索引，创建映射，增删改查文档。

查文档分为带分词器的query string查询和不带索引的term查询。

二，前期准备

安装Elasticsearch，头信息，ik分词器。

创建索引库xccourse

创建映射

三，logstash将mysql数据库中的内容同步到ES索引库中的

课程系统远程通过feign调用CMS内容管理系统，并在课程系统中完成发布，并页面保存到本地，，同时将信息保存在coursepub中，完成课程发布。

基于logstash同步mysql数据库中coursepub表的信息到es后台，可以在ES后台“数据浏览”看到与coursepub完全一致的内容。

配置文件mysql.config

input {
  stdin {
  }
  jdbc {
  jdbc_connection_string => "jdbc:mysql://localhost:3306/xc_course?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC"
  # the user we wish to excute our statement as
  jdbc_user => "root"
  jdbc_password => root
  # the path to our downloaded jdbc driver  
  jdbc_driver_library => "E:/soft/apache-maven-3.3.9/repository/mysql/mysql-connector-java/5.1.36/mysql-connector-java-5.1.36.jar"
  # the name of the driver class for mysql
  jdbc_driver_class => "com.mysql.jdbc.Driver"
  jdbc_paging_enabled => "true"
  jdbc_page_size => "50000"
  #要执行的sql文件
  #statement_filepath => "/conf/course.sql"
  statement => "select * from course_pub where timestamp > date_add(:sql_last_value,INTERVAL 8 HOUR)"
  #定时配置
  schedule => "* * * * *"
  record_last_run => true
  last_run_metadata_path => "‪E:/xuecheng/es/logstash-6.2.1/config/logstash_metadata"
  }
}


output {
  elasticsearch {
  #ES的ip地址和端口
  hosts => "localhost:9200"
  #hosts => ["localhost:9200","localhost:9202","localhost:9203"]
  #ES索引库名称
  index => "xc_course"
  document_id => "%{id}"
  document_type => "doc"
  template =>"E:/xuecheng/es/logstash-6.2.1/config/xc_course_template.json"
  template_name =>"xc_course"
  template_overwrite =>"true"
  }
  stdout {
 #日志输出
  codec => json_lines
  }
}

21行的logstash_metadata文件

--- 2018-06-30 11:26:00.150000000 Z

启动logstash

数据浏览显示mysql的数据。

四，搜索微服务普通分级搜索和按照关键字查询，以及按照分类等级查询，按照分页查询

配置类

 ElasticsearchConfig

import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

/**
 * @author Administrator
 * @version 1.0
 **/
@Configuration
public class ElasticsearchConfig {

    @Value("${xuecheng.elasticsearch.hostlist}")
    private String hostlist;

    @Bean
    public RestHighLevelClient restHighLevelClient(){
        //解析hostlist配置信息
        String[] split = hostlist.split(",");
        //创建HttpHost数组，其中存放es主机和端口的配置信息
        HttpHost[] httpHostArray = new HttpHost[split.length];
        for(int i=0;i<split.length;i++){
            String item = split[i];
            httpHostArray[i] = new HttpHost(item.split(":")[0], Integer.parseInt(item.split(":")[1]), "http");
        }
        //创建RestHighLevelClient客户端
        return new RestHighLevelClient(RestClient.builder(httpHostArray));
    }

    //项目主要使用RestHighLevelClient，对于低级的客户端暂时不用
    @Bean
    public RestClient restClient(){
        //解析hostlist配置信息
        String[] split = hostlist.split(",");
        //创建HttpHost数组，其中存放es主机和端口的配置信息
        HttpHost[] httpHostArray = new HttpHost[split.length];
        for(int i=0;i<split.length;i++){
            String item = split[i];
            httpHostArray[i] = new HttpHost(item.split(":")[0], Integer.parseInt(item.split(":")[1]), "http");
        }
        return RestClient.builder(httpHostArray).build();
    }

}

api

@Api(value = "课程搜索",description = "课程搜索",tags = {"课程搜索"})
public interface EsCourseControllerApi {

    @ApiOperation("课程搜索")
   QueryResponseResult<CoursePub> searchCourse(int page, int size, CourseSearchParam courseSearchParam);

}

controller

@RestController
@RequestMapping("/search/course")
public class EsCourseController implements EsCourseControllerApi {

    @Autowired
    private CourseSearchService courseSearchService;


    @Override
    @GetMapping("/list/{page}/{size}")
    public QueryResponseResult<CoursePub> searchCourse(@PathVariable("page") int page, @PathVariable("size")int size, CourseSearchParam courseSearchParam) {
        return courseSearchService.searchCourse(page,size,courseSearchParam);
    }
}

yml

server:
  port: ${port:40100}
spring:
  application:
    name: xc-search-service
xc:
  elasticsearch:
    hostlist: ${eshostlist:127.0.0.1:9200} #多个结点中间用逗号分隔
    course:
      index: xc_course
      type: doc

service

import com.xuecheng.filesystem.framework.domain.course.response.CoursePub;
import com.xuecheng.filesystem.framework.domain.search.CourseSearchParam;
import com.xuecheng.filesystem.framework.model.response.CommonCode;
import com.xuecheng.filesystem.framework.model.response.QueryResponseResult;
import com.xuecheng.filesystem.framework.model.response.QueryResult;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

@Service
public class CourseSearchService {

    @Autowired
    private RestHighLevelClient restHighLevelClient;

    @Value("${xuecheng.elasticsearch.course.index}")
    private String index;

    @Value("${xuecheng.elasticsearch.course.type}")
    private String type;

    /**
     * 课程搜索
     * 关键字查询
     * 一级分类，二级分类，难度等级
     * 分页查询
     * 高亮查询
     * @param page
     * @param size
     * @param courseSearchParam
     * @return
     */
    public QueryResponseResult<CoursePub> searchCourse(int page, int size, CourseSearchParam courseSearchParam) {

        SearchRequest searchRequest = new SearchRequest(index);
        searchRequest.types(type);

        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();

        //boolQueryBuilder  must  C1  C2 C1 AND C2        or  C1 OR C2
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();

        //按照关键字查询
        if (StringUtils.isNotEmpty(courseSearchParam.getKeyword())){

            MultiMatchQueryBuilder multiMatchQueryBuilder = QueryBuilders.multiMatchQuery(courseSearchParam.getKeyword(), "name", "description", "teachplan");
            //查询权重
            multiMatchQueryBuilder.field("name",10);
            multiMatchQueryBuilder.minimumShouldMatch("70%");

            boolQueryBuilder.must(multiMatchQueryBuilder);
        }

        //过滤查询&等值查询
        //一级分类查询
        if (StringUtils.isNotEmpty(courseSearchParam.getMt())){
            boolQueryBuilder.filter(QueryBuilders.termQuery("mt",courseSearchParam.getMt()));
        }

        //二级分类查询
        if (StringUtils.isNotEmpty(courseSearchParam.getSt())){
            boolQueryBuilder.filter(QueryBuilders.termQuery("st",courseSearchParam.getSt()));
        }

        //难度等级查询
        if (StringUtils.isNotEmpty(courseSearchParam.getGrade())){
            boolQueryBuilder.filter(QueryBuilders.termQuery("grade",courseSearchParam.getGrade()));
        }

        //分页查询
        if (page <=0){ //当前页
            page =1;
        }
        if (size <= 0 ){
            size = 10;
        }

        int start = (page-1)*size;
        //从哪开始查
        sourceBuilder.from(start);
        //每页显示多少
        sourceBuilder.size(size);

        //高亮查询
        HighlightBuilder highlightBuilder = new HighlightBuilder();
        //高亮前缀
        highlightBuilder.preTags("<font class='eslight'>");
        //高亮后缀
        highlightBuilder.postTags("</font>");
        //高亮域
        highlightBuilder.fields().add(new HighlightBuilder.Field("name"));
        sourceBuilder.highlighter(highlightBuilder);


        sourceBuilder.query(boolQueryBuilder);

        searchRequest.source(sourceBuilder);

        SearchResponse searchResponse = null;
        try {
            searchResponse = restHighLevelClient.search(searchRequest);
        } catch (IOException e) {
            e.printStackTrace();
        }
        //获取查询结果
        SearchHits hits = searchResponse.getHits();
        SearchHit[] searchHits = hits.getHits();

        List<CoursePub> coursePubList = new ArrayList<>();

        for (SearchHit searchHit : searchHits) {

            Map<String, Object> sourceAsMap = searchHit.getSourceAsMap();
            CoursePub coursePub = new CoursePub();

            String id = (String) sourceAsMap.get("id");
            coursePub.setId(id);

            //名称
            String name = (String) sourceAsMap.get("name");
            //获取高亮
            Map<String, HighlightField> highlightFields = searchHit.getHighlightFields();
            if (highlightFields != null){
                HighlightField highlightField = highlightFields.get("name");
                if (highlightField != null){
                    Text[] fragments = highlightField.fragments();
                    if (fragments !=null){
                        StringBuffer stringBuffer = new StringBuffer();
                        for (Text fragment : fragments) {
                            stringBuffer.append(fragment);
                        }
                        name = stringBuffer.toString();
                    }
                }
                coursePub.setName(name);
            }


            //图片
            String pic = (String) sourceAsMap.get("pic");
            coursePub.setPic(pic);

            //价格
            Float price = null;
            if (sourceAsMap.get("price") != null){
                price = Float.parseFloat(String.valueOf(sourceAsMap.get("price")));
            }
            coursePub.setPrice(price);

            //原价
            Float price_old = null;
            if (sourceAsMap.get("price_old")!=null){
                price_old = Float.parseFloat(String.valueOf(sourceAsMap.get("price_old")));
            }
            coursePub.setPrice_old(price_old);

            coursePubList.add(coursePub);

        }

        //数据封装
        QueryResult queryResult = new QueryResult();
        queryResult.setTotal(hits.getTotalHits());
        queryResult.setList(coursePubList);
        return new QueryResponseResult<CoursePub>(CommonCode.SUCCESS,queryResult);
    }
}

nginx代理转发，用户请求/course/search的nginx将请求转发给nuxt。js服务。nginx在转发时根据每台nuxt服务器的负载情况进行转发，实现负载均衡

最后实现走进‘’‘课程搜索’可以进行关键字查询并对关键字实现高亮显示，并采用二级联动实现分级。