elasticsearch 7.9.3知识归纳整理（四）之 java实现深分页scroll

布满荆棘的人生 2024-03-30 14:41 61阅读 0赞

### java实现深分页scroll ###

> ES对from+size是有限制的，from和size的和不能超过1w。超过后效率十分低下。 ES查询过程：  
> ①将用户的指定的关键字进行分词  
> ②将词汇去分词库中进行检索，得到多个文档的id  
> ③去各个分片中拉去数据指定的数据(耗时长)  
> ④将数据根据score进行排序(耗时长)  
> ⑤根据from的值将查询到的数据舍弃一部分  
> ⑥返回结果  
> scroll在ES的数据查询方式：  
> ①将用户的指定的关键字进行分词  
> ②将词汇去分词库中进行检索，得到多个文档的id  
> ③将文档的id存放在es的上下文  
> ④根据指定的size去ES中检索指定的数据。拿完数据的文档id，会从上下文移除  
> ⑤如果需要下一页数据，直接去ES的上下文中找后续数据  
> ⑥循环第4、5步 scroll方式，不适合做适时查询：

#### 1.导包 ####

<dependency>
        <groupId>org.elasticsearch.client</groupId>
        <artifactId>elasticsearch-rest-high-level-client</artifactId>
        <version>7.4.2</version>
    </dependency>

#### 2.配置 ####

@Configuration
    public class MyEsSearchConfig {
        
    	/**
    	* es在yml中的配置(这里只有ip没有端口)
    	*es:
      	*	host1: es-node1
      	*	host2: es-node2
      	*	host3: es-node3
    	**/
        @Value("${es.host1}")
        private String host1;
        @Value("${es.host2}")
        private String host2;
        @Value("${es.host3}")
        private String host3;
    
        public static final RequestOptions COMMON_OPTIONS;
        static {
        
            RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
            COMMON_OPTIONS = builder.build();
        }
        /**
         *方式二 使用账号密码连接
         **/
        @Bean
        public RestHighLevelClient esRestClient(){
        
            RestClientBuilder builder = RestClient.builder(
                    new HttpHost(host1,9200,"http"),
                    new HttpHost(host2,9200,"http"),
                    new HttpHost(host3,9200,"http"));
            CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            credentialsProvider .setCredentials(AuthScope.ANY,new UsernamePasswordCredentials("elastic","123456"));
            builder.setHttpClientConfigCallback(f->f.setDefaultCredentialsProvider(credentialsProvider ));
            RestHighLevelClient client = new RestHighLevelClient (builder);
            return client;
        }
    }

#### 3.实现深分页 ####

@RestController
    @RequestMapping("/esScroll")
    public class EsScrollController {
        
        /**
         * es对from + size 是有限制的 from+size二者之和超过1w会报错
         */
        @Autowired
        private RestHighLevelClient client;
    
        @PostMapping("/getEsScroll")
        public R getEsScroll(){
        
            List<Map<String,Object>> list = new ArrayList<>();
            list.clear();
            String indexName = "esIndexAccessControl";
            //1、创建SearchRequest
            SearchRequest request = new SearchRequest(indexName);
            //2、指定scroll的生存时间
            request.scroll(TimeValue.timeValueMinutes(2L));
    
            SearchSourceBuilder builder = new SearchSourceBuilder();
            ///3、指定每页查询条数
            builder.size(20);
            //3.1、排序
            builder.sort("createTimeMilliss", SortOrder.DESC);
            //3.2、查询全部
            builder.query(QueryBuilders.matchAllQuery());
            request.source(builder);
    
            try {
        
                //4、获取返回结果 scrollId,source
                SearchResponse response = client.search(request, RequestOptions.DEFAULT);
                String scrollId = response.getScrollId();
                for(SearchHit hit:response.getHits().getHits()){
        
                    //4.1、首页数据
                    Map<String, Object> sourceAsMap = hit.getSourceAsMap();
                    list.add(sourceAsMap);
                }
                while(true){
        
                    // 5、循环创建SearchScrollRequest
                    SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
                    //6、再指定scroll的生存时间，若不指定它会归零
                    scrollRequest.scroll(TimeValue.timeValueMinutes(2L));
                    //7、执行查询获取结果
                    SearchResponse scrollResp = client.scroll(scrollRequest, RequestOptions.DEFAULT);
                    //8、判断是否查询到了数据输出
                    SearchHit[] hits = scrollResp.getHits().getHits();
                    if(hits != null && hits.length>0){
        
                        for(SearchHit hit : hits){
        
                            //循环输出
                            Map<String, Object> sourceAsMap = hit.getSourceAsMap();
                            list.add(sourceAsMap);
                        }
                    }else {
        
                        //9、若无数据则退出
                        break;
                    }
                }
                //10、创建ClearScrollRequest
                ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
                //11、指定scrollId
                clearScrollRequest.addScrollId(scrollId);
                //12、删除scrollId
                ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
               //13、根据它返回判断删除成功没
                boolean succeeded = clearScrollResponse.isSucceeded();
            } catch (IOException e) {
        
                e.printStackTrace();
            }
            return R.ok("查询成功",list);
        }