elasticsearch 分词器介绍和使用

迷南。 2022-10-22 07:57 157阅读 0赞

# 分词器介绍和使用 #

什么是分词器?

就是将一段文本text，拆分成多个关键词keyword 的工具。

**常用分词器**

*  standard analyzer
 *  simple analyzer
 *  whitespace analyzer
 *  stop analyzer
 *  language analyzer
 *  pattern analyzer

### standard analyzer ###

# 标准分词器,默认的。
    # 大写变小写
    # 去掉了符号
    POST localhost:9200/_analyze
    { 
        "analyzer": "standard",
        "text": "The best 3-points shooter is Curry!"
    }
    
    # 返回的信息
    >>>>>>>>>>>>>>>>>>>>>>>>>>>
    { 
        "tokens": [
            { 
                "token": "the",
                "start_offset": 0,
                "end_offset": 3,
                "type": "<ALPHANUM>",
                "position": 0
            },
            { 
                "token": "best",
                "start_offset": 4,
                "end_offset": 8,
                "type": "<ALPHANUM>",
                "position": 1
            },
            { 
                "token": "3",
                "start_offset": 9,
                "end_offset": 10,
                "type": "<NUM>",
                "position": 2
            },
            { 
                "token": "points",
                "start_offset": 11,
                "end_offset": 17,
                "type": "<ALPHANUM>",
                "position": 3
            },
            { 
                "token": "shooter",
                "start_offset": 18,
                "end_offset": 25,
                "type": "<ALPHANUM>",
                "position": 4
            },
            { 
                "token": "is",
                "start_offset": 26,
                "end_offset": 28,
                "type": "<ALPHANUM>",
                "position": 5
            },
            { 
                "token": "curry",
                "start_offset": 29,
                "end_offset": 34,
                "type": "<ALPHANUM>",
                "position": 6
            }
        ]
    }

### simple analyzer ###

# 大写转小写
    # 去掉所有不是字母的
    
    POST localhost:9200/_analyze
    { 
        "analyzer": "simple",
        "text": "The best 3-points shooter is Curry!"
    }
    
    >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
    { 
        "tokens": [
            { 
                "token": "the",
                "start_offset": 0,
                "end_offset": 3,
                "type": "word",
                "position": 0
            },
            { 
                "token": "best",
                "start_offset": 4,
                "end_offset": 8,
                "type": "word",
                "position": 1
            },
            { 
                "token": "points",
                "start_offset": 11,
                "end_offset": 17,
                "type": "word",
                "position": 2
            },
            { 
                "token": "shooter",
                "start_offset": 18,
                "end_offset": 25,
                "type": "word",
                "position": 3
            },
            { 
                "token": "is",
                "start_offset": 26,
                "end_offset": 28,
                "type": "word",
                "position": 4
            },
            { 
                "token": "curry",
                "start_offset": 29,
                "end_offset": 34,
                "type": "word",
                "position": 5
            }
        ]
    }

### whitespace analyzer ###

# 空格分开
    POST localhost:9200/_analyze
    { 
        "analyzer": "whitespace",
        "text": "The best 3-points shooter is Curry!"
    }
    
    >>>>>>>>>>>>>>>>>>>>>>>>>>>>>
    { 
        "tokens": [
            { 
                "token": "The",
                "start_offset": 0,
                "end_offset": 3,
                "type": "word",
                "position": 0
            },
            { 
                "token": "best",
                "start_offset": 4,
                "end_offset": 8,
                "type": "word",
                "position": 1
            },
            { 
                "token": "3-points",
                "start_offset": 9,
                "end_offset": 17,
                "type": "word",
                "position": 2
            },
            { 
                "token": "shooter",
                "start_offset": 18,
                "end_offset": 25,
                "type": "word",
                "position": 3
            },
            { 
                "token": "is",
                "start_offset": 26,
                "end_offset": 28,
                "type": "word",
                "position": 4
            },
            { 
                "token": "Curry!",
                "start_offset": 29,
                "end_offset": 35,
                "type": "word",
                "position": 5
            }
        ]
    }

### stop analyzer ###

# 停止器
    # stopwords 停止词, 比如 the is a an this of等
    # 默认 __english__
    # 会过滤掉这些停止词,其他的和whitespace很像
    POST localhost:9200/_analyze
    { 
        "analyzer": "stop",
        "text": "The best 3-points shooter is Curry!"
    }
    
    >>>>>>>>>>>>>>>>>>>>>>>>>>>>>
    { 
        "tokens": [
            { 
                "token": "best",
                "start_offset": 4,
                "end_offset": 8,
                "type": "word",
                "position": 1
            },
            { 
                "token": "points",
                "start_offset": 11,
                "end_offset": 17,
                "type": "word",
                "position": 2
            },
            { 
                "token": "shooter",
                "start_offset": 18,
                "end_offset": 25,
                "type": "word",
                "position": 3
            },
            { 
                "token": "curry",
                "start_offset": 29,
                "end_offset": 34,
                "type": "word",
                "position": 5
            }
        ]
    }

### language analyzer ###

POST localhost:9200/_analyze
    { 
        "analyzer": "english",
        "text": "The best 3-points shooter is Curry!"
    }
    >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
    { 
        "tokens": [
            { 
                "token": "best",
                "start_offset": 4,
                "end_offset": 8,
                "type": "<ALPHANUM>",
                "position": 1
            },
            { 
                "token": "3",
                "start_offset": 9,
                "end_offset": 10,
                "type": "<NUM>",
                "position": 2
            },
            { 
                "token": "point",
                "start_offset": 11,
                "end_offset": 17,
                "type": "<ALPHANUM>",
                "position": 3
            },
            { 
                "token": "shooter",
                "start_offset": 18,
                "end_offset": 25,
                "type": "<ALPHANUM>",
                "position": 4
            },
            { 
                "token": "curri",
                "start_offset": 29,
                "end_offset": 34,
                "type": "<ALPHANUM>",
                "position": 6
            }
        ]
    }

### pattern analyzer ###

# 正则分词器,默认使用 \W+ 分单词字符分割
    POST localhost:9200/_analyze
    { 
        "analyzer": "pattern",
        "text": "The best 3-points shooter is Curry!"
    }
    
    { 
        "tokens": [
            { 
                "token": "the",
                "start_offset": 0,
                "end_offset": 3,
                "type": "word",
                "position": 0
            },
            { 
                "token": "best",
                "start_offset": 4,
                "end_offset": 8,
                "type": "word",
                "position": 1
            },
            { 
                "token": "3",
                "start_offset": 9,
                "end_offset": 10,
                "type": "word",
                "position": 2
            },
            { 
                "token": "points",
                "start_offset": 11,
                "end_offset": 17,
                "type": "word",
                "position": 3
            },
            { 
                "token": "shooter",
                "start_offset": 18,
                "end_offset": 25,
                "type": "word",
                "position": 4
            },
            { 
                "token": "is",
                "start_offset": 26,
                "end_offset": 28,
                "type": "word",
                "position": 5
            },
            { 
                "token": "curry",
                "start_offset": 29,
                "end_offset": 34,
                "type": "word",
                "position": 6
            }
        ]
    }

# 分词器简单应用 #

### 创建索引,指定分词器 ###

# 创建索引,指定分词器
    PUT localhost:9200/my_index
    { 
        "settings": { 
            "analysis": { 
                "analyzer": { 
                    "my_analyzer": { 
                        "type": "whitespace"
                    }
                }
            }
        },
        "mappings": { 
            "properties": { 
                "name": { 
                    "type": "text"
                },
                "team_name": { 
                    "type": "text"
                },
                "position": { 
                    "type": "text"
                },
                "play_year": { 
                    "type": "long"
                },
                "jerse_no": { 
                    "type": "keyword"
                },
                "title": { 
                    "type": "text",
                    "analyzer": "my_analyzer"
                }
            }
        }
    }

### 预置数据 ###

PUT localhost:9200/my_index/_doc/1
    { 
        "name": "库里",
        "team_name": "勇士",
        "position": "控球后卫",
        "play_year": 10,
        "jerse_no": "30",
        "title": "The best 3-points shooter is Curry!"
    }

### 查询 ###

# 因为我们指定的分词器whitespace
    # 使用Curry!可以查询的到
    # Curry不能
    POST localhost:9200/my_index/_search
    { 
        "query": { 
            "match": { 
                "title": "Curry!"
            }
        }
    }