cdh solr自定义索引, 导入mysql数据

落日映苍穹つ 2022-12-24 05:58 181阅读 0赞

solrctl instancedir --generate /var/lib/solr/config/test
    
    solrctl instancedir --create collection_jdbc /var/lib/solr/config/test
    solrctl instancedir --list
    
    solrctl collection --create test # -s 1 -r 1 -m 1 
    solrctl collection --list

### 文章目录 ###

*  solr 配置概览: solrchonfig.xml, schema.xml
 *  1. 创建collection配置
 *  2. 创建/修改配置文件,并上传到 zookeeper
 *   *  a, solrconfig.xml
     *  b, schema.xml
 *  3. 创建 collection
 *  4. 使用solr admin ：导入mysql数据
 *  5. 修改solr.xml, solrconfig.xml，添加字段
 *   *  a, solrconfig.xml
     *  b, schema.xml

# solr 配置概览: solrchonfig.xml, schema.xml #

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU_size_16_color_FFFFFF_t_70_pic_center]

# 1. 创建collection配置 #

#1.创建集合-配置文件
     #instancedir [--generate path [-schemaless]]
     # [--create name path]
     # [--update name path]
     # [--get name path]
     # [--delete name]
     # [--list]
    
    solrctl instancedir --generate /var/lib/solr/collection_jdbc
    			      			 #/var/lib/solr/collection_jdbc/conf这个目录下产生很多配置文件

# 2. 创建/修改配置文件,并上传到 zookeeper #

## a, solrconfig.xml ##

*  1, solrconfig.xml 添加data-import相关jar包  
    ![在这里插入图片描述][20201127095434554.png_pic_center]
 *  2, 配置solrconfig.xml中requestHandle引用到的配置文件db-data-config.xml  
    ![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU_size_16_color_FFFFFF_t_70_pic_center 1]

## b, schema.xml ##

solr 字段定义说明：[https://lucene.apache.org/solr/guide/6\_6/documents-fields-and-schema-design.html][https_lucene.apache.org_solr_guide_6_6_documents-fields-and-schema-design.html]

*  1, schema.xml 添加自定义索引字段  
    ![在这里插入图片描述][20201127095325318.png_pic_center]
 *  2, schema.xml 删除默认id字段，修改uniqueKey使用自定义的字段名  
    ![在这里插入图片描述][20201127100059539.png_pic_center]  
    ![在这里插入图片描述][20201127095902955.png_pic_center]

#2.创建配置文件,并上传到 zookeeper
    #添加配置: 使用data-importor,来抽取jdbc数据
    sed -i '/luceneMatchVersion/a <lib dir="/var/lib/solr/jars/" regex=".*\\.jar" /> ' /var/lib/solr/collection_jdbc/conf/solrconfig.xml 
    sed -i '/<\/requestDispatcher/a \ <requestHandler name="/dataimport" class="solr.DataImportHandler"> \ <lst name="defaults"> \ <str name="config">db-data-config.xml</str> \ </lst> \ </requestHandler> ' \
    /var/lib/solr/collection_jdbc/conf/solrconfig.xml 
    
    #配置data-import
    [root@test-c6 ~]# cat /var/lib/solr/collection_jdbc/conf/db-data-config.xml
    <dataConfig>
        <dataSource driver="com.mysql.jdbc.Driver" url="jdbc:mysql://192.168.56.161:3306/test" user="root" password="123456" />
        <document>
            <entity name="t1" query="select * from t1"
                    deltaQuery="select * from t1 where last_modified > '${dataimporter.last_index_time}'">
                <field column="name" name="my_name" />
                <field column="id" name="my_id" />
                <field column="age" name="my_age" />
            </entity>
        </document>
    </dataConfig>
    
    
    #拷贝jar包到指定的位置
    [root@eadage img]# ls /opt/cloudera/parcels/CDH/lib/solr/solr-dataimporthandler*
    /opt/cloudera/parcels/CDH/lib/solr/solr-dataimporthandler-4.10.3-cdh5.12.0.jar         /opt/cloudera/parcels/CDH/lib/solr/solr-dataimporthandler-extras.jar
    /opt/cloudera/parcels/CDH/lib/solr/solr-dataimporthandler-extras-4.10.3-cdh5.12.0.jar  /opt/cloudera/parcels/CDH/lib/solr/solr-dataimporthandler.jar
    
    [root@test-c6 ~]# ls /var/lib/solr/jars/ 
    mysql-connector-java.jar                     solr-dataimporthandler-extras-4.10.3-cdh5.12.0.jar  solr-dataimporthandler.jar
    solr-dataimporthandler-4.10.3-cdh5.12.0.jar  solr-dataimporthandler-extras.jar
    
    #添加配置: schema.xml, 字段映射
    sed -i '/<fields>/a <field name="my_id" type="string" required="true" indexed="true" stored="true" multiValued="false" /> ' /var/lib/solr/collection_jdbc/conf/schema.xml
    sed -i '/<fields>/a <field name="my_name" type="text_general" indexed="true" stored="true" multiValued="false" /> ' /var/lib/solr/collection_jdbc/conf/schema.xml
    sed -i '/<fields>/a <field name="my_age" type="int" indexed="true" stored="true" multiValued="false" /> ' /var/lib/solr/collection_jdbc/conf/schema.xml
    #替换旧的默认id, 指定uniquekey为自己的字段名
    sed -i 's@<uniqueKey>id</uniqueKey>@<uniqueKey>my_id</uniqueKey>@' /var/lib/solr/collection_jdbc/conf/schema.xml
    sed -i '/<field name="id"/d'  /var/lib/solr/collection_jdbc/conf/schema.xml 
    
    #上传配置到zk
    solrctl instancedir --create collection_jdbc /var/lib/solr/collection_jdbc
    solrctl instancedir --list

# 3. 创建 collection #

#3.创建 collection:
    #collection [--create name -s <numShards>
    # [-a Create collection with autoAddReplicas=true]
    # [-c <collection.configName>]
    # [-r <replicationFactor>]
    # [-m <maxShardsPerNode>]
    # [-n <createNodeSet>]]
    # [--delete name]
    # [--reload name]
    # [--stat name]
    # [--deletedocs name]
    # [--list]
    
    solrctl collection --create collection_jdbc # -s 1 -r 1 -m 1 
    solrctl collection --list

# 4. 使用solr admin ：导入mysql数据 #

*  登录mysql: 查看数据

mysql> select * from  test.t1;
    +------+------+------+
    | id   | name | age  |
    +------+------+------+
    | NULL | a    |   28 |
    | 2    | b    |   24 |
    | 3    | c    |   22 |
    +------+------+------+
    3 rows in set (0.00 sec)

*  使用solr admin导入数据  
    ![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU_size_16_color_FFFFFF_t_70_pic_center 2]
 *  查询数据  
    ![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU_size_16_color_FFFFFF_t_70_pic_center 3]

# 5. 修改solr.xml, solrconfig.xml，添加字段 #

## a, solrconfig.xml ##

*  solrsconfig.xml中引用的db-data-config.xml添加字段  
    ![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU_size_16_color_FFFFFF_t_70_pic_center 4]

## b, schema.xml ##

*  schema.xml添加一个字段  
    ![在这里插入图片描述][20201127103335184.png_pic_center]
 *  重载配置

[root@test-c6 ~]# solrctl instancedir --update collection_jdbc /var/lib/solr/collection_jdbc/
    Uploading configs from /var/lib/solr/collection_jdbc//conf to test-c6:2181/solr. This may take up to a minute.
    
    [root@test-c6 ~]# solrctl collection --reload collection_jdbc
    [root@test-c6 ~]#

*  再次导入数据，并查询

mysql> select * from  test.t1;
    +------+------+------+---------------------+
    | id   | name | age  | regist_time         |
    +------+------+------+---------------------+
    | 1    | a    |   28 | 2018-10-10 10:12:10 |
    | 2    | b    |   24 | 2019-09-00 09:00:00 |
    | 3    | c    |   22 | 2018-10-10 10:12:10 |
    | 4    | NULL | NULL | 2018-10-10 10:12:10 |
    | 5    | e    |   33 | 2020-11-27 10:27:00 |
    +------+------+------+---------------------+
    5 rows in set (0.00 sec)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU_size_16_color_FFFFFF_t_70_pic_center 5]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU_size_16_color_FFFFFF_t_70_pic_center]: /images/20221120/538e7193e9c54968bb28cc8126b0ebd3.png
[20201127095434554.png_pic_center]: /images/20221120/17c46cc504b0419c934fd0c4ba999bdc.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU_size_16_color_FFFFFF_t_70_pic_center 1]: https://img-blog.csdnimg.cn/20201127095739533.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU=,size_16,color_FFFFFF,t_70#pic_center
[https_lucene.apache.org_solr_guide_6_6_documents-fields-and-schema-design.html]: https://lucene.apache.org/solr/guide/6_6/documents-fields-and-schema-design.html
[20201127095325318.png_pic_center]: https://img-blog.csdnimg.cn/20201127095325318.png#pic_center
[20201127100059539.png_pic_center]: https://img-blog.csdnimg.cn/20201127100059539.png#pic_center
[20201127095902955.png_pic_center]: https://img-blog.csdnimg.cn/20201127095902955.png#pic_center
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU_size_16_color_FFFFFF_t_70_pic_center 2]: https://img-blog.csdnimg.cn/20201127100431637.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU=,size_16,color_FFFFFF,t_70#pic_center
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU_size_16_color_FFFFFF_t_70_pic_center 3]: https://img-blog.csdnimg.cn/20201127100703678.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU=,size_16,color_FFFFFF,t_70#pic_center
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU_size_16_color_FFFFFF_t_70_pic_center 4]: https://img-blog.csdnimg.cn/20201127103422447.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU=,size_16,color_FFFFFF,t_70#pic_center
[20201127103335184.png_pic_center]: https://img-blog.csdnimg.cn/20201127103335184.png#pic_center
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU_size_16_color_FFFFFF_t_70_pic_center 5]: https://img-blog.csdnimg.cn/20201127103724830.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2V5ZW9mZWFnbGU=,size_16,color_FFFFFF,t_70#pic_center