solr4的solr-ik插件

我不是女神ヾ 2022-08-03 11:56 162阅读 0赞

solr4的solr-ik插件  http://www.chepoo.com/solr4-plugin-solr-ik.html

solr4中添加中文分词ik。  
1.首先从[https://code.google.com/p/ik-analyzer/][https_code.google.com_p_ik-analyzer]下载[IK Analyzer 2012FF\_hf1\_source.rar][IK Analyzer 2012FF_hf1_source.rar]打包成jar包，然后放入到solr程序中WEB-INF/lib/下。

2.这样做有一个很不方便的地方就是，每次我们要修改词库，我们都要重新打包，很不方便。所以我就修改了其中的部分源码。在词库更新时，直接更新WEB-INF/classes/ik目录下的文件就成。主要修改的源码为Dictionary.java.

<table style="border:none!important; margin:0px!important; padding:0px!important; border-collapse:collapse!important; width:764px"> 
 <tbody style="border:0px; margin:0px; padding:0px"> 
 <tr style="border:0px; margin:0px; padding:0px"> 
 <td style="margin:0px; width:764px; border:none!important; padding:0px!important; vertical-align:top!important; background-color:rgb(238,238,238)"> <pre><code>public class Dictionary { 
&nbsp;
&nbsp;
	/* * 词典单子实例 */
	private static Dictionary singleton;
&nbsp;
	/* * 主词典对象 */
	private DictSegment _MainDict;
&nbsp;
	/* * 停止词词典 */
	private DictSegment _StopWordDict;
	/* * 量词词典 */
	private DictSegment _QuantifierDict;
&nbsp;
	/** * 配置对象 */
	private Configuration cfg;
&nbsp;
	private Dictionary(Configuration cfg){ 
		this.cfg = cfg;
		this.loadMainDict();
		this.loadStopWordDict();
		this.loadQuantifierDict();
	}
&nbsp;
	/** * 词典初始化 * 由于IK Analyzer的词典采用Dictionary类的静态方法进行词典初始化 * 只有当Dictionary类被实际调用时，才会开始载入词典， * 这将延长首次分词操作的时间 * 该方法提供了一个在应用加载阶段就初始化字典的手段 * @return Dictionary */
	public static Dictionary initial(Configuration cfg){ 
		if(singleton == null){ 
			synchronized(Dictionary.class){ 
				if(singleton == null){ 
					singleton = new Dictionary(cfg);
					return singleton;
				}
			}
		}
		return singleton;
	}
&nbsp;
	/** * 获取词典单子实例 * @return Dictionary 单例对象 */
	public static Dictionary getSingleton(){ 
		if(singleton == null){ 
			throw new IllegalStateException("词典尚未初始化，请先调用initial方法");
		}
		return singleton;
	}
&nbsp;
	/** * 批量加载新词条 * @param words Collection&lt;String&gt;词条列表 */
	public void addWords(Collection&lt;String&gt; words){ 
		if(words != null){ 
			for(String word : words){ 
				if (word != null) { 
					//批量加载词条到主内存词典中
					singleton._MainDict.fillSegment(word.trim().toLowerCase().toCharArray());
				}
			}
		}
	}
&nbsp;
	/** * 批量移除（屏蔽）词条 * @param words */
	public void disableWords(Collection&lt;String&gt; words){ 
		if(words != null){ 
			for(String word : words){ 
				if (word != null) { 
					//批量屏蔽词条
					singleton._MainDict.disableSegment(word.trim().toLowerCase().toCharArray());
				}
			}
		}
	}
&nbsp;
	/** * 检索匹配主词典 * @param charArray * @return Hit 匹配结果描述 */
	public Hit matchInMainDict(char[] charArray){ 
		return singleton._MainDict.match(charArray);
	}
&nbsp;
	/** * 检索匹配主词典 * @param charArray * @param begin * @param length * @return Hit 匹配结果描述 */
	public Hit matchInMainDict(char[] charArray , int begin, int length){ 
		return singleton._MainDict.match(charArray, begin, length);
	}
&nbsp;
	/** * 检索匹配量词词典 * @param charArray * @param begin * @param length * @return Hit 匹配结果描述 */
	public Hit matchInQuantifierDict(char[] charArray , int begin, int length){ 
		return singleton._QuantifierDict.match(charArray, begin, length);
	}
&nbsp;
&nbsp;
	/** * 从已匹配的Hit中直接取出DictSegment，继续向下匹配 * @param charArray * @param currentIndex * @param matchedHit * @return Hit */
	public Hit matchWithHit(char[] charArray , int currentIndex , Hit matchedHit){ 
		DictSegment ds = matchedHit.getMatchedDictSegment();
		return ds.match(charArray, currentIndex, 1 , matchedHit);
	}
&nbsp;
&nbsp;
	/** * 判断是否是停止词 * @param charArray * @param begin * @param length * @return boolean */
	public boolean isStopWord(char[] charArray , int begin, int length){ 			
		return singleton._StopWordDict.match(charArray, begin, length).isMatch();
	}	
&nbsp;
	/** * 加载主词典及扩展词典 */
	private void loadMainDict(){ 
		//建立一个主词典实例
		_MainDict = new DictSegment((char)0);
		//读取主词典文件
 InputStream is = this.getClass().getClassLoader().getResourceAsStream(cfg.getMainDictionary());
 if(is == null){ 
 	throw new RuntimeException("Main Dictionary not found!!!");
 }
&nbsp;
		try { 
			BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
			String theWord = null;
			do { 
				theWord = br.readLine();
				if (theWord != null &amp;&amp; !"".equals(theWord.trim())) { 
					_MainDict.fillSegment(theWord.trim().toLowerCase().toCharArray());
				}
			} while (theWord != null);
&nbsp;
		} catch (IOException ioe) { 
			System.err.println("Main Dictionary loading exception.");
			ioe.printStackTrace();
&nbsp;
		}finally{ 
			try { 
				if(is != null){ 
 is.close();
 is = null;
				}
			} catch (IOException e) { 
				e.printStackTrace();
			}
		}
		//加载扩展词典
		this.loadExtDict();
	}	
&nbsp;
	/** * 加载用户配置的扩展词典到主词库表 */
	private void loadExtDict(){ 
		//加载扩展词典配置
		List&lt;String&gt; extDictFiles = cfg.getExtDictionarys();
		if(extDictFiles != null){ 
			InputStream is = null;
			for(String extDictName : extDictFiles){ 
				//读取扩展词典文件
				System.out.println("加载扩展词典：" + extDictName);
				is = this.getClass().getClassLoader().getResourceAsStream(extDictName);
				//如果找不到扩展的字典，则忽略
				if(is == null){ 
					continue;
				}
				try { 
					BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
					String theWord = null;
					do { 
						theWord = br.readLine();
						if (theWord != null &amp;&amp; !"".equals(theWord.trim())) { 
							//加载扩展词典数据到主内存词典中
							//System.out.println(theWord);
							_MainDict.fillSegment(theWord.trim().toLowerCase().toCharArray());
						}
					} while (theWord != null);
&nbsp;
				} catch (IOException ioe) { 
					System.err.println("Extension Dictionary loading exception.");
					ioe.printStackTrace();
&nbsp;
				}finally{ 
					try { 
						if(is != null){ 
		 is.close();
		 is = null;
						}
					} catch (IOException e) { 
						e.printStackTrace();
					}
				}
			}
		}		
	}
&nbsp;
	/** * 加载用户扩展的停止词词典 */
	private void loadStopWordDict(){ 
		//建立一个主词典实例
		_StopWordDict = new DictSegment((char)0);
		//加载扩展停止词典
		List&lt;String&gt; extStopWordDictFiles = cfg.getExtStopWordDictionarys();
		if(extStopWordDictFiles != null){ 
			InputStream is = null;
			for(String extStopWordDictName : extStopWordDictFiles){ 
				System.out.println("加载扩展停止词典：" + extStopWordDictName);
				//读取扩展词典文件
				is = this.getClass().getClassLoader().getResourceAsStream(extStopWordDictName);
				//如果找不到扩展的字典，则忽略
				if(is == null){ 
					continue;
				}
				try { 
					BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
					String theWord = null;
					do { 
						theWord = br.readLine();
						if (theWord != null &amp;&amp; !"".equals(theWord.trim())) { 
							//System.out.println(theWord);
							//加载扩展停止词典数据到内存中
							_StopWordDict.fillSegment(theWord.trim().toLowerCase().toCharArray());
						}
					} while (theWord != null);
&nbsp;
				} catch (IOException ioe) { 
					System.err.println("Extension Stop word Dictionary loading exception.");
					ioe.printStackTrace();
&nbsp;
				}finally{ 
					try { 
						if(is != null){ 
		 is.close();
		 is = null;
						}
					} catch (IOException e) { 
						e.printStackTrace();
					}
				}
			}
		}		
	}
&nbsp;
	/** * 加载量词词典 */
	private void loadQuantifierDict(){ 
		//建立一个量词典实例
		_QuantifierDict = new DictSegment((char)0);
		//读取量词词典文件
 InputStream is = this.getClass().getClassLoader().getResourceAsStream(cfg.getQuantifierDicionary());
 if(is == null){ 
 	throw new RuntimeException("Quantifier Dictionary not found!!!");
 }
		try { 
			BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
			String theWord = null;
			do { 
				theWord = br.readLine();
				if (theWord != null &amp;&amp; !"".equals(theWord.trim())) { 
					_QuantifierDict.fillSegment(theWord.trim().toLowerCase().toCharArray());
				}
			} while (theWord != null);
&nbsp;
		} catch (IOException ioe) { 
			System.err.println("Quantifier Dictionary loading exception.");
			ioe.printStackTrace();
&nbsp;
		}finally{ 
			try { 
				if(is != null){ 
 is.close();
 is = null;
				}
			} catch (IOException e) { 
				e.printStackTrace();
			}
		}
	}
&nbsp;
}</code></pre> </td> 
 </tr> 
 </tbody> 
</table>

3.相关程序已经放入到了[https://github.com/awnuxkjy/solr-ik][https_github.com_awnuxkjy_solr-ik]

4.使用方法。配置solrhome下的collection1\\conf\\schema.xml

<table style="border:none!important; margin:0px!important; padding:0px!important; border-collapse:collapse!important; width:679px"> 
 <tbody style="border:0px; margin:0px; padding:0px"> 
 <tr style="border:0px; margin:0px; padding:0px"> 
 <td style="margin:0px; width:679px; border:none!important; padding:0px!important; vertical-align:top!important; background-color:rgb(238,238,238)"> <pre><code>&lt;fieldType name="ik_text" class="solr.TextField"&gt; 
 &lt;analyzer class="org.wltea.analyzer.lucene.IKAnalyzer" useSmart="true"/&gt; 
&lt;/fieldType&gt;</code></pre> </td> 
 </tr> 
 </tbody> 
</table>

在你的solr程序中/classes文件中新建ik目录，拷贝main2012.dic，ext.dic,IKAnalyzer.cfg.xml,quantifier.dic, stopword.dic等文件。

将ik源码打成jar包。放入到WEB-INF/lib/下

本文固定链接: [http://www.chepoo.com/solr4-plugin-solr-ik.html | IT技术精华网][http_www.chepoo.com_solr4-plugin-solr-ik.html _ IT]

[https_code.google.com_p_ik-analyzer]: https://code.google.com/p/ik-analyzer/
[IK Analyzer 2012FF_hf1_source.rar]: https://code.google.com/p/ik-analyzer/downloads/detail?name=IK%20Analyzer%202012FF_hf1_source.rar&can=2&q=
[https_github.com_awnuxkjy_solr-ik]: https://github.com/awnuxkjy/solr-ik
[http_www.chepoo.com_solr4-plugin-solr-ik.html _ IT]: http://www.chepoo.com/solr4-plugin-solr-ik.html