一个简单的中文分词 
sphinx简析
09/08
21
Lucene中文分词的highlight显示 
1、问题的来源
增加分词以后结果的准确度提高了，但是用户反映返回结果的速度很慢。原因是，Lucene做每一篇文档的相关关键词的高亮显示时，在运行时执行了很多遍的分词操作。这样降低了性能。
2、解决方法
在 Lucene1.4.3版本中的一个新功能可以解决这个问题。Term Vector现在支持保存Token.getPositionIncrement() 和Token.startOffset() 以及Token.endOffset() 信息。利用Lucene中新增加的Token信息的保存结果以后，就不需要为了高亮显示而在运行时解析每篇文档。通过Field方法控制是否保存该信息。修改HighlighterTest.java的代码如下：
//增加文档时保存Term位置信息。
private void addDoc(IndexWriter writer, String text) throws IOException
{
Document d = new Document();
//Field f = new Field(FIELD_NAME, text, true, true, true);
Field f = new Field(FIELD_NAME, text ,
Field.Store.YES, Field.Index.TOKENIZED,
Field.TermVector.WITH_POSITIONS_OFFSETS);
d.add(f);
writer.addDocument(d);
}
//利用Term位置信息节省Highlight时间。
void doStandardHighlights() throws Exception
{
Highlighter highlighter =new Highlighter(this,new QueryScorer(query));
highlighter.setTextFragmenter(new SimpleFragmenter(20));
for (int i = 0; i < hits.length(); i++)
{
String text = hits.doc(i).get(FIELD_NAME);
int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
TermPositionVector tpv = (TermPositionVector)reader.getTermFreqVector(hits.id(i),FIELD_NAME);
//如果没有stop words去除还可以改成 TokenSources.getTokenStream(tpv,true); 进一步提速。
TokenStream tokenStream=TokenSources.getTokenStream(tpv);
//analyzer.tokenStream(FIELD_NAME,new StringReader(text));
String result =
highlighter.getBestFragments(
tokenStream,
text,
maxNumFragmentsRequired,
fragmentSeparator);
System.out.println("\t" + result);
}
}
最后把highlight包中的一个额外的判断去掉。对于中文来说没有明显的单词界限，所以下面这个判断是错误的：
tokenGroup.isDistinct(token)
这样中文分词就不会影响到查询速度了。
给Lucene.NET增加中文分词
一、Lucene的.NET版本介绍
到目前为止，Lucene的C#移植有三个版本，最开始是NLucene，然后是Lucene.NET，当Lucene.NET转向商业化之后，SourceForge上又出现了dotLucene项目。
猎兔推出完全使用C#开发的，支持Lucene.NET的中文分词模块。
二、调用接口
seg.result.CnTokenizer，该类继承Lucene.Net.Analysis.TokenStream。
其中环境变量dic.dir指定数据文件路径，如：
"-Ddic.dir=d:/lg/work/SSeg/dic"
一个简单的使用例子是：
using System;
using System.Runtime.InteropServices;
using seg.result;
using Lucene.Net.Analysis;
namespace ConsoleApplication1
{
///
/// Class1 的摘要说明。
///
class Class1
{
///
/// 应用程序的主入口点。
///
[DllImport("Kernel32.DLL", SetLastError=true)]
public static extern bool SetEnvironmentVariable(string lpName, string lpValue);
[STAThread]
static void Main(string[] args)
{
SetEnvironmentVariable( "dic.dir", "d:/lg/work/SSeg/dic");
//
// TODO: 在此处添加代码以启动应用程序
//
testCnAnalyzer();
System.Console.Read();
}
public static void testCnAnalyzer()
{
System.IO.TextReader input;
CnTokenizer.makeTag= true;
string sentence = "邀请王振国今年9月参加在洛杉矶举行的30届美国治癌成就大奖会";
input = new System.IO.StringReader(sentence);
TokenStream tokenizer = new seg.result.CnTokenizer(input);
for (Token t = tokenizer.Next(); t != null; t = tokenizer.Next())
{
System.Console.WriteLine(t.TermText() + " " + t.StartOffset() + " "
+ t.EndOffset() + " "+t.Type());
}
}
}
}
三、输出结果介绍
输出结果中的词性标注代码和分词效果与当前Java版的一样，可以参考向Lucene增加中文分词功能。
dotLucene中文分词的highlight显示
1、准备的原料
lucene.net的1.4.3版本比java版的Lucene 1.4.3功能要少，所以需要lucene.net-1.9的版本。highlighter.net也用当前最新的版本1.4.0，但是这个版本的功能也比java当前版的功能要少，缺少一个实现快速显示highlight的类TokenSources。
2、TokenSources.cs的代码
using System;
using IComparer = System.Collections.IComparer;
using ArrayList = System.Collections.ArrayList;
using Analyzer = Lucene.Net.Analysis.Analyzer;
using Token = Lucene.Net.Analysis.Token;
using TokenStream = Lucene.Net.Analysis.TokenStream;
using IndexReader = Lucene.Net.Index.IndexReader;
using TermFreqVector = Lucene.Net.Index.TermFreqVector;
using TermPositionVector = Lucene.Net.Index.TermPositionVector;
using TermVectorOffsetInfo = Lucene.Net.Index.TermVectorOffsetInfo;
using Document = Lucene.Net.Documents.Document;
namespace Lucene.Net.Search.Highlight
{
///
/// TokenSources used for fast highlight,it's a must for chinese word segment.
///
public class TokenSources
{
/**
* A convenience method that tries a number of approaches to getting a token stream.
* The cost of finding there are no termVectors in the index is minimal (1000 invocations still
* registers 0 ms). So this "lazy" (flexible?) approach to coding is probably acceptable
* @param reader
* @param docId
* @param field
* @param analyzer
* @return null if field not stored correctly
* @throws IOException
*/
public static TokenStream GetAnyTokenStream(IndexReader reader,int docId, String field,Analyzer analyzer)
{
TokenStream ts=null;
TermFreqVector tfv=(TermFreqVector) reader.GetTermFreqVector(docId,field);
if(tfv!=null)
{
if(tfv is TermPositionVector)
{
ts=GetTokenStream((TermPositionVector) tfv);
}
}
//No token info stored so fall back to analyzing raw content
if(ts==null)
{
ts=GetTokenStream(reader,docId,field,analyzer);
}
return ts;
}
/**
*
* */
public static TokenStream GetTokenStream(TermPositionVector tpv)
{
//assumes the worst and makes no assumptions about token position sequences.
return GetTokenStream(tpv,false);
}
/**
* an object used to iterate across an array of tokens
* */
public class StoredTokenStream : TokenStream
{
Token[] tokens;
int currentToken=0;
/**
* */
public StoredTokenStream(Token[] tokens)
{
this.tokens=tokens;
}
/**
* */
public override Token Next()
{
if(currentToken>=tokens.Length)
{
return null;
}
return tokens[currentToken++];
}
}
class CompareClass : IComparer
{
public Int32 Compare(Object o1, Object o2)
{
Token t1=(Token) o1;
Token t2=(Token) o2;
if(t1.StartOffset()>t2.StartOffset())
return 1;
if(t1.StartOffset()
return -1;
return 0;
}
}
/**
* Low level api.
* Returns a token stream or null if no offset info available in index.
* This can be used to feed the highlighter with a pre-parsed token stream
*
* In my tests the speeds to recreate 1000 token streams using this method are:
* - with TermVector offset only data stored - 420 milliseconds
* - with TermVector offset AND position data stored - 271 milliseconds
* (nb timings for TermVector with position data are based on a tokenizer with contiguous
* positions - no overlaps or gaps)
* The cost of not using TermPositionVector to store
* pre-parsed content and using an analyzer to re-parse the original content:
* - reanalyzing the original content - 980 milliseconds
*
* The re-analyze timings will typically vary depending on -
* 1) The complexity of the analyzer code (timings above were using a
* stemmer/lowercaser/stopword combo)
* 2) The number of other fields (Lucene reads ALL fields off the disk
* when accessing just one document field - can cost dear!)
* 3) Use of compression on field storage - could be faster cos of compression (less disk IO)
* or slower (more CPU burn) depending on the content.
*
* @param tpv
* @param tokenPositionsGuaranteedContiguous true if the token position numbers have no overlaps or gaps. If looking
* to eek out the last drops of performance, set to true. If in doubt, set to false.
*/
public static TokenStream GetTokenStream(TermPositionVector tpv, bool tokenPositionsGuaranteedContiguous)
{
//System.out.println("fastfastfast");
//code to reconstruct the original sequence of Tokens
String[] terms=tpv.GetTerms();
int[] freq=tpv.GetTermFrequencies();
int totalTokens=0;
for (int t = 0; t < freq.Length; t++)
{
totalTokens+=freq[t];
}
Token[] tokensInOriginalOrder=new Token[totalTokens];
ArrayList unsortedTokens = null;
for (int t = 0; t < freq.Length; t++)
{
TermVectorOffsetInfo[] offsets=tpv.GetOffsets(t);
if(offsets==null)
{
return null;
}
int[] pos=null;
if(tokenPositionsGuaranteedContiguous)
{
//try get the token position info to speed up assembly of tokens into sorted sequence
pos=tpv.GetTermPositions(t);
}
if(pos==null)
{
//tokens NOT stored with positions or not guaranteed contiguous - must add to list and sort later
if(unsortedTokens==null)
{
unsortedTokens=new ArrayList();
}
for (int tp = 0; tp < offsets.Length; tp++)
{
unsortedTokens.Add(new Token(terms[t],
offsets[tp].GetStartOffset(),
offsets[tp].GetEndOffset()));
}
}
else
{
//We have positions stored and a guarantee that the token position information is contiguous
// This may be fast BUT wont work if Tokenizers used which create >1 token in same position or
// creates jumps in position numbers - this code would fail under those circumstances
//tokens stored with positions - can use this to index straight into sorted array
for (int tp = 0; tp < pos.Length; tp++)
{
tokensInOriginalOrder[pos[tp]]=new Token(terms[t],
offsets[tp].GetStartOffset(),
offsets[tp].GetEndOffset());
}
}
}
//If the field has been stored without position data we must perform a sort
if(unsortedTokens!=null)
{
tokensInOriginalOrder=(Token[]) unsortedTokens.ToArray(typeof( Token) );
System.Array.Sort(tokensInOriginalOrder, new CompareClass() );
}
return new StoredTokenStream(tokensInOriginalOrder);
}
/**
* */
public static TokenStream GetTokenStream(IndexReader reader,int docId, String field)
{
TermFreqVector tfv=(TermFreqVector) reader.GetTermFreqVector(docId,field);
if(tfv==null)
{
throw new Exception(field+" in doc #"+docId
+"does not have any term position data stored");
}
if(tfv is TermPositionVector)
{
TermPositionVector tpv=(TermPositionVector) reader.GetTermFreqVector(docId,field);
return GetTokenStream(tpv);
}
throw new Exception(field+" in doc #"+docId
+"does not have any term position data stored");
}
//convenience method
public static TokenStream GetTokenStream(IndexReader reader,int docId, String field,Analyzer analyzer)
{
Document doc=reader.Document(docId);
String contents=doc.Get(field);
if(contents==null)
{
throw new Exception("Field "+field +" in document #"+docId+ " is not stored and cannot be analyzed");
}
return analyzer.TokenStream(field,new System.IO.StringReader(contents));
}
}
}
3、 附加工作
去掉highlight包中的单词界限判断：
tokenGroup.isDistinct(token)
增加分词以后结果的准确度提高了，但是用户反映返回结果的速度很慢。原因是，Lucene做每一篇文档的相关关键词的高亮显示时，在运行时执行了很多遍的分词操作。这样降低了性能。
2、解决方法
在 Lucene1.4.3版本中的一个新功能可以解决这个问题。Term Vector现在支持保存Token.getPositionIncrement() 和Token.startOffset() 以及Token.endOffset() 信息。利用Lucene中新增加的Token信息的保存结果以后，就不需要为了高亮显示而在运行时解析每篇文档。通过Field方法控制是否保存该信息。修改HighlighterTest.java的代码如下：
//增加文档时保存Term位置信息。
private void addDoc(IndexWriter writer, String text) throws IOException
{
Document d = new Document();
//Field f = new Field(FIELD_NAME, text, true, true, true);
Field f = new Field(FIELD_NAME, text ,
Field.Store.YES, Field.Index.TOKENIZED,
Field.TermVector.WITH_POSITIONS_OFFSETS);
d.add(f);
writer.addDocument(d);
}
//利用Term位置信息节省Highlight时间。
void doStandardHighlights() throws Exception
{
Highlighter highlighter =new Highlighter(this,new QueryScorer(query));
highlighter.setTextFragmenter(new SimpleFragmenter(20));
for (int i = 0; i < hits.length(); i++)
{
String text = hits.doc(i).get(FIELD_NAME);
int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
TermPositionVector tpv = (TermPositionVector)reader.getTermFreqVector(hits.id(i),FIELD_NAME);
//如果没有stop words去除还可以改成 TokenSources.getTokenStream(tpv,true); 进一步提速。
TokenStream tokenStream=TokenSources.getTokenStream(tpv);
//analyzer.tokenStream(FIELD_NAME,new StringReader(text));
String result =
highlighter.getBestFragments(
tokenStream,
text,
maxNumFragmentsRequired,
fragmentSeparator);
System.out.println("\t" + result);
}
}
最后把highlight包中的一个额外的判断去掉。对于中文来说没有明显的单词界限，所以下面这个判断是错误的：
tokenGroup.isDistinct(token)
这样中文分词就不会影响到查询速度了。
给Lucene.NET增加中文分词
一、Lucene的.NET版本介绍
到目前为止，Lucene的C#移植有三个版本，最开始是NLucene，然后是Lucene.NET，当Lucene.NET转向商业化之后，SourceForge上又出现了dotLucene项目。
猎兔推出完全使用C#开发的，支持Lucene.NET的中文分词模块。
二、调用接口
seg.result.CnTokenizer，该类继承Lucene.Net.Analysis.TokenStream。
其中环境变量dic.dir指定数据文件路径，如：
"-Ddic.dir=d:/lg/work/SSeg/dic"
一个简单的使用例子是：
using System;
using System.Runtime.InteropServices;
using seg.result;
using Lucene.Net.Analysis;
namespace ConsoleApplication1
{
///
/// Class1 的摘要说明。
///
class Class1
{
///
/// 应用程序的主入口点。
///
[DllImport("Kernel32.DLL", SetLastError=true)]
public static extern bool SetEnvironmentVariable(string lpName, string lpValue);
[STAThread]
static void Main(string[] args)
{
SetEnvironmentVariable( "dic.dir", "d:/lg/work/SSeg/dic");
//
// TODO: 在此处添加代码以启动应用程序
//
testCnAnalyzer();
System.Console.Read();
}
public static void testCnAnalyzer()
{
System.IO.TextReader input;
CnTokenizer.makeTag= true;
string sentence = "邀请王振国今年9月参加在洛杉矶举行的30届美国治癌成就大奖会";
input = new System.IO.StringReader(sentence);
TokenStream tokenizer = new seg.result.CnTokenizer(input);
for (Token t = tokenizer.Next(); t != null; t = tokenizer.Next())
{
System.Console.WriteLine(t.TermText() + " " + t.StartOffset() + " "
+ t.EndOffset() + " "+t.Type());
}
}
}
}
三、输出结果介绍
输出结果中的词性标注代码和分词效果与当前Java版的一样，可以参考向Lucene增加中文分词功能。
dotLucene中文分词的highlight显示
1、准备的原料
lucene.net的1.4.3版本比java版的Lucene 1.4.3功能要少，所以需要lucene.net-1.9的版本。highlighter.net也用当前最新的版本1.4.0，但是这个版本的功能也比java当前版的功能要少，缺少一个实现快速显示highlight的类TokenSources。
2、TokenSources.cs的代码
using System;
using IComparer = System.Collections.IComparer;
using ArrayList = System.Collections.ArrayList;
using Analyzer = Lucene.Net.Analysis.Analyzer;
using Token = Lucene.Net.Analysis.Token;
using TokenStream = Lucene.Net.Analysis.TokenStream;
using IndexReader = Lucene.Net.Index.IndexReader;
using TermFreqVector = Lucene.Net.Index.TermFreqVector;
using TermPositionVector = Lucene.Net.Index.TermPositionVector;
using TermVectorOffsetInfo = Lucene.Net.Index.TermVectorOffsetInfo;
using Document = Lucene.Net.Documents.Document;
namespace Lucene.Net.Search.Highlight
{
///
/// TokenSources used for fast highlight,it's a must for chinese word segment.
///
public class TokenSources
{
/**
* A convenience method that tries a number of approaches to getting a token stream.
* The cost of finding there are no termVectors in the index is minimal (1000 invocations still
* registers 0 ms). So this "lazy" (flexible?) approach to coding is probably acceptable
* @param reader
* @param docId
* @param field
* @param analyzer
* @return null if field not stored correctly
* @throws IOException
*/
public static TokenStream GetAnyTokenStream(IndexReader reader,int docId, String field,Analyzer analyzer)
{
TokenStream ts=null;
TermFreqVector tfv=(TermFreqVector) reader.GetTermFreqVector(docId,field);
if(tfv!=null)
{
if(tfv is TermPositionVector)
{
ts=GetTokenStream((TermPositionVector) tfv);
}
}
//No token info stored so fall back to analyzing raw content
if(ts==null)
{
ts=GetTokenStream(reader,docId,field,analyzer);
}
return ts;
}
/**
*
* */
public static TokenStream GetTokenStream(TermPositionVector tpv)
{
//assumes the worst and makes no assumptions about token position sequences.
return GetTokenStream(tpv,false);
}
/**
* an object used to iterate across an array of tokens
* */
public class StoredTokenStream : TokenStream
{
Token[] tokens;
int currentToken=0;
/**
* */
public StoredTokenStream(Token[] tokens)
{
this.tokens=tokens;
}
/**
* */
public override Token Next()
{
if(currentToken>=tokens.Length)
{
return null;
}
return tokens[currentToken++];
}
}
class CompareClass : IComparer
{
public Int32 Compare(Object o1, Object o2)
{
Token t1=(Token) o1;
Token t2=(Token) o2;
if(t1.StartOffset()>t2.StartOffset())
return 1;
if(t1.StartOffset()
return -1;
return 0;
}
}
/**
* Low level api.
* Returns a token stream or null if no offset info available in index.
* This can be used to feed the highlighter with a pre-parsed token stream
*
* In my tests the speeds to recreate 1000 token streams using this method are:
* - with TermVector offset only data stored - 420 milliseconds
* - with TermVector offset AND position data stored - 271 milliseconds
* (nb timings for TermVector with position data are based on a tokenizer with contiguous
* positions - no overlaps or gaps)
* The cost of not using TermPositionVector to store
* pre-parsed content and using an analyzer to re-parse the original content:
* - reanalyzing the original content - 980 milliseconds
*
* The re-analyze timings will typically vary depending on -
* 1) The complexity of the analyzer code (timings above were using a
* stemmer/lowercaser/stopword combo)
* 2) The number of other fields (Lucene reads ALL fields off the disk
* when accessing just one document field - can cost dear!)
* 3) Use of compression on field storage - could be faster cos of compression (less disk IO)
* or slower (more CPU burn) depending on the content.
*
* @param tpv
* @param tokenPositionsGuaranteedContiguous true if the token position numbers have no overlaps or gaps. If looking
* to eek out the last drops of performance, set to true. If in doubt, set to false.
*/
public static TokenStream GetTokenStream(TermPositionVector tpv, bool tokenPositionsGuaranteedContiguous)
{
//System.out.println("fastfastfast");
//code to reconstruct the original sequence of Tokens
String[] terms=tpv.GetTerms();
int[] freq=tpv.GetTermFrequencies();
int totalTokens=0;
for (int t = 0; t < freq.Length; t++)
{
totalTokens+=freq[t];
}
Token[] tokensInOriginalOrder=new Token[totalTokens];
ArrayList unsortedTokens = null;
for (int t = 0; t < freq.Length; t++)
{
TermVectorOffsetInfo[] offsets=tpv.GetOffsets(t);
if(offsets==null)
{
return null;
}
int[] pos=null;
if(tokenPositionsGuaranteedContiguous)
{
//try get the token position info to speed up assembly of tokens into sorted sequence
pos=tpv.GetTermPositions(t);
}
if(pos==null)
{
//tokens NOT stored with positions or not guaranteed contiguous - must add to list and sort later
if(unsortedTokens==null)
{
unsortedTokens=new ArrayList();
}
for (int tp = 0; tp < offsets.Length; tp++)
{
unsortedTokens.Add(new Token(terms[t],
offsets[tp].GetStartOffset(),
offsets[tp].GetEndOffset()));
}
}
else
{
//We have positions stored and a guarantee that the token position information is contiguous
// This may be fast BUT wont work if Tokenizers used which create >1 token in same position or
// creates jumps in position numbers - this code would fail under those circumstances
//tokens stored with positions - can use this to index straight into sorted array
for (int tp = 0; tp < pos.Length; tp++)
{
tokensInOriginalOrder[pos[tp]]=new Token(terms[t],
offsets[tp].GetStartOffset(),
offsets[tp].GetEndOffset());
}
}
}
//If the field has been stored without position data we must perform a sort
if(unsortedTokens!=null)
{
tokensInOriginalOrder=(Token[]) unsortedTokens.ToArray(typeof( Token) );
System.Array.Sort(tokensInOriginalOrder, new CompareClass() );
}
return new StoredTokenStream(tokensInOriginalOrder);
}
/**
* */
public static TokenStream GetTokenStream(IndexReader reader,int docId, String field)
{
TermFreqVector tfv=(TermFreqVector) reader.GetTermFreqVector(docId,field);
if(tfv==null)
{
throw new Exception(field+" in doc #"+docId
+"does not have any term position data stored");
}
if(tfv is TermPositionVector)
{
TermPositionVector tpv=(TermPositionVector) reader.GetTermFreqVector(docId,field);
return GetTokenStream(tpv);
}
throw new Exception(field+" in doc #"+docId
+"does not have any term position data stored");
}
//convenience method
public static TokenStream GetTokenStream(IndexReader reader,int docId, String field,Analyzer analyzer)
{
Document doc=reader.Document(docId);
String contents=doc.Get(field);
if(contents==null)
{
throw new Exception("Field "+field +" in document #"+docId+ " is not stored and cannot be analyzed");
}
return analyzer.TokenStream(field,new System.IO.StringReader(contents));
}
}
}
3、 附加工作
去掉highlight包中的单词界限判断：
tokenGroup.isDistinct(token)
