井长 Jason Yu

ftp:

http://ftp.ubuntu.org.cn






163 source.list

deb http://mirrors.163.com/ubuntu/ jaunty main restricted universe multiverse
deb http://mirrors.163.com/ubuntu/ jaunty-security main restricted universe multiverse
deb http://mirrors.163.com/ubuntu/ jaunty-updates main restricted universe multiverse
deb http://mirrors.163.com/ubuntu/ jaunty-proposed main restricted universe multiverse
deb http://mirrors.163.com/ubuntu/ jaunty-backports main restricted universe multiverse
deb-src http://mirrors.163.com/ubuntu/ jaunty main restricted universe multiverse
deb-src http://mirrors.163.com/ubuntu/ jaunty-security main restricted universe multiverse
deb-src http://mirrors.163.com/ubuntu/ jaunty-updates main restricted universe multiverse
deb-src http://mirrors.163.com/ubuntu/ jaunty-proposed main restricted universe multiverse
deb-src http://mirrors.163.com/ubuntu/ jaunty-backports main restricted universe multiverse

cat /proc/version

uname -a

lsb_release -a

对于当今大流量的网站，每天几千万甚至上亿的流量,是如何解决访问量问题的呢?以下是一些总结的方法:

[SWF(width="800", height="600", backgroundColor="#fffff", frameRate="31")] //定义场景
import flash.display.Stage;//表示场景类
import flash.display.StageScaleMode;//调整大小场景类，常用有NO_SCALE跟据场景大小来调整自适应大小
import flash.display.StageAlign;//调整对齐场景类
import flash.display.StageDisplayState//调整场景是否全屏
import flash.events.FullScreenEvent;//用于侦听"调整场景是否全屏"

StageScaleMode.EXACT_FIT 按比例缩放 SWF。
StageScaleMode.SHOW_ALL 确定是否显示边框（就像在标准电视上观看宽屏电影时显示的黑条）。
StageScaleMode.NO_BORDER 确定是否可以部分裁切内容。
StageScaleMode.NO_SCALE，则当查看者调整 Flash Player 窗口大小时，舞台内容将保持定义的大小。
swfStage.addEventListener(Event.RESIZE, resizeDisplay);
mySprite.stage.displayState = StageDisplayState.FULL_SCREEN;//全屏mySprite.stage.displayState = StageDisplayState.NORMAL;//退出全屏 mySprite.stage.addEventListener(FullScreenEvent.FULL_SCREEN, fullScreenRedraw);
swfStage.align = StageAlign.TOP_LEFT;//左上角对齐swfStage.align = StageAlign.TOP_RIGHT;//右上角对齐swfStage.align = StageAlign.TOP;//顶对齐swfStage.align = StageAlign.RIGHT;//右对齐swfStage.align = StageAlign.LEFT;//左对齐swfStage.align = StageAlign.BOTTOM;//底对齐swfStage.align = StageAlign.BOTTOM_LEFT;//左下角对齐swfStage.align = StageAlign.BOTTOM_RIGHT;//右下角对齐

package {
import flash.display.Sprite;
import flash.display.MovieClip;
import flash.display.Stage;
import flash.display.StageScaleMode;
import flash.display.StageAlign;
import flash.events.Event;
public class StageScaleMode1 extends Sprite {  
private var swfStage:Stage;//定义变量swfStage为场景变量***  
private var top:my_top=new my_top();  
private var bot:my_top=new my_top();  
public function StageScaleMode1 () {  
addChild(top);  
addChild(bot);  
swfStage = top.stage;//定义一个要跟随场景变化的变量***  
swfStage.scaleMode = StageScaleMode.NO_SCALE;//申明场景变swfStage大小为自定义于场景大小***   swfStage.align = StageAlign.TOP_LEFT;//对齐方试跟据元件内***  
swfStage.addEventListener (Event.RESIZE,stagescale);//大小侦听***  
}  
private function stagescale (e:Event) {  
top.scaleX = swfStage.stage.stageWidth;//top的自动宽度  
bot.scaleX = swfStage.stage.stageWidth;//bot的自动宽度  
bot.y=stage.stageHeight;   bot.alpha=.2;  
}
}
}

From: http://highscalability.com

收藏一些常用的快捷方式

1、问题的来源

增加分词以后结果的准确度提高了，但是用户反映返回结果的速度很慢。原因是，Lucene做每一篇文档的相关关键词的高亮显示时，在运行时执行了很多遍的分词操作。这样降低了性能。



2、解决方法

在 Lucene1.4.3版本中的一个新功能可以解决这个问题。Term Vector现在支持保存Token.getPositionIncrement() 和Token.startOffset() 以及Token.endOffset() 信息。利用Lucene中新增加的Token信息的保存结果以后，就不需要为了高亮显示而在运行时解析每篇文档。通过Field方法控制是否保存该信息。修改HighlighterTest.java的代码如下：



//增加文档时保存Term位置信息。

private void addDoc(IndexWriter writer, String text) throws IOException

{

Document d = new Document();

//Field f = new Field(FIELD_NAME, text, true, true, true);

Field f = new Field(FIELD_NAME, text ,

Field.Store.YES, Field.Index.TOKENIZED,

Field.TermVector.WITH_POSITIONS_OFFSETS);

d.add(f);

writer.addDocument(d);

}



//利用Term位置信息节省Highlight时间。

void doStandardHighlights() throws Exception

{

Highlighter highlighter =new Highlighter(this,new QueryScorer(query));

highlighter.setTextFragmenter(new SimpleFragmenter(20));

for (int i = 0; i < hits.length(); i++)

{

String text = hits.doc(i).get(FIELD_NAME);

int maxNumFragmentsRequired = 2;

String fragmentSeparator = "...";

TermPositionVector tpv = (TermPositionVector)reader.getTermFreqVector(hits.id(i),FIELD_NAME);

//如果没有stop words去除还可以改成 TokenSources.getTokenStream(tpv,true); 进一步提速。

TokenStream tokenStream=TokenSources.getTokenStream(tpv);

//analyzer.tokenStream(FIELD_NAME,new StringReader(text));



String result =

highlighter.getBestFragments(

tokenStream,

text,

maxNumFragmentsRequired,

fragmentSeparator);

System.out.println("\t" + result);

}

}



最后把highlight包中的一个额外的判断去掉。对于中文来说没有明显的单词界限，所以下面这个判断是错误的：

tokenGroup.isDistinct(token)



这样中文分词就不会影响到查询速度了。



给Lucene.NET增加中文分词
一、Lucene的.NET版本介绍

到目前为止，Lucene的C#移植有三个版本，最开始是NLucene，然后是Lucene.NET，当Lucene.NET转向商业化之后，SourceForge上又出现了dotLucene项目。

猎兔推出完全使用C#开发的，支持Lucene.NET的中文分词模块。



二、调用接口



seg.result.CnTokenizer，该类继承Lucene.Net.Analysis.TokenStream。



其中环境变量dic.dir指定数据文件路径，如：

"-Ddic.dir=d:/lg/work/SSeg/dic"



一个简单的使用例子是：

using System;

using System.Runtime.InteropServices;

using seg.result;

using Lucene.Net.Analysis;



namespace ConsoleApplication1

{

///

/// Class1 的摘要说明。

///

class Class1

{

///

/// 应用程序的主入口点。

///

[DllImport("Kernel32.DLL", SetLastError=true)]

public static extern bool SetEnvironmentVariable(string lpName, string lpValue);



[STAThread]

static void Main(string[] args)

{

SetEnvironmentVariable( "dic.dir", "d:/lg/work/SSeg/dic");

//

// TODO: 在此处添加代码以启动应用程序

//

testCnAnalyzer();

System.Console.Read();

}



public static void testCnAnalyzer()

{

System.IO.TextReader input;



CnTokenizer.makeTag= true;

string sentence = "邀请王振国今年9月参加在洛杉矶举行的30届美国治癌成就大奖会";



input = new System.IO.StringReader(sentence);

TokenStream tokenizer = new seg.result.CnTokenizer(input);



for (Token t = tokenizer.Next(); t != null; t = tokenizer.Next())

{

System.Console.WriteLine(t.TermText() + " " + t.StartOffset() + " "

+ t.EndOffset() + " "+t.Type());

}

}

}

}



三、输出结果介绍

输出结果中的词性标注代码和分词效果与当前Java版的一样，可以参考向Lucene增加中文分词功能。




dotLucene中文分词的highlight显示
1、准备的原料

lucene.net的1.4.3版本比java版的Lucene 1.4.3功能要少，所以需要lucene.net-1.9的版本。highlighter.net也用当前最新的版本1.4.0，但是这个版本的功能也比java当前版的功能要少，缺少一个实现快速显示highlight的类TokenSources。



2、TokenSources.cs的代码



using System;



using IComparer = System.Collections.IComparer;

using ArrayList = System.Collections.ArrayList;



using Analyzer = Lucene.Net.Analysis.Analyzer;

using Token = Lucene.Net.Analysis.Token;

using TokenStream = Lucene.Net.Analysis.TokenStream;

using IndexReader = Lucene.Net.Index.IndexReader;

using TermFreqVector = Lucene.Net.Index.TermFreqVector;

using TermPositionVector = Lucene.Net.Index.TermPositionVector;

using TermVectorOffsetInfo = Lucene.Net.Index.TermVectorOffsetInfo;

using Document = Lucene.Net.Documents.Document;



namespace Lucene.Net.Search.Highlight

{

///

/// TokenSources used for fast highlight,it's a must for chinese word segment.

///

public class TokenSources

{

/**

* A convenience method that tries a number of approaches to getting a token stream.

* The cost of finding there are no termVectors in the index is minimal (1000 invocations still

* registers 0 ms). So this "lazy" (flexible?) approach to coding is probably acceptable

* @param reader

* @param docId

* @param field

* @param analyzer

* @return null if field not stored correctly

* @throws IOException

*/

public static TokenStream GetAnyTokenStream(IndexReader reader,int docId, String field,Analyzer analyzer)

{

TokenStream ts=null;



TermFreqVector tfv=(TermFreqVector) reader.GetTermFreqVector(docId,field);

if(tfv!=null)

{

if(tfv is TermPositionVector)

{

ts=GetTokenStream((TermPositionVector) tfv);

}

}

//No token info stored so fall back to analyzing raw content

if(ts==null)

{

ts=GetTokenStream(reader,docId,field,analyzer);

}

return ts;

}



/**

*

* */

public static TokenStream GetTokenStream(TermPositionVector tpv)

{

//assumes the worst and makes no assumptions about token position sequences.

return GetTokenStream(tpv,false);

}





/**

* an object used to iterate across an array of tokens

* */

public class StoredTokenStream : TokenStream

{

Token[] tokens;

int currentToken=0;



/**

* */

public StoredTokenStream(Token[] tokens)

{

this.tokens=tokens;

}



/**

* */

public override Token Next()

{

if(currentToken>=tokens.Length)

{

return null;

}

return tokens[currentToken++];

}

}



class CompareClass : IComparer

{

public Int32 Compare(Object o1, Object o2)

{

Token t1=(Token) o1;

Token t2=(Token) o2;

if(t1.StartOffset()>t2.StartOffset())

return 1;

if(t1.StartOffset()
return -1;

return 0;

}

}

/**

* Low level api.

* Returns a token stream or null if no offset info available in index.

* This can be used to feed the highlighter with a pre-parsed token stream

*

* In my tests the speeds to recreate 1000 token streams using this method are:

* - with TermVector offset only data stored - 420 milliseconds

* - with TermVector offset AND position data stored - 271 milliseconds

* (nb timings for TermVector with position data are based on a tokenizer with contiguous

* positions - no overlaps or gaps)

* The cost of not using TermPositionVector to store

* pre-parsed content and using an analyzer to re-parse the original content:

* - reanalyzing the original content - 980 milliseconds

*

* The re-analyze timings will typically vary depending on -

* 1) The complexity of the analyzer code (timings above were using a

* stemmer/lowercaser/stopword combo)

* 2) The number of other fields (Lucene reads ALL fields off the disk

* when accessing just one document field - can cost dear!)

* 3) Use of compression on field storage - could be faster cos of compression (less disk IO)

* or slower (more CPU burn) depending on the content.

*

* @param tpv

* @param tokenPositionsGuaranteedContiguous true if the token position numbers have no overlaps or gaps. If looking

* to eek out the last drops of performance, set to true. If in doubt, set to false.

*/

public static TokenStream GetTokenStream(TermPositionVector tpv, bool tokenPositionsGuaranteedContiguous)

{

//System.out.println("fastfastfast");

//code to reconstruct the original sequence of Tokens

String[] terms=tpv.GetTerms();

int[] freq=tpv.GetTermFrequencies();

int totalTokens=0;

for (int t = 0; t < freq.Length; t++)

{

totalTokens+=freq[t];

}

Token[] tokensInOriginalOrder=new Token[totalTokens];

ArrayList unsortedTokens = null;

for (int t = 0; t < freq.Length; t++)

{

TermVectorOffsetInfo[] offsets=tpv.GetOffsets(t);

if(offsets==null)

{

return null;

}



int[] pos=null;

if(tokenPositionsGuaranteedContiguous)

{

//try get the token position info to speed up assembly of tokens into sorted sequence

pos=tpv.GetTermPositions(t);

}

if(pos==null)

{

//tokens NOT stored with positions or not guaranteed contiguous - must add to list and sort later

if(unsortedTokens==null)

{

unsortedTokens=new ArrayList();

}

for (int tp = 0; tp < offsets.Length; tp++)

{

unsortedTokens.Add(new Token(terms[t],

offsets[tp].GetStartOffset(),

offsets[tp].GetEndOffset()));

}

}

else

{

//We have positions stored and a guarantee that the token position information is contiguous



// This may be fast BUT wont work if Tokenizers used which create >1 token in same position or

// creates jumps in position numbers - this code would fail under those circumstances



//tokens stored with positions - can use this to index straight into sorted array

for (int tp = 0; tp < pos.Length; tp++)

{

tokensInOriginalOrder[pos[tp]]=new Token(terms[t],

offsets[tp].GetStartOffset(),

offsets[tp].GetEndOffset());

}

}

}

//If the field has been stored without position data we must perform a sort

if(unsortedTokens!=null)

{

tokensInOriginalOrder=(Token[]) unsortedTokens.ToArray(typeof( Token) );



System.Array.Sort(tokensInOriginalOrder, new CompareClass() );

}

return new StoredTokenStream(tokensInOriginalOrder);

}



/**

* */

public static TokenStream GetTokenStream(IndexReader reader,int docId, String field)

{

TermFreqVector tfv=(TermFreqVector) reader.GetTermFreqVector(docId,field);

if(tfv==null)

{

throw new Exception(field+" in doc #"+docId

+"does not have any term position data stored");

}

if(tfv is TermPositionVector)

{

TermPositionVector tpv=(TermPositionVector) reader.GetTermFreqVector(docId,field);

return GetTokenStream(tpv);

}

throw new Exception(field+" in doc #"+docId

+"does not have any term position data stored");

}



//convenience method

public static TokenStream GetTokenStream(IndexReader reader,int docId, String field,Analyzer analyzer)

{

Document doc=reader.Document(docId);

String contents=doc.Get(field);

if(contents==null)

{

throw new Exception("Field "+field +" in document #"+docId+ " is not stored and cannot be analyzed");

}

return analyzer.TokenStream(field,new System.IO.StringReader(contents));

}

}

}



3、 附加工作

去掉highlight包中的单词界限判断：

tokenGroup.isDistinct(token)

CLucene - a C++ search engine http://sourceforge.net/projects/clucene/

传统的全文检索都是基于数据库的，Sql Server Oracle mysql 都提供全文检索，但这些比较大，不适合单机或小应用程序(Mysql4.0以上可以作为整合开发)，Mysql也不支持中文。
后来得知Apache有一个开源的全文检索引擎，而且应用比较广，Lucene是Apache旗下的JAVA版的全文检索引擎,性能相当出色，可惜是 java版的，我一直在想有没有C或C++版的，终于有一天在http://sourceforge.net 淘到一个好东东，Clucene！CLucene是C++版的全文检索引擎,完全移植于Lucene，不过对中文支持不好，而且有很多的内存泄露，
Cluene 不支持中文的分词，我就写了一个简单的中文分词，大概思路就是传统的二分词法，因为中文的分词不像英文这类的语言，一遇到空格或标点就认为是一个词的结束，所以就采用二分词法，二分词法就是例如:北京市，就切成 北京 ，京市。这样一来词库就会很大，不过是一种简单的分词方法(过段时间我再介绍我对中文分词的一些思路) ，当然了，在检索时就不能输入“北京市”了，这样就检索不到，只要输入：“+北京 +京市”,就可以检索到北京市了，虽然精度不是很高，但适合简单的分词，而且不怕会漏掉某些单词。
我照着Clucene的分词模块，做了一个ChineseTokenizer，这个模块就负责分词工作了，我把主要的函数写出来

ChineseTokenizer.cpp:

Token* ChineseTokenizer::next() {


while(!rd.Eos())
{
char_t ch = rd.GetNext();


if( isSpace((char_t)ch)!=0 )
{
continue;
}
// Read for Alpha-Nums and Chinese
if( isAlNum((char_t)ch)!=0 )
{
start = rd.Column();

return ReadChinese(ch);
}
}
return NULL;
}

Token* ChineseTokenizer::ReadChinese(const char_t prev)
{
bool isChinese = false;
StringBuffer str;
str.append(prev);

char_t ch = prev;

if(((char_t)ch>>&&(char_t)ch>=0xa0)
isChinese = true;

while(!rd.Eos() && isSpace((char_t)ch)==0 )
{

ch = rd.GetNext();

if(isAlNum((char_t)ch)!=0)
{
//是数学或英语就读到下一个空格.或下一个汉字
//是汉字.就读下一个汉字组成词组,或读到空格或英文结束
if(isChinese)
{
//汉字,并且ch是汉字
if(((char_t)ch>>&&(char_t)ch>=0xa0)
{
// 返回上一个汉字
str.append(ch);
rd.UnGet();
// wprintf(_T("[%s]"),str);
return new Token(str.getBuffer(), start, rd.Column(), tokenImage[lucene::analysis::chinese::CHINESE] );
}
else
{
//是字母或数字或空格
rd.UnGet();
// wprintf(_T("[%s]"),str);
return new Token(str.getBuffer(), start, rd.Column(), tokenImage[lucene::analysis::chinese::CHINESE] );
}
}
else
{
//非汉字
// ch是汉字
if(((char_t)ch>>&&(char_t)ch>=0xa0)
{
// wprintf(_T("[%s]"),str);
rd.UnGet();
return new Token(str.getBuffer(), start, rd.Column(), tokenImage[lucene::analysis::chinese::CHINESE] );
}
str.append( ch );
}
}

}
// wprintf(_T("[%s]"),str);
return new Token(str.getBuffer(), start, rd.Column(), tokenImage[lucene::analysis::chinese::ALPHANUM] );
}


同时，这个中文分词不支持文件，只能支持内存流的形式，因为我用到了rd.UnGet();如果是文件的话，嘿嘿，只能回退半个字节哦