中文分词

ChineseParse.cs

@H_403_5@using System; using System.Collections; using System.IO; using System.Text.RegularExpressions; namespace FullTextSearch.Common { /// <summary> /// 中文分词器。 /// </summary> public class ChineseParse { private static readonly ChineseWordsHashCountSet _countTable; static ChineseParse() { _countTable = new ChineseWordsHashCountSet(); InitFromFile("ChineseDictionary.txt"); } /// <summary> /// 从指定的文件中初始化中文词语字典和字符串次数字典。 /// </summary> /// <param name="fileName">文件名</param> private static void InitFromFile(string fileName) { string path = Path.Combine(Directory.GetCurrentDirectory(),@"..\..\Common\",fileName); if (File.Exists(path)) { using (StreamReader sr = File.OpenText(path)) { string s = ""; while ((s = sr.ReadLine()) != null) { ChineseWordUnit _tempUnit = InitUnit(s); _countTable.InsertWord(_tempUnit.Word); } } } } /// <summary> /// 将一个字符串解析为ChineseWordUnit。 /// </summary> /// <param name="s">字符串</param> /// <returns>解析得到的ChineseWordUnit</returns> /// 4 /// 0 private static ChineseWordUnit InitUnit(string s) { var reg = new Regex(@"\s+"); string[] temp = reg.Split(s); //if (temp.Length != 2) //{ // throw new Exception("字符串解析错误：" + s); //} if (temp.Length != 1) { throw new Exception("字符串解析错误：" + s); } return new ChineseWordUnit(temp[0],Int32.Parse("1")); } /// <summary> /// 分析输入的字符串，将其切割成一个个的词语。 /// </summary> /// <param name="s">待切割的字符串</param> /// <returns>所切割得到的中文词语数组</returns> public static string[] ParseChinese(string s) { int _length = s.Length; string _temp = String.Empty; var _words = new ArrayList(); for (int i = 0; i < s.Length;) { _temp = s.Substring(i,1); if (_countTable.GetCount(_temp) > 1) { int j = 2; for (; i + j < s.Length + 1 && _countTable.GetCount(s.Substring(i,j)) > 0; j++) { } _temp = s.Substring(i,j - 1); i = i + j - 2; } i++; _words.Add(_temp); } var _tempStringArray = new string[_words.Count]; _words.CopyTo(_tempStringArray); return _tempStringArray; } } }

ChineseWordsHashCountSet.cs

@H_403_5@using System.Collections; namespace FullTextSearch.Common { /// <summary> /// 记录字符串出现在中文字典所录中文词语的前端的次数的字典类。如字符串"中"出现在"中国"的前端，则在字典中记录一个次数。 /// </summary> public class ChineseWordsHashCountSet { /// <summary> /// 记录字符串在中文词语中出现次数的Hashtable。键为特定的字符串，值为该字符串在中文词语中出现的次数。 /// </summary> private readonly Hashtable _rootTable; /// <summary> /// 类型初始化。 /// </summary> public ChineseWordsHashCountSet() { _rootTable = new Hashtable(); } /// <summary> /// 查询指定字符串出现在中文字典所录中文词语的前端的次数。 /// </summary> /// <param name="s">指定字符串</param> /// <returns>字符串出现在中文字典所录中文词语的前端的次数。若为-1，表示不出现。</returns> public int GetCount(string s) { if (!_rootTable.ContainsKey(s.Length)) { return -1; } var _tempTable = (Hashtable) _rootTable[s.Length]; if (!_tempTable.ContainsKey(s)) { return -1; } return (int) _tempTable[s]; } /// <summary> /// 向次数字典中插入一个词语。解析该词语，插入次数字典。 /// </summary> /// <param name="s">所处理的字符串。</param> public void InsertWord(string s) { for (int i = 0; i < s.Length; i++) { string _s = s.Substring(0,i + 1); InsertSubString(_s); } } /// <summary> /// 向次数字典中插入一个字符串的次数记录。 /// </summary> /// <param name="s">所插入的字符串。</param> private void InsertSubString(string s) { if (!_rootTable.ContainsKey(s.Length) && s.Length > 0) { var _newHashtable = new Hashtable(); _rootTable.Add(s.Length,_newHashtable); } var _tempTable = (Hashtable) _rootTable[s.Length]; if (!_tempTable.ContainsKey(s)) { _tempTable.Add(s,1); } else { _tempTable[s] = (int) _tempTable[s] + 1; } } } }

ChineseWordUnit.cs

@H_403_5@namespace FullTextSearch.Common { public struct ChineseWordUnit { private readonly int _power; private readonly string _word; /// <summary> /// 结构初始化。 /// </summary> /// <param name="word">中文词语</param> /// <param name="power">该词语的权重</param> public ChineseWordUnit(string word,int power) { _word = word; _power = power; } /// <summary> /// 中文词语单元所对应的中文词。 /// </summary> public string Word { get { return _word; } } /// <summary> /// 该中文词语的权重。 /// </summary> public int Power { get { return _power; } } } }

ChineseDictionary.txt

主窗体界面

MainManager.cs

@H_403_5@using System; using System.Collections.Generic; using System.Data; using System.Drawing; using System.Windows.Forms; using FullTextSearch.Common; using Npgsql; namespace FullTextSearch { public partial class MainManager : Form { private readonly Postgresql pg = new Postgresql(); private readonly sqlquerys sqlQuerys = new sqlquerys(); private char analysisType; private string createConnString = ""; private DataSet dataSet = new DataSet(); private DataTable dataTable = new DataTable(); private char odabirAndOr; private char vrstaPretrazivanja; public MainManager() { InitializeComponent(); rbtn_AND.Checked = true; rbtnNeizmjenjeni.Checked = true; odabirAndOr = '*'; radioButton_Day.Checked = true; radioButton_Day.Checked = true; } private void Form1_Load(object sender,EventArgs e) { gb_unosPodataka.Enabled = false; groupBox_Search.Enabled = false; groupBox_Analysis.Enabled = false; button_Disconnect.Enabled = false; button_Pretrazi.BackColor = Color.WhiteSmoke; button_Disconnect.BackColor = Color.WhiteSmoke; button_unosTekstaUBazu.BackColor = Color.WhiteSmoke; button1.BackColor = Color.WhiteSmoke; } private void button_unosTekstaUBazu_Click(object sender,EventArgs e) { string searchTextBoxString = rTB_unosTextaUBazu.Text; if (searchTextBoxString != "") { pg.insertIntoTable(searchTextBoxString,pg.conn); MessageBox.Show(searchTextBoxString + " 添加到数据库!"); rTB_unosTextaUBazu.Clear(); } else { MessageBox.Show("不允许空数据!"); } } private void button_Pretrazi_Click(object sender,EventArgs e) { string stringToSearch; string sql; string highlitedText; string rank; string check; stringToSearch = txt_Search.Text.Trim(); var list = new List<string>(ChineseParse.ParseChinese(stringToSearch)); ; sql = sqlQuerys.createsqlString(list,odabirAndOr,vrstaPretrazivanja); richTextBox1.Text = sql; check = sqlQuerys.testIfEmpty(stringToSearch); pg.insertIntoAnalysisTable(stringToSearch,pg.conn); pg.openConnection(); var command = new NpgsqlCommand(sql,pg.conn); NpgsqlDataReader reader = command.ExecuteReader(); int count = 0; linkLabel_Rezultat.Text = " "; while (reader.Read()) { highlitedText = reader[1].ToString(); rank = reader[3].ToString(); linkLabel_Rezultat.Text += highlitedText + "[" + rank + "]\n"; count++; } labelBrojac.Text = "找到的文件数量: " + count; pg.closeConnection(); } private void rbtn_AND_CheckedChanged(object sender,EventArgs e) { odabirAndOr = '*'; } private void rbtn_OR_CheckedChanged(object sender,EventArgs e) { odabirAndOr = '+'; } private void rbtnNeizmjenjeni_CheckedChanged(object sender,EventArgs e) { vrstaPretrazivanja = 'A'; } private void rbtn_Rijecnici_CheckedChanged(object sender,EventArgs e) { vrstaPretrazivanja = 'B'; } private void rbtn_Fuzzy_CheckedChanged(object sender,EventArgs e) { vrstaPretrazivanja = 'C'; } private void button_Connect_Click(object sender,EventArgs e) { if (connectMe()) { gb_unosPodataka.Enabled = true; groupBox_Search.Enabled = true; groupBox_Analysis.Enabled = true; textBox_Database.Enabled = false; textBox_IP.Enabled = false; textBox_Port.Enabled = false; textBox_Password.Enabled = false; textBox_UserID.Enabled = false; button_Connect.Enabled = false; button_Disconnect.Enabled = true; button_Pretrazi.BackColor = Color.SkyBlue; button_Disconnect.BackColor = Color.IndianRed; button_unosTekstaUBazu.BackColor = Color.MediumSeaGreen; button1.BackColor = Color.MediumSeaGreen; button_Connect.BackColor = Color.WhiteSmoke; } } private void button_Disconnect_Click(object sender,EventArgs e) { gb_unosPodataka.Enabled = false; groupBox_Search.Enabled = false; groupBox_Analysis.Enabled = false; textBox_Database.Enabled = true; textBox_IP.Enabled = true; textBox_Port.Enabled = true; textBox_Password.Enabled = true; textBox_UserID.Enabled = true; button_Connect.Enabled = true; button_Disconnect.Enabled = false; button_Pretrazi.BackColor = Color.WhiteSmoke; button_Disconnect.BackColor = Color.WhiteSmoke; button_unosTekstaUBazu.BackColor = Color.WhiteSmoke; button1.BackColor = Color.WhiteSmoke; button_Connect.BackColor = Color.MediumSeaGreen; txt_Search.Text = ""; linkLabel_Rezultat.Text = ""; richTextBox1.Text = ""; labelBrojac.Text = ""; } private bool connectMe() { createConnString += "Server=" + textBox_IP.Text + ";Port=" + textBox_Port.Text + ";User Id=" + textBox_UserID.Text + ";Password=" + textBox_Password.Text + ";Database=" + textBox_Database.Text + ";"; sqlQuerys.setTheKey(createConnString); pg.setConnectionString(); pg.setConnection(); if (pg.openConnection()) { MessageBox.Show("您已成功连接!"); pg.closeConnection(); return true; } return false; } private void button1_Click(object sender,EventArgs e) { string selectedTimestamp; selectedTimestamp = dateTimePicker_From.Value.ToString("dd-MM-yyyy hh:mm:ss") + " " + dateTimePicker_To.Value.ToString("dd-MM-yyyy hh:mm:ss"); var analize = new Analysis(selectedTimestamp,analysisType); analize.Show(); } private void radioButton_Day_CheckedChanged(object sender,EventArgs e) { analysisType = 'D'; } private void radioButton_Hour_CheckedChanged(object sender,EventArgs e) { analysisType = 'H'; } } }

sqlquerys.cs代码:

@H_403_5@using System.Collections.Generic; namespace FullTextSearch { internal class sqlquerys { private static string giveMeTheKey; private static int tempInt = 1; //设置连接字符串 public void setTheKey(string connString) { giveMeTheKey = connString; giveMeTheKey += ""; } //将连接字符串存储在静态变量中 public string getTheKey() { giveMeTheKey += ""; return giveMeTheKey; } public void setCounter() { tempInt = 1; } //根据AND和OR的选择分析字符串进行搜索 public string createFunctionString(List<string> searchList,char selector) { string TempString = ""; string[] TempField = null; int i = 0; int j = 0; foreach (string searchStringInList in searchList) { if (j != 0) { if (selector == '+') TempString = TempString + " | "; else if (selector == '*') TempString = TempString + " & "; } j = 1; TempField = splitListForInput(searchStringInList); TempString = TempString + "("; foreach (string justTempString in TempField) { if (i != 0) { TempString = TempString + " & "; } TempString = TempString + justTempString; i = 1; } TempString = TempString + ")"; i = 0; } return TempString; } //帮助方法 public List<string> splitInputField(string[] inputField) { var unfinishedList = new List<string>(); foreach (string splitString in inputField) { unfinishedList.Add(splitString); } return unfinishedList; } //帮助方法 public string[] splitListForInput(string inputString) { string[] parsedList = null; parsedList = inputString.Split(' '); return parsedList; } //在Postgresql中创建ts功能的功能，用于字典搜索 public string createTsFunction(string tsString) { string tsHeadline = ""; string tsRank = ""; string tsFunction = ""; tsHeadline = ",\n ts_headline(\"content\",to_tsquery('" + tsString + "')),\"content\""; tsRank = ",\n ts_rank(to_tsvector(\"content\"),to_tsquery('" + tsString + "')) rank"; tsFunction = tsHeadline + tsRank; return tsFunction; } //创建SQL查询依赖于选择哪种类型的搜索，也取决于AND或OR选择器 public string createsqlString(List<string> searchList,char selector,char vrstaPretrazivanja) { string selectString = ""; string myTempString = ""; string TempString = ""; int i = 0; TempString = createFunctionString(searchList,selector); TempString = createTsFunction(TempString); selectString = "SELECT \"id\"" + TempString + "\nFROM \"texttable\" \nWHERE "; if (vrstaPretrazivanja == 'A') { foreach (string myString in searchList) { if (i == 0) { myTempString = myTempString + "\"content\" LIKE '%" + myString + "%' "; i++; } else { if (selector == '*') myTempString = myTempString + "\nAND \"content\" LIKE '%" + myString + "%' "; else if (selector == '+') myTempString = myTempString + "\nOR \"content\" LIKE '%" + myString + "%' "; } } } else if (vrstaPretrazivanja == 'B') { foreach (string myString in searchList) { string temporalString = ""; string[] testingString = myString.Split(' '); for (int k = 0; k < testingString.Length; k++) { if (k != testingString.Length - 1) { temporalString += testingString[k] + " & "; } else { temporalString += testingString[k]; } } if (i == 0) { myTempString = myTempString + "to_tsvector(\"content\") @@ to_tsquery('english','" + temporalString + "')"; i++; } else { if (selector == '*') myTempString = myTempString + "\nAND to_tsvector(\"content\") @@ to_tsquery('english','" + temporalString + "')"; else if (selector == '+') myTempString = myTempString + "\nOR to_tsvector(\"content\") @@ to_tsquery('english','" + temporalString + "')"; } } } if (vrstaPretrazivanja == 'C') { foreach (string myString in searchList) { if (i == 0) { myTempString = myTempString + "\"content\" % '" + myString + "' "; i++; } else { if (selector == '*') myTempString = myTempString + "\nAND \"content\" % '" + myString + "' "; else if (selector == '+') myTempString = myTempString + "\nOR \"content\" % '" + myString + "' "; } } } selectString = selectString + myTempString + "\nORDER BY rank DESC"; return selectString; } public string testIfEmpty(string searchedText) { string checkingIfEmpty = "SELECT * FROM \"analysisTable\" WHERE \"searchedtext\" =' " + searchedText + "'"; return checkingIfEmpty; } public string queryForAnalysis(char analysisChoice) { string myTestsql = ""; if (analysisChoice == 'H') { //这个查询是这样写的只是为了测试的目的，它需要改变 myTestsql = "SELECT * FROM crosstab('SELECT CAST((\"searchedtext\") AS text) searchedText," + " CAST(EXTRACT(HOUR FROM \"timeOfSearch\") AS int) AS sat,CAST(COUNT(*) AS int) AS broj FROM \"analysisTable\" GROUP BY \"searchedText\",sat" + " ORDER BY \"searchedtext\",sat','SELECT rbrSata FROM sat ORDER BY rbrSata') AS pivotTable (\"searchedText\" TEXT,t0_1 INT,t1_2 INT" + ",t2_3 INT,t3_4 INT,t4_5 INT,t5_6 INT,t6_7 INT,t7_8 INT,t8_9 INT,t9_10 INT,t10_11 INT,t11_12 INT,t12_13 INT" + ",t13_14 INT,t14_15 INT,t15_16 INT,t16_17 INT,t17_18 INT,t18_19 INT,t19_20 INT,t20_21 INT,t21_22 INT,t22_23 INT,t23_00 INT) ORDER BY \"searchedText\""; return myTestsql; } if (analysisChoice == 'D') { //这个查询是这样写的只是为了测试的目的，它需要改变 myTestsql += "SELECT *FROM crosstab ('SELECT CAST((\"searchedtext\") AS text) AS searchedText,CAST(EXTRACT(DAY FROM \"dateOfSearch\") AS int) AS dan" + "," + "dan ORDER BY \"searchedtext\",dan','SELECT rbrDana FROM dan ORDER BY rbrDana') AS pivotTable(\"searchedtext\" TEXT"; return myTestsql; } return myTestsql; } //此方法用于解析日期 public int[] parseForDates(string date) { string[] temp; var tempInt = new int[3]; temp = date.Split('-'); for (int i = 0; i < 3; i++) { tempInt[i] = int.Parse(temp[i]); } return tempInt; } //此代码用于创建分析，它执行一些日期/时间操作，以便能够为选定的日期/时间创建分析。 public string createsqlForDayAnalysis(string dateFrom,string dateTo) { string insertIntoTempTable = ""; string dateTimeForAnalysis = ""; int[] tempFrom = parseForDates(dateFrom); int[] tempTo = parseForDates(dateTo); //月份变更算法 while (tempFrom[0] != tempTo[0] || tempFrom[1] != tempTo[1]) { if (tempFrom[1] == tempTo[1]) { if (tempFrom[0] != tempTo[0]) { for (int i = tempInt + 1; tempFrom[0] + 2 < tempTo[0] + 2; i++) { insertIntoTempTable += "INSERT INTO \"dan\" VALUES (" + i + ");"; dateTimeForAnalysis += ",dd" + tempFrom[0] + tempFrom[1] + tempFrom[2] + " INT"; tempInt = i; tempFrom[0]++; } } } if (tempFrom[1] != tempTo[1]) { if (tempFrom[1]%2 == 0 || tempFrom[1] == 7 || tempFrom[1] == 1) { for (int i = tempInt; tempFrom[0] < 31 && tempFrom[1] != tempTo[1]; i++) { insertIntoTempTable += "INSERT INTO \"dan\" VALUES (" + i + ");"; dateTimeForAnalysis += ",dd" + tempFrom[0] + tempFrom[1] + tempFrom[2] + " INT"; tempInt = i; tempFrom[0]++; if (tempFrom[0] == 31) { tempFrom[1]++; tempFrom[0] = 1; } } } } } dateTimeForAnalysis += ") ORDER BY \"searchedtext\""; return dateTimeForAnalysis + "#" + insertIntoTempTable; } } }

Postgresql.cs代码:

@H_403_5@using System; using System.Windows.Forms; using Npgsql; using NpgsqlTypes; namespace FullTextSearch { public class Postgresql { private static int tempInt = 1; private readonly sqlquerys sql = new sqlquerys(); public NpgsqlConnection conn; public string connectionstring; private string newConnString; public Postgresql() { setConnectionString(); setConnection(); } public void setConnectionString() { newConnString = sql.getTheKey(); connectionstring = String.Format(newConnString); setConnection(); } public void setConnection() { conn = new NpgsqlConnection(connectionstring); } public bool openConnection() { try { conn.Open(); return true; } catch { MessageBox.Show("Unable to connect! Check parameters!"); return false; } } public void closeConnection() { conn.Close(); } public void insertIntoTable(string textToInsert,NpgsqlConnection nsqlConn) { string MysqLString = "INSERT INTO \"texttable\" (\"content\") VALUES (@Param1)"; var myParameter = new NpgsqlParameter("@Param1",NpgsqlDbType.Text); myParameter.Value = textToInsert; openConnection(); var myCommand = new NpgsqlCommand(MysqLString,nsqlConn); myCommand.Parameters.Add(myParameter); myCommand.ExecuteNonQuery(); closeConnection(); } public void insertIntoAnalysisTable(string textToInsert,NpgsqlConnection nsqlConn) { string dateTime = DateTime.Now.ToString(); string[] temp; temp = dateTime.Split(' '); string MysqLString = "INSERT INTO \"analysistable\" (\"searchedtext\",\"dateofsearch\",\"timeofsearch\") VALUES ('" + textToInsert + "','" + temp[0] + "'" + ",'" + temp[1] + "');"; openConnection(); var myCommand = new NpgsqlCommand(MysqLString,nsqlConn); myCommand.ExecuteNonQuery(); closeConnection(); } public void executeQuery(string queryText,NpgsqlConnection nsqlConn) { openConnection(); var myCommand = new NpgsqlCommand(queryText,nsqlConn); myCommand.ExecuteNonQuery(); closeConnection(); } public void createTempTable(NpgsqlConnection nsqlConn,char analysisType,string dateFrom,string dateTo,string splitMe) { if (analysisType == 'H') { string dropIfExists = "DROP TABLE IF EXISTS \"sat\";"; string createTempTable = "CREATE TABLE IF NOT EXISTS \"sat\" (rbrSata INT);"; string insertIntoTempTable = ""; for (int i = 0; i < 24; i++) { insertIntoTempTable += "INSERT INTO \"sat\" VALUES (" + i + ");"; } openConnection(); var commandDrop = new NpgsqlCommand(dropIfExists,nsqlConn); commandDrop.ExecuteNonQuery(); var commandCreate = new NpgsqlCommand(createTempTable,nsqlConn); commandCreate.ExecuteNonQuery(); var commandInsert = new NpgsqlCommand(insertIntoTempTable,nsqlConn); commandInsert.ExecuteNonQuery(); closeConnection(); } else if (analysisType == 'D') { string dropIfExists = "DROP TABLE IF EXISTS \"dan\";"; string createTempTable = "CREATE TABLE IF NOT EXISTS \"dan\" (rbrDana INT);"; string insertIntoTempTable = splitMe; openConnection(); var commandDrop = new NpgsqlCommand(dropIfExists,nsqlConn); commandInsert.ExecuteNonQuery(); closeConnection(); } } } }

PostGresql sql脚本:

@H_403_5@CREATE TABLE public.analysistable ( id integer NOT NULL DEFAULT nextval('analysistable_id_seq'::regclass),searchedtext text COLLATE pg_catalog."default" NOT NULL,dateofsearch date NOT NULL,timeofsearch time without time zone NOT NULL,CONSTRAINT analysistable_pkey PRIMARY KEY (id) ) WITH ( OIDS = FALSE ) TABLESPACE pg_default; ALTER TABLE public.analysistable OWNER to king; @H_403_5@CREATE TABLE public.texttable ( id integer NOT NULL DEFAULT nextval('texttable_id_seq'::regclass),content text COLLATE pg_catalog."default" NOT NULL,CONSTRAINT texttable_pkey PRIMARY KEY (id) ) WITH ( OIDS = FALSE ) TABLESPACE pg_default; ALTER TABLE public.texttable OWNER to king;

使用PostGreSQL数据库进行text录入和text检索