被管理员和谐了的最高票答案“知乎数据抓取程序”（.net、c#数据挖掘）

<span class="name">伍新生</span>，<span class="bio" title="5颜6色的情感，我终生的寻求！">5颜6色的情感，我终生的寻求！</span> </div> </div> <div class="body clearfix"> <div class="zm-profile-header-avatar-container self"> <img alt="伍新生" src="http://pic4.zhimg.com/94cc60166_l.jpg" class="zm-profile-header-img zg-avatar-big zm-avatar-editor-preview"/> <span class="zm-entry-head-avatar-edit-button">修改头像</span>

private List<Question> GetQuestions(string source) { List<Question> questions = new List<Question>(); string startStr = "<a class="question_link""; if (source.IndexOf(startStr) != ⑴) { Question q = new Question(); string content = Cinser.Common.StringPlus.SubString(source, startStr, "</a>"); q.Id = Cinser.Common.StringPlus.SubString(content, "question/", """); q.Title = content.Substring(content.IndexOf(">") + 1); q.Time = DateTime.Now; questions.Add(q); source = source.Substring(source.IndexOf(startStr) + startStr.Length); questions.AddRange(GetQuestions(source)); } return questions; }

public partial class Form1 : Form { DataProvider dal; string watchingURL = string.Empty; int LoopCount = 0; public Form1() { InitializeComponent(); dal = new DataProvider(); dal.AddLog("程序启动"); base.WindowState = FormWindowState.Minimized; base.Show(); base.Hide(); base.WindowState = FormWindowState.Normal; base.ShowInTaskbar = false; base.TopMost = false; base.MaximizeBox = false; base.MinimizeBox = false; base.ControlBox = false; //设置循环监测时间 int interval = int.Parse(Cinser.Common.ConfigurationHelper.GetAppSettingsValue("Interval")); timer1.Interval = interval * 1000; watchingURL = Cinser.Common.ConfigurationHelper.GetAppSettingsValue("WatchingURL"); RunWhenStart(true, "zhApp.exe", """ + Application.StartupPath + "zhApp.exe" AutoRun"); Run(); dal.AddLog("程序初始化成功"); LoopCount += 1; } //设置程序开机自启动 public void RunWhenStart(bool Started, string name, string path) { RegistryKey HKLM = Registry.CurrentUser; RegistryKey Run = HKLM.CreateSubKey(@"SOFTWAREMicrosoftWindowsCurrentVersionRun"); if (Started == true) { try { Run.SetValue(name, path); HKLM.Close(); } catch(Exception ex)//没有权限会异常 { throw ex; } } else { try { Run.DeleteValue(name); HKLM.Close(); } catch (Exception ex)//没有权限会异常 { throw ex; } } } /// <summary> /// 运行监测流程 /// </summary> private void Run() { List<Question> questions = GetQuestions(Cinser.Common.HttpHelper.GetString(watchingURL)); string ids = dal.GetExistIdsStr(); for (int i = 0; i < questions.Count; i++) { if (ids.IndexOf(questions[i].Id) != ⑴) { questions.Remove(questions[i]); i--; } else { if (ids == string.Empty) ids = questions[i].Id; else ids += "," + questions[i].Id; } } if (questions.Count > 0) { SendQuestions(questions); dal.WriteIdStrToTxt(ids); dal.Add(questions); dal.AddLog(string.Format("获得了{0}条新数据。", questions.Count)); } } /// <summary> /// 从监测站点源数据中抓取问题 /// </summary> private List<Question> GetQuestions(string source) { List<Question> questions = new List<Question>(); string startStr = "<a class="question_link""; if (source.IndexOf(startStr) != ⑴) { Question q = new Question(); string content = Cinser.Common.StringPlus.SubString(source, startStr, "</a>"); q.Id = Cinser.Common.StringPlus.SubString(content, "question/", """); q.Title = content.Substring(content.IndexOf(">") + 1); q.Time = DateTime.Now; questions.Add(q); source = source.Substring(source.IndexOf(startStr) + startStr.Length); questions.AddRange(GetQuestions(source)); } return questions; } private bool SendQuestions(List<Question> questions) { bool bSuccess = true; List<string> strMailAddressTo = Cinser.Common.ConfigurationHelper.GetAppSettingsValue("strMailAddressTo").Split(',').ToList(); string smtpAddress = Cinser.Common.ConfigurationHelper.GetAppSettingsValue("smtpAddress"); string sendEmailFrom = Cinser.Common.ConfigurationHelper.GetAppSettingsValue("sendEmailFrom"); string sendEmailFromPwd = Cinser.Common.ConfigurationHelper.GetAppSettingsValue("sendEmailFromPwd"); string emailName = Cinser.Common.ConfigurationHelper.GetAppSettingsValue("EmailName"); //密码解密，开源的话就去掉这步吧，这样配置的时候直接配置明文密码就行。 //sendEmailFromPwd = Cinser.Common.Security.DecryptDES(sendEmailFromPwd, "yuiophgf"); string msg = string.Empty; SendCompletedEventHandler s = new SendCompletedEventHandler(SendCompleted); string content = GetQustionsListStr(questions); content += " 信息来源于：" + watchingURL; Cinser.Common.SmtpEmailSend.SendEmail(strMailAddressTo, emailName + DateTime.Now.ToString(), content, smtpAddress, 0x19, sendEmailFrom, sendEmailFromPwd, "163测试邮箱", null, out msg, s); return bSuccess; } private void SendCompleted(object sender, AsyncCompletedEventArgs e) { } private string GetQustionsListStr(List<Question> questions) { string content = string.Empty; if (questions != null && questions.Count > 0) { content = string.Format("名称：{0},url：{1}", questions[0].Title, questions[0].Url); for (int i = 1; i < questions.Count; i++) { content += string.Format(" 名称：{0},url：{1}", questions[i].Title, questions[i].Url); } } return content; } private void timer1_Tick(object sender, EventArgs e) { int interval = int.Parse(Cinser.Common.ConfigurationHelper.GetAppSettingsValue("Interval")); timer1.Interval = interval * 1000; Run(); dal.AddLog(string.Format("程序循环次数：{0}", LoopCount++)); } }

public class Question { string id, title, url, type, remark; DateTime time; public string Remark { get { return remark; } set { remark = value; } } public DateTime Time { get { return time; } set { time = value; } } public string Type { get { return type; } set { type = value; } } public string Url { get { if (string.IsNullOrEmpty(url)) { url = string.Format("http://www.zhihu.com/question/{0}", Id); } return url; } set { url = value; } } public string Title { get { return title; } set { title = value; } } public string Id { get { return id; } set { id = value; } } }

/// <summary> /// 知乎问题数据表操作Provider /// </summary> public class DataProvider { private string connStr = string.Empty; Cinser.DBUtility.DAL.OracleDALCommon dal; string txtPath = ""; string logPath = ""; string debugPath = string.Empty; public string DebugPath { get { if (debugPath == string.Empty) debugPath = System.AppDomain.CurrentDomain.BaseDirectory; if (debugPath.EndsWith("") == false) { debugPath += ""; } return debugPath; } set { debugPath = value; } } public string LogPath { get { if (logPath == string.Empty) { logPath = DebugPath + "Log.txt"; } return logPath; } } public DataProvider() { dal = new Cinser.DBUtility.DAL.OracleDALCommon(this.ConnStr); } public string TxtPath { get { if (txtPath == string.Empty) { txtPath = DebugPath + "Ids.txt"; } return txtPath; } } public string ConnStr { get { if (connStr == string.Empty) { connStr = Cinser.Common.ConfigurationHelper.GetAppSettingsValue("connStr"); } return connStr; } set { connStr = value; } } public bool CanConnectOracleServer { get { return dal.Open(); } } /// <summary> /// 将抓取到的问题写入oracle数据库中 /// </summary> /// <param name="questions"></param> /// <returns></returns> public bool Add(List<Question> questions) { bool bReturn = false; try { for (int i = 0; i < questions.Count; i++) { dal.Add("qustions", questions[i]); } bReturn = true; } catch { } return bReturn; } public DataTable GetQustions(string sqlWhere = "1=1") { try { DataTable dt = dal.GetDataList("qustions", sqlWhere); return dt; } catch { return null; } } public bool IsExist(string id) { try { string sqlWhere = "id='" + id + "'"; DataTable dt = dal.GetDataList("qustions", sqlWhere); return dt.Rows.Count > 0; } catch { return false; } } /// <summary> /// 获得已抓取过的问题ID字符串 /// </summary> /// <returns></returns> public string GetExistIdsStr() { string ids = string.Empty; //如果能连上远程的oracle服务器则从oracle数据库中取ID字符串 if (CanConnectOracleServer) { DataTable dt = GetQustions(); if (dt != null && ᨲ
鱸Ç૴Ç⚶ﭺ嫭蛿윺ꕎ腄衐⬤첁ꍪ쪀ྤ㐉妐揉䂲▱䧅䈩∧䤄閵ᢙꈴᦉ‴आᤩ꘤䤂ꒁ级鿩俐헕掱롸羖껡㺵���⎾ɂ꩒ሴ붤ﭽ粰髹｝￳忺磩ꄲ쿚㭅吺꺤絞⨏痦襸鞘詨韠⩳㕌蝹䴒壉턼骢⎪勰標째죶俶섾ﾟ䧸ﱛ瞴䯆ﱤ轧䭲ﳷ燱ﱺ쓻ᴗ﮿鴢╉ⴓ濴㑇튡玙������䏿㉳���躑짿���槰过￟矲勞旙ꓩ瓱녽���軿퇾俧ﲺ쯙閹습맵㙜宛熹䳄羡荷ꓔ턴籕���ﯳ꧆䣫빱ꛯ㴿㽆鿼ம藹���坢槨﹡慾Ꞵ���ㄕஏ靣︽Ὠ봯䟱爖駙쏼ﲮ⹟뷻ﻋ잵楱籘⓰횿羕춝ꏾ喇Ꮋ콁쩺㷋맧訯圗娩䴖낭嚼뮘宜���㑳煨굾歴鿞䢿穖䕨培拾⍋蓴댖줝㚶瞿缥맣滛ꇽ㐛展ⴛ῞Ꮄ렡⿤ꊾ渎쎛談콫���ﺇ㦝耗얹୥㐠ﯷ䩺䏝犑���ꈝ︉滼㬩迣팎଴緽觹뒧죓촂㼶騲ᵛ៷誖믐皁﫾繂糰���緊優ጏ㳽뾕䵵濤���寸ⓗ㍭ㇰ薀꓆朧ដﰖㇹꭧ럼᤯뵚︽꽴댿䃂绪샃忎䩾끫䝃哪毚髳䍉抝���矟⧕폠뿃穷麝뉽躻흎備쾣⯨鉺見淽㪝伛櫬ྰむ껅ꁮ꥜걫횸ᡛ尟䟴靮䝺敨䞅秧厉貣菣뒽뙐ﻹ홰绯ᾕ쟮⳴킐煿땾㊰䎻佇ḵￏ