文件名称:
C# 蜘蛛Spider 网页抓取器 Crawler
开发工具:
文件大小: 56kb
下载次数: 0
上传时间: 2009-12-30
详细说明： C# 蜘蛛Spider 网页抓取器 void ParseUri(MyUri uri, ref MyWebRequest request) { string strStatus = ""; // check if connection is kept alive from previous connections or not if(request != null && request.response.KeepAlive) strStatus += "Connection live to: "+uri.Host+"\r\n\r\n"; else strStatus += "Connecting: "+uri.Host+"\r\n\r\n"; ListViewItem itemLog = null; Monitor.Enter(this.listViewThreads); try { // update thread information in the threads view list itemLog = this.listViewThreads.Items[int.Parse(Thread.CurrentThread.Name)]; int nDepth = uri.Depth; itemLog.SubItems[1].Text = nDepth.ToString(); itemLog.ImageIndex = 1; itemLog.BackColor = Color.WhiteSmoke; // initialize status to Connect itemLog.SubItems[2].Text = "Connect"; itemLog.ForeColor = Color.Red; itemLog.SubItems[3].Text = uri.AbsoluteUri; itemLog.SubItems[4].Text = ""; itemLog.SubItems[5].Text = ""; } catch(Exception) { } Monitor.Exit(this.listViewThreads); try { // create web request request = MyWebRequest.Create(uri, request, KeepAlive); // set request timeout request.Timeout = RequestTimeout*1000; // retrieve response from web request MyWebResponse response = request.GetResponse(); // update status text with the request and response headers strStatus += request.Header+response.Header; // check for redirection if(response.ResponseUri.Equals(uri) == false) { // add the new uri to the queue this.EnqueueUri(new MyUri(response.ResponseUri.AbsoluteUri), true); // update status strStatus += "Redirected to: "+response.ResponseUri+"\r\n"; // log current uri status LogUri(uri.AbsoluteUri, strStatus); // reset current request to avoid response socket opening case request = null; return; } // check for allowed MIME types if(AllMIMETypes == false && response.ContentType != null && MIMETypes.Length > 0) { string strContentType = response.ContentType.ToLower(); int nExtIndex = strContentType.IndexOf(';'); if(nExtIndex != -1) strContentType = strContentType.Substring(0, nExtIndex); if(strContentType.IndexOf('*') == -1 && (nExtIndex = MIMETypes.IndexOf(strContentType)) == -1) { LogError(uri.AbsoluteUri, strStatus+"\r\nUnlisted Content-Type ("+strContentType+"), check settings."); request = null; return; } // find numbers Match match = new Regex(@"\d+").Match(MIMETypes, nExtIndex); int nMin = int.Parse(match.Value)*1024; match = match.NextMatch(); int nMax = int.Parse(match.Value)*1024; if(nMin < nMax && (response.ContentLength < nMin || response.ContentLength > nMax)) { LogError(uri.AbsoluteUri, strStatus+"\r\nContentLength limit error ("+response.ContentLength+")"); request = null; return; } } // check for response extention string[] ExtArray = { ".gif", ".jpg", ".css", ".zip", ".exe" }; bool bParse = true; foreach(string ext in ExtArray) if(uri.AbsoluteUri.ToLower().EndsWith(ext) == true) { bParse = false; break; } foreach(string ext in ExcludeFiles) if(ext.Trim().Length > 0 && uri.AbsoluteUri.ToLower().EndsWith(ext) == true) { bParse = false; break; } // construct path in the hard disk string strLocalPath = uri.LocalPath; // check if the path ends with / to can crate the file on the HD if(strLocalPath.EndsWith("/") == true) // check if there is no query like (.asp?i=32&j=212) if(uri.Query == "") // add a default name for / ended pathes strLocalPath += "default.html"; // check if the uri includes a query string if(uri.Query != "") // construct the name from the query hash value to be the same if we download it again strLocalPath += uri.Query.GetHashCode()+".html"; // construct the full path folder string BasePath = this.Downloadfolder+"\\"+uri.Host+Path.GetDirectoryName(uri.AbsolutePath); // check if the folder not found if(Directory.Exists(BasePath) == false) // create the folder Directory.CreateDirectory(BasePath); // construct the full path name of the file string PathName = this.Downloadfolder+"\\"+uri.Host+strLocalPath.Replace("%20", " "); // open the output file FileStream streamOut = File.Open(PathName, FileMode.Create, FileAccess.Write, FileShare.ReadWrite); BinaryWriter writer = new BinaryWriter(streamOut); itemLog.SubItems[2].Text = "Download"; itemLog.ForeColor = Color.Black; // receive response buffer string strResponse = ""; byte[] RecvBuffer = new byte[10240]; int nBytes, nTotalBytes = 0; // loop to receive response buffer while((nBytes = response.socket.Receive(RecvBuffer, 0, 10240, SocketFlags.None)) > 0) { // increment total received bytes nTotalBytes += nBytes; // write received buffer to file writer.Write(RecvBuffer, 0, nBytes); // check if the uri type not binary to can be parsed for refs if(bParse == true) // add received buffer to response string strResponse += Encoding.ASCII.GetString(RecvBuffer, 0, nBytes); // update view text itemLog.SubItems[4].Text = Commas(nTotalBytes); if(response.ContentLength > 0) itemLog.SubItems[5].Text = '%'+(100-(response.ContentLength-nTotalBytes)*100/response.ContentLength).ToString(); // check if connection Keep-Alive to can break the loop if response completed if(response.KeepAlive && nTotalBytes >= response.ContentLength && response.ContentLength > 0) break; } // close output stream writer.Close(); streamOut.Close(); if(response.KeepAlive) strStatus += "Connection kept alive to be used in subpages.\r\n"; else { // close response response.Close(); strStatus += "Connection closed.\r\n"; } // update status strStatus += Commas(nTotalBytes)+" bytes, downloaded to \""+PathName+"\"\r\n"; // increment total file count FileCount++; // increment total bytes count ByteCount += nTotalBytes; if(ThreadsRunning == true && bParse == true && uri.Depth < WebDepth) { strStatus += "\r\nParsing page ...\r\n"; // check for restricted words foreach(string strExcludeWord in ExcludeWords) if(strExcludeWord.Trim().Length > 0 && strResponse.IndexOf(strExcludeWord) != -1) { LogError(uri.AbsoluteUri, strStatus+"\r\nPage includes reserved word ("+strExcludeWord+")"); EraseItem(itemLog); File.Delete(PathName); return; } // parse the page to search for refs string strRef = @"(href|HREF|src|SRC)[ ]*=[ ]*[""'][^""'#>]+[""']"; MatchCollection matches = new Regex(strRef).Matches(strResponse); strStatus += "Found: "+matches.Count+" ref(s)\r\n"; URLCount += matches.Count; foreach(Match match in matches) { strRef = match.Value.Substring(match.Value.IndexOf('=')+1).Trim('"', '\'', '#', ' ', '>'); try { if(strRef.IndexOf("..") != -1 || strRef.StartsWith("/") == true || strRef.StartsWith("http://") == false) strRef = new Uri(uri, strRef).AbsoluteUri; Normalize(ref strRef); MyUri newUri = new MyUri(strRef); if(newUri.Scheme != Uri.UriSchemeHttp && newUri.Scheme != Uri.UriSchemeHttps) continue; if(newUri.Host != uri.Host && KeepSameServer == true) continue; newUri.Depth = uri.Depth+1; if(this.EnqueueUri(newUri, true) == true) strStatus += newUri.AbsoluteUri+"\r\n"; } catch(Exception) { } } } LogUri(uri.AbsoluteUri, strStatus); } catch(Exception e) { LogError(uri.AbsoluteUri, strStatus+e.Message); request = null; } finally { EraseItem(itemLog); } } ...展开收缩
(系统自动生成,下载前可以参看下载内容)
下载文件列表
相关说明
- 本站资源为会员上传分享交流与学习,如有侵犯您的权益,请联系我们删除.
- 本站是交换下载平台，提供交流渠道，下载内容来自于网络，除下载问题外，其它问题请自行百度。
- 本站已设置防盗链，请勿用迅雷、QQ旋风等多线程下载软件下载资源，下载后用WinRAR最新版进行解压.
- 如果您发现内容无法下载，请稍后再次尝试；或者到消费记录里找到下载记录反馈给我们.
- 下载后发现下载的内容跟说明不相乎，请到消费记录里找到下载记录反馈给我们，经确认后退回积分.
- 如下载前有疑问，可以通过点击"提供者"的名字，查看对方的联系方式，联系对方咨询.
