public static string GetWebSiteContent(string url, string type)
{
try
{
//CrawlingData cData = new CrawlingData();
string _requestUrl = url.Trim();
string _encodingType = "utf-8";
FileWebRequest request = (FileWebRequest)WebRequest.Create(_requestUrl);
//request..AllowAutoRedirect = true;
//request.MaximumAutomaticRedirections = 3;
//request.UserAgent = "Mozilla/6.0 (MSIE 6.0; Windows NT 5.1; Arya.NET Robot)";
//request. = true; request.Timeout = 15 * 1000;
System.Net.FileWebResponse response = (FileWebResponse)request.GetResponse();
//showExtractConetent.InnerHtml = ((HttpWebResponse)response).StatusDescription;
//showExtractConetent.InnerHtml += "<br/>";
//_encodingType = GetEncodingType(response);
System.IO.StreamReader reader = new System.IO.StreamReader
(response.GetResponseStream(), Encoding.GetEncoding("utf-8"));
// Read the content.
string responseFromServer = reader.ReadToEnd();
return(responseFromServer);
}
catch (WebException ex)
{
string response = GetWebSiteContent(url);
return(response);
}
}