private bool FetchWebPage(ref String result, String url)
{
bool ret = false;
try {
WebClient client = new WebClient();
String ua = "Mozilla/5.0 (compatible; PRoCon 1; " + GetPluginName() + ")";
// XXX String ua = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; .NET CLR 3.5.30729)";
if (DebugLevel >= 8) DebugFetch("Using user-agent: " + ua);
client.Headers.Add("user-agent", ua);
DateTime since = DateTime.Now;
result = client.DownloadString(url);
/* TESTS
String testUrl = "http://status.savanttools.com/?code=";
html_data = client.DownloadString(testUrl + "429%20Too%20Many%20Requests");
//html_data = client.DownloadString(testUrl + "509%20Bandwidth%20Limit%20Exceeded");
//html_data = client.DownloadString(testUrl + "408%20Request%20Timeout");
//html_data = client.DownloadString(testUrl + "404%20Not%20Found");
*/
DebugFetch("^2^bTIME^n took " + DateTime.Now.Subtract(since).TotalSeconds.ToString("F2") + " secs, url: " + url);
if (Regex.Match(result, @"that\s+page\s+doesn't\s+exist", RegexOptions.IgnoreCase | RegexOptions.Singleline).Success) {
DebugFetch("^b" + url + "^n does not exist", 3);
result = String.Empty;
return false;
}
ret = true;
} catch (WebException e) {
if (DebugLevel >= 3 && DebugLevel < 7) DebugFetch("FAILED for url: " + url, 3);
if (e.Status.Equals(WebExceptionStatus.Timeout)) {
if (DebugLevel >= 3) DebugFetch("WEB EXCEPTION: HTTP request timed-out", 3);
} else {
if (DebugLevel >= 3) DebugFetch("WEB EXCEPTION: " + e.Message, 3);
}
DebugWrite("Full exception: " + e.ToString(), 7);
ret = false;
} catch (Exception ae) {
if (DebugLevel >= 3 && DebugLevel < 7) DebugFetch("FAILED for url: " + url, 3);
if (DebugLevel >= 3) DebugFetch("EXCEPTION: " + ae.Message, 3);
DebugWrite("Full exception: " + ae.ToString(), 7);
ret = false;
}
return ret;
}