public static string DownloadHTMLPage(string URL)
{
string pageContent = null;
try
{
HttpWebRequest httpWebRequest = (HttpWebRequest)HttpWebRequest.Create(URL);
httpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)";
httpWebRequest.Referer = "http://www.google.com/";
httpWebRequest.Timeout = 10000;
WebResponse response = httpWebRequest.GetResponse();
Stream webStream = response.GetResponseStream();
StreamReader streamReader = new StreamReader(webStream);
pageContent = streamReader.ReadToEnd();
streamReader.Close();
webStream.Close();
response.Close();
}
catch (WebException ex)
{
Console.WriteLine("HTTP ERROR [{0}] : {1}", URL, ex.ToString());
return null;
}
return pageContent;
}