Check the ContentType : Web Crawler : Network C# Source Code

    



using System;
using System.IO;
using System.Net;
   
class HtmlDump
{
     public static int Main(string[] astrArgs)
     {
          WebRequest webreq;
          WebResponse webres;
   
          try
          {
               webreq = WebRequest.Create("http://www.navioo.com/");
               webres = webreq.GetResponse();
          }
          catch (Exception exc)
          {
               Console.WriteLine("HtmlDump: {0}", exc.Message);
               return 1;
          }
   
          if (webres.ContentType.Substring(0, 4) != "text")
          {
               Console.WriteLine("HtmlDump: URI must be a text type.");
               return 1;
          }
   
          Stream       stream = webres.GetResponseStream();
          StreamReader strrdr = new StreamReader(stream);
          string       strLine;
   
          while ((strLine = strrdr.ReadLine()) != null){
               Console.WriteLine(strLine);
          }
          stream.Close();
          return 0;
     }
}