Also seems to be a problem with txt files and robots, googlebot is unable to read robots.txt meaning it ignores the rules set in there.
It shows this as the code downloaded, anyway of fixing this?
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<title>Download robots.txt</title>
<script type="text/javascript" language="JavaScript"><!--
function docountdown(){
s_div = document.getElementById("dldiv");
count = count - 1;
if (count > 0){
s_div.innerHTML = "Your request to download robots.txt is being processed.<"+"p>Please Wait... ("+count+" seconds remaining)<"+"/p>";
setTimeout("docountdown();", 1000);
} else {
s_div.innerHTML = "Your file is ready for download.<"+"p><"+"a href=\"/robots.txt?\">Click here to Download File<"+"/a><"+"/p>";
}
}
var count = 60;
window.onload = docountdown;
--></script>
</head>
<body>
<div align="center">
<script type="text/javascript"><!--
google_ad_client = "pub-2927823587298693";
google_alternate_color = "FFFFFF";
google_ad_width = 468;
google_ad_height = 60;
google_ad_format = "468x60_as";
google_ad_channel ="";
google_color_border = "336699";
google_color_bg = "FFFFFF";
google_color_link = "0000FF";
google_color_url = "008000";
google_color_text = "000000";
//--></script>
<script type="text/javascript"
src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
</script>
<br /><br />
<table border="0" width="500">
<tr>
<td>
<fieldset style="padding: 2">
<legend>Download robots.txt</legend>
<br />
<div id="dldiv">
Your request to download robots.txt is being processed.<p>Please Wait...</p>
</div>
<br />
</fieldset>
<p> </td>
</tr>
</table>
<script type="text/javascript"><!--
google_ad_client = "pub-2927823587298693";
google_alternate_color = "FFFFFF";
google_ad_width = 468;
google_ad_height = 60;
google_ad_format = "468x60_as";
google_ad_channel ="";
google_color_border = "336699";
google_color_bg = "FFFFFF";
google_color_link = "0000FF";
google_color_url = "008000";
google_color_text = "000000";
//--></script>
<script type="text/javascript"
src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
</script>
</div>
</body>
</html>