diff --git a/public/robots.txt b/public/robots.txt --- a/public/robots.txt +++ b/public/robots.txt @@ -1,1 +1,5 @@ -# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file \ No newline at end of file +# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file +# +# To ban all spiders from the entire site uncomment the next two lines: +# User-Agent: * +# Disallow: /