# robots.txt stops or allows certain robots acessing certain pages of your site. # Commands: # Allow: ((/) ALL) eg: "/example/", "example.html" # Disallow: ((/) ALL) eg: "/example/", "example.html" #User-Agent: (the robots this applies to) ((*) ALL) eg: googlebot User-Agent: * Allow: /