modify robots.txt

3 years ago · 5da6ca020a
1 changed files with 4 additions and 1 deletions
--- a/robots.txt
+++ b/robots.txt
@ -4,6 +4,9 @@ sitemap:
  exclude: 'yes'
 ---
 User-agent: *
-Disallow:
+Disallow: /search
+
+User-agent: ia_archiver
+Disallow: /

 Sitemap: {{ site.url }}/sitemap.xml