modify robots.txt

2018-12-21 14:25:54 -05:00 · 2018-12-21 14:25:54 -05:00 · 5da6ca020a
parent 0235f7b750
commit 5da6ca020a
1 changed files with 4 additions and 1 deletions
--- a/robots.txt
+++ b/robots.txt
@ -4,6 +4,9 @@ sitemap:
  exclude: 'yes'
 ---
 User-agent: *
-Disallow:
+Disallow: /search
+
+User-agent: ia_archiver
+Disallow: /

 Sitemap: {{ site.url }}/sitemap.xml