add gptbot to robots.txt

2023-08-08 12:30:08 -06:00 · 2023-08-08 12:30:08 -06:00 · 924775a055
parent 1e665764df
commit 924775a055
1 changed files with 3 additions and 3 deletions
--- a/robots.txt
+++ b/robots.txt
@ -3,10 +3,10 @@ layout: null
 sitemap:
  exclude: 'yes'
 ---
-User-agent: *
-Disallow: /search
-
 User-agent: ia_archiver
 Disallow: /

+User-agent: GPTBot
+Disallow: /
+
 Sitemap: {{ site.url }}/sitemap.xml