From bb474315ab62456dbd54fcff45f43160f901b4ca Mon Sep 17 00:00:00 2001
From: Ethan Yoo <hello@ethanyoo.com>
Date: Sat, 30 Nov 2024 11:53:47 -0500
Subject: [PATCH] Update robots.txt and set X-Robots-Tag header

---
 static/.htaccess  | 5 +++++
 static/robots.txt | 1 -
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/static/.htaccess b/static/.htaccess
index 0cca2eb..06de424 100644
--- a/static/.htaccess
+++ b/static/.htaccess
@@ -15,3 +15,8 @@ ErrorDocument 404 /404.html
 <FilesMatch "\.(css|ico|jpg|jpeg|png|svg)$">
 	Header set Cache-Control "max-age=2592000, public, immutable"
 </FilesMatch>
+
+# Prevent indexing of PDF files
+<FilesMatch "\.(pdf)$">
+    Header always set X-Robots-Tag "noindex, nofollow"
+</FilesMatch>
diff --git a/static/robots.txt b/static/robots.txt
index 54b2c34..eb803b1 100644
--- a/static/robots.txt
+++ b/static/robots.txt
@@ -1,7 +1,6 @@
 # Disallow crawling of specific directories and file types
 User-agent: *
 Disallow: /*.asc.txt$
-Disallow: /*.pdf$
 Disallow: /documents/
 Disallow: /img/