Update robots.txt and set X-Robots-Tag header

static/.htaccess

@@ -15,3 +15,8 @@ ErrorDocument 404 /404.html
 <FilesMatch "\.(css|ico|jpg|jpeg|png|svg)$">
 	Header set Cache-Control "max-age=2592000, public, immutable"
 </FilesMatch>
+
+# Prevent indexing of PDF files
+<FilesMatch "\.(pdf)$">
+    Header always set X-Robots-Tag "noindex, nofollow"
+</FilesMatch>

static/robots.txt

@@ -1,7 +1,6 @@
 # Disallow crawling of specific directories and file types
 User-agent: *
 Disallow: /*.asc.txt$
-Disallow: /*.pdf$
 Disallow: /documents/
 Disallow: /img/