From 45783a46b3d9bda74c05c81e024586c73a75d760 Mon Sep 17 00:00:00 2001
From: MDLeom <2809763-curben@users.noreply.gitlab.com>
Date: Mon, 17 Mar 2025 11:51:53 +0000
Subject: [PATCH] perf: rewrite IDS rule creation in javascript

"while do" can be inefficient
previously took >5 minutes is now less than 1 second
---
 src/ids.js    | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++
 src/script.sh | 49 ++-----------------------------------------------
 2 files changed, 53 insertions(+), 47 deletions(-)
 create mode 100644 src/ids.js

diff --git a/src/ids.js b/src/ids.js
new file mode 100644
index 00000000..b95b926a
--- /dev/null
+++ b/src/ids.js
@@ -0,0 +1,51 @@
+import { createWriteStream } from 'node:fs'
+import { open } from 'node:fs/promises'
+
+const domains = await open('phishing-notop-domains.txt')
+const urls = await open('phishing-url-top-domains-raw.txt')
+
+const snort2 = createWriteStream('../public/phishing-filter-snort2.rules', {
+  encoding: 'utf8',
+  flags: 'a'
+})
+const snort3 = createWriteStream('../public/phishing-filter-snort3.rules', {
+  encoding: 'utf8',
+  flags: 'a'
+})
+const suricata = createWriteStream('../public/phishing-filter-suricata.rules', {
+  encoding: 'utf8',
+  flags: 'a'
+})
+const splunk = createWriteStream('../public/phishing-filter-splunk.csv', {
+  encoding: 'utf8',
+  flags: 'a'
+})
+
+let sid = 200000001
+
+for await (const domain of domains.readLines()) {
+  snort2.write(`alert tcp $HOME_NET any -> $EXTERNAL_NET [80,443] (msg:"phishing-filter phishing website detected"; flow:established,from_client; content:"GET"; http_method; content:"${domain}"; content:"Host"; http_header; classtype:attempted-recon; sid:${sid}; rev:1;)\n`)
+  snort3.write(`alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"phishing-filter phishing website detected"; http_header:field host; content:"${domain}",nocase; classtype:attempted-recon; sid:${sid}; rev:1;)\n`)
+  suricata.write(`alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"phishing-filter phishing website detected"; flow:established,from_client; http.method; content:"GET"; http.host; content:"${domain}"; classtype:attempted-recon; sid:${sid} rev:1;)\n`)
+  splunk.write(`"${domain}","","phishing-filter phishing website detected","${process.env.CURRENT_TIME}"\n`)
+
+  sid++
+}
+
+for await (const line of urls.readLines()) {
+  const url = new URL(`http://${line}`)
+  const { hostname } = url
+  let pathname = url.pathname.replace(';', '\\;')
+  snort2.write(`alert tcp $HOME_NET any -> $EXTERNAL_NET [80,443] (msg:"phishing-filter phishing website detected"; flow:established,from_client; content:"GET"; http_method; content:"${pathname.substring(0, 2048)}"; http_uri; nocase; content:"${hostname}"; content:"Host"; http_header; classtype:attempted-recon; sid:$SID; rev:1;)\n`)
+  snort3.write(`alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"phishing-filter phishing website detected"; http_header:field host; content:"${hostname}",nocase; http_uri; content:"${pathname}",nocase; classtype:attempted-recon; sid:$SID; rev:1;)\n`)
+  suricata.write(`alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"phishing-filter phishing website detected"; flow:established,from_client; http.method; content:"GET"; http.uri; content:"${pathname}"; endswith; nocase; http.host; content:"${hostname}"; classtype:attempted-recon; sid:$SID; rev:1;)\n`)
+  pathname = url.pathname
+  splunk.write(`"${hostname}","${pathname}","phishing-filter phishing website detected","${process.env.CURRENT_TIME}"\n`)
+
+  sid++
+}
+
+snort2.close()
+snort3.close()
+suricata.close()
+splunk.close()
diff --git a/src/script.sh b/src/script.sh
index 8a5cca6c..e5a8c0c2 100644
--- a/src/script.sh
+++ b/src/script.sh
@@ -414,59 +414,14 @@ sed "1i $COMMENT" | \
 sed "1s/Domains/Wildcard Asterisk/" > "../public/phishing-filter-wildcard.txt"
 
 
-## Temporarily disable command print
-set +x
-
 ## Snort & Suricata rulesets
 rm "../public/phishing-filter-snort2.rules" \
   "../public/phishing-filter-snort3.rules" \
   "../public/phishing-filter-suricata.rules" \
   "../public/phishing-filter-splunk.csv"
 
-SID="200000001"
-while read DOMAIN; do
-  SN_RULE="alert tcp \$HOME_NET any -> \$EXTERNAL_NET [80,443] (msg:\"phishing-filter phishing website detected\"; flow:established,from_client; content:\"GET\"; http_method; content:\"$DOMAIN\"; content:\"Host\"; http_header; classtype:attempted-recon; sid:$SID; rev:1;)"
-
-  SN3_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"phishing-filter phishing website detected\"; http_header:field host; content:\"$DOMAIN\",nocase; classtype:attempted-recon; sid:$SID; rev:1;)"
-
-  SR_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"phishing-filter phishing website detected\"; flow:established,from_client; http.method; content:\"GET\"; http.host; content:\"$DOMAIN\"; classtype:attempted-recon; sid:$SID; rev:1;)"
-
-  SP_RULE="\"$DOMAIN\",\"\",\"phishing-filter phishing website detected\",\"$CURRENT_TIME\""
-
-  echo "$SN_RULE" >> "../public/phishing-filter-snort2.rules"
-  echo "$SN3_RULE" >> "../public/phishing-filter-snort3.rules"
-  echo "$SR_RULE" >> "../public/phishing-filter-suricata.rules"
-  echo "$SP_RULE" >> "../public/phishing-filter-splunk.csv"
-
-  SID=$(( $SID + 1 ))
-done < "phishing-notop-domains.txt"
-
-while read URL; do
-  DOMAIN=$(echo "$URL" | cut -d"/" -f1)
-  # escape ";"
-  PATHNAME=$(echo "$URL" | sed -e "s/^$DOMAIN//" -e "s/;/\\\;/g")
-
-  # Snort2 only supports <=2047 characters of `content`
-  SN_RULE="alert tcp \$HOME_NET any -> \$EXTERNAL_NET [80,443] (msg:\"phishing-filter phishing website detected\"; flow:established,from_client; content:\"GET\"; http_method; content:\"$(echo $PATHNAME | cut -c -2047)\"; http_uri; nocase; content:\"$DOMAIN\"; content:\"Host\"; http_header; classtype:attempted-recon; sid:$SID; rev:1;)"
-
-  SN3_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"phishing-filter phishing website detected\"; http_header:field host; content:\"$DOMAIN\",nocase; http_uri; content:\"$PATHNAME\",nocase; classtype:attempted-recon; sid:$SID; rev:1;)"
-
-  SR_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"phishing-filter phishing website detected\"; flow:established,from_client; http.method; content:\"GET\"; http.uri; content:\"$PATHNAME\"; endswith; nocase; http.host; content:\"$DOMAIN\"; classtype:attempted-recon; sid:$SID; rev:1;)"
-
-  PATHNAME=$(echo "$URL" | sed "s/^$DOMAIN//")
-
-  SP_RULE="\"$DOMAIN\",\"$PATHNAME\",\"phishing-filter phishing website detected\",\"$CURRENT_TIME\""
-
-  echo "$SN_RULE" >> "../public/phishing-filter-snort2.rules"
-  echo "$SN3_RULE" >> "../public/phishing-filter-snort3.rules"
-  echo "$SR_RULE" >> "../public/phishing-filter-suricata.rules"
-  echo "$SP_RULE" >> "../public/phishing-filter-splunk.csv"
-
-  SID=$(( $SID + 1 ))
-done < "phishing-url-top-domains-raw.txt"
-
-## Re-enable command print
-set -x
+export CURRENT_TIME
+node "../src/ids.js"
 
 sed -i "1i $COMMENT" "../public/phishing-filter-snort2.rules"
 sed -i "1s/Domains Blocklist/URL Snort2 Ruleset/" "../public/phishing-filter-snort2.rules"