From d2f18b753cb84b3ce5486f00965fa7e52be52200 Mon Sep 17 00:00:00 2001
From: MDLeom <2809763-curben@users.noreply.gitlab.com>
Date: Mon, 17 Mar 2025 12:15:37 +0000
Subject: [PATCH] perf: rewrite IDS rule creation in javascript

"while do" can be inefficient
previously took >20s is now 0.2s
---
 src/ids.js    | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++
 src/script.sh | 50 ++------------------------------------------------
 2 files changed, 53 insertions(+), 48 deletions(-)
 create mode 100644 src/ids.js

diff --git a/src/ids.js b/src/ids.js
new file mode 100644
index 00000000..c13ab9bc
--- /dev/null
+++ b/src/ids.js
@@ -0,0 +1,51 @@
+import { createWriteStream } from 'node:fs'
+import { open } from 'node:fs/promises'
+
+const domains = await open('malware-domains-online.txt')
+const urls = await open('malware-url-top-domains-raw-online.txt')
+
+const snort2 = createWriteStream('../public/urlhaus-filter-snort2-online.rules', {
+  encoding: 'utf8',
+  flags: 'a'
+})
+const snort3 = createWriteStream('../public/urlhaus-filter-snort3-online.rules', {
+  encoding: 'utf8',
+  flags: 'a'
+})
+const suricata = createWriteStream('../public/urlhaus-filter-suricata-online.rules', {
+  encoding: 'utf8',
+  flags: 'a'
+})
+const splunk = createWriteStream('../public/urlhaus-filter-splunk-online.csv', {
+  encoding: 'utf8',
+  flags: 'a'
+})
+
+let sid = 100000001
+
+for await (const domain of domains.readLines()) {
+  snort2.write(`alert tcp $HOME_NET any -> $EXTERNAL_NET [80,443] (msg:"urlhaus-filter malicious website detected"; flow:established,from_client; content:"GET"; http_method; content:"${domain}"; content:"Host"; http_header; classtype:trojan-activity; sid:${sid}; rev:1;)\n`)
+  snort3.write(`alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"urlhaus-filter malicious website detected"; http_header:field host; content:"${domain}",nocase; classtype:trojan-activity; sid:${sid}; rev:1;)\n`)
+  suricata.write(`alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"urlhaus-filter malicious website detected"; flow:established,from_client; http.method; content:"GET"; http.host; content:"${domain}"; classtype:trojan-activity; sid:${sid} rev:1;)\n`)
+  splunk.write(`"${domain}","","urlhaus-filter malicious website detected","${process.env.CURRENT_TIME}"\n`)
+
+  sid++
+}
+
+for await (const line of urls.readLines()) {
+  const url = new URL(`http://${line}`)
+  const { hostname } = url
+  let pathname = url.pathname.replace(';', '\\;')
+  snort2.write(`alert tcp $HOME_NET any -> $EXTERNAL_NET [80,443] (msg:"urlhaus-filter malicious website detected"; flow:established,from_client; content:"GET"; http_method; content:"${pathname.substring(0, 2048)}"; http_uri; nocase; content:"${hostname}"; content:"Host"; http_header; classtype:trojan-activity; sid:${sid}; rev:1;)\n`)
+  snort3.write(`alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"urlhaus-filter malicious website detected"; http_header:field host; content:"${hostname}",nocase; http_uri; content:"${pathname}",nocase; classtype:trojan-activity; sid:${sid}; rev:1;)\n`)
+  suricata.write(`alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"urlhaus-filter malicious website detected"; flow:established,from_client; http.method; content:"GET"; http.uri; content:"${pathname}"; endswith; nocase; http.host; content:"${hostname}"; classtype:trojan-activity; sid:${sid}; rev:1;)\n`)
+  pathname = url.pathname
+  splunk.write(`"${hostname}","${pathname}","urlhaus-filter malicious website detected","${process.env.CURRENT_TIME}"\n`)
+
+  sid++
+}
+
+snort2.close()
+snort3.close()
+suricata.close()
+splunk.close()
diff --git a/src/script.sh b/src/script.sh
index b7d718c2..d93a6964 100644
--- a/src/script.sh
+++ b/src/script.sh
@@ -441,60 +441,14 @@ sed "1i $COMMENT" | \
 sed "1s/Blocklist/Wildcard Asterisk Blocklist/" > "../public/urlhaus-filter-wildcard-online.txt"
 
 
-## Temporarily disable command print
-set +x
-
-
 # Snort, Suricata, Splunk
 rm "../public/urlhaus-filter-snort2-online.rules" \
   "../public/urlhaus-filter-snort3-online.rules" \
   "../public/urlhaus-filter-suricata-online.rules" \
   "../public/urlhaus-filter-splunk-online.csv"
 
-SID="100000001"
-while read DOMAIN; do
-  SN_RULE="alert tcp \$HOME_NET any -> \$EXTERNAL_NET [80,443] (msg:\"urlhaus-filter malicious website detected\"; flow:established,from_client; content:\"GET\"; http_method; content:\"$DOMAIN\"; content:\"Host\"; http_header; classtype:trojan-activity; sid:$SID; rev:1;)"
-
-  SN3_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"urlhaus-filter malicious website detected\"; http_header:field host; content:\"$DOMAIN\",nocase; classtype:trojan-activity; sid:$SID; rev:1;)"
-
-  SR_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"urlhaus-filter malicious website detected\"; flow:established,from_client; http.method; content:\"GET\"; http.host; content:\"$DOMAIN\"; classtype:trojan-activity; sid:$SID; rev:1;)"
-
-  SP_RULE="\"$DOMAIN\",\"\",\"urlhaus-filter malicious website detected\",\"$CURRENT_TIME\""
-
-  echo "$SN_RULE" >> "../public/urlhaus-filter-snort2-online.rules"
-  echo "$SN3_RULE" >> "../public/urlhaus-filter-snort3-online.rules"
-  echo "$SR_RULE" >> "../public/urlhaus-filter-suricata-online.rules"
-  echo "$SP_RULE" >> "../public/urlhaus-filter-splunk-online.csv"
-
-  SID=$(( $SID + 1 ))
-done < "malware-domains-online.txt"
-
-while read URL; do
-  DOMAIN=$(echo "$URL" | cut -d"/" -f1)
-  # escape ";"
-  PATHNAME=$(echo "$URL" | sed -e "s/^$DOMAIN//" -e "s/;/\\\;/g")
-
-  # Snort2 only supports <=2047 characters of `content`
-  SN_RULE="alert tcp \$HOME_NET any -> \$EXTERNAL_NET [80,443] (msg:\"urlhaus-filter malicious website detected\"; flow:established,from_client; content:\"GET\"; http_method; content:\"$(echo $PATHNAME | cut -c -2047)\"; http_uri; nocase; content:\"$DOMAIN\"; content:\"Host\"; http_header; classtype:trojan-activity; sid:$SID; rev:1;)"
-
-  SN3_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"urlhaus-filter malicious website detected\"; http_header:field host; content:\"$DOMAIN\",nocase; http_uri; content:\"$PATHNAME\",nocase; classtype:trojan-activity; sid:$SID; rev:1;)"
-
-  SR_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"urlhaus-filter malicious website detected\"; flow:established,from_client; http.method; content:\"GET\"; http.uri; content:\"$PATHNAME\"; endswith; nocase; http.host; content:\"$DOMAIN\"; classtype:trojan-activity; sid:$SID; rev:1;)"
-
-  PATHNAME=$(echo "$URL" | sed "s/^$DOMAIN//")
-
-  SP_RULE="\"$DOMAIN\",\"$PATHNAME\",\"urlhaus-filter malicious website detected\",\"$CURRENT_TIME\""
-
-  echo "$SN_RULE" >> "../public/urlhaus-filter-snort2-online.rules"
-  echo "$SN3_RULE" >> "../public/urlhaus-filter-snort3-online.rules"
-  echo "$SR_RULE" >> "../public/urlhaus-filter-suricata-online.rules"
-  echo "$SP_RULE" >> "../public/urlhaus-filter-splunk-online.csv"
-
-  SID=$(( $SID + 1 ))
-done < "malware-url-top-domains-raw-online.txt"
-
-## Re-enable command print
-set -x
+export CURRENT_TIME
+node "../src/ids.js"
 
 sed -i "1i $COMMENT_ONLINE" "../public/urlhaus-filter-snort2-online.rules"
 sed -i "1s/Domains Blocklist/URL Snort2 Ruleset/" "../public/urlhaus-filter-snort2-online.rules"