curben
/
urlhaus-filter
mirror of https://gitlab.com/malware-filter/urlhaus-filter
1
0
Fork 0
Code Issues Releases Wiki Activity

perf: rewrite IDS rule creation in javascript 

"while do" can be inefficient
previously took >20s is now 0.2s
This commit is contained in:
MDLeom 2025-03-17 12:15:37 +00:00
parent c28f23bc2a
commit d2f18b753c
No known key found for this signature in database
GPG Key ID: 32D3E28E96A695E8
2 changed files with 53 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
src/ids.js Normal file
View File

@ -0,0 +1,51 @@
import { createWriteStream } from 'node:fs'
import { open } from 'node:fs/promises'
const domains = await open('malware-domains-online.txt')
const urls = await open('malware-url-top-domains-raw-online.txt')
const snort2 = createWriteStream('../public/urlhaus-filter-snort2-online.rules', {
encoding: 'utf8',
flags: 'a'
})
const snort3 = createWriteStream('../public/urlhaus-filter-snort3-online.rules', {
encoding: 'utf8',
flags: 'a'
})
const suricata = createWriteStream('../public/urlhaus-filter-suricata-online.rules', {
encoding: 'utf8',
flags: 'a'
})
const splunk = createWriteStream('../public/urlhaus-filter-splunk-online.csv', {
encoding: 'utf8',
flags: 'a'
})
let sid = 100000001
for await (const domain of domains.readLines()) {
snort2.write(`alert tcp $HOME_NET any -> $EXTERNAL_NET [80,443] (msg:"urlhaus-filter malicious website detected"; flow:established,from_client; content:"GET"; http_method; content:"${domain}"; content:"Host"; http_header; classtype:trojan-activity; sid:${sid}; rev:1;)\n`)
snort3.write(`alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"urlhaus-filter malicious website detected"; http_header:field host; content:"${domain}",nocase; classtype:trojan-activity; sid:${sid}; rev:1;)\n`)
suricata.write(`alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"urlhaus-filter malicious website detected"; flow:established,from_client; http.method; content:"GET"; http.host; content:"${domain}"; classtype:trojan-activity; sid:${sid} rev:1;)\n`)
splunk.write(`"${domain}","","urlhaus-filter malicious website detected","${process.env.CURRENT_TIME}"\n`)
sid++
}
for await (const line of urls.readLines()) {
const url = new URL(`http://${line}`)
const { hostname } = url
let pathname = url.pathname.replace(';', '\\;')
snort2.write(`alert tcp $HOME_NET any -> $EXTERNAL_NET [80,443] (msg:"urlhaus-filter malicious website detected"; flow:established,from_client; content:"GET"; http_method; content:"${pathname.substring(0, 2048)}"; http_uri; nocase; content:"${hostname}"; content:"Host"; http_header; classtype:trojan-activity; sid:${sid}; rev:1;)\n`)
snort3.write(`alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"urlhaus-filter malicious website detected"; http_header:field host; content:"${hostname}",nocase; http_uri; content:"${pathname}",nocase; classtype:trojan-activity; sid:${sid}; rev:1;)\n`)
suricata.write(`alert http $HOME_NET any -> $EXTERNAL_NET any (msg:"urlhaus-filter malicious website detected"; flow:established,from_client; http.method; content:"GET"; http.uri; content:"${pathname}"; endswith; nocase; http.host; content:"${hostname}"; classtype:trojan-activity; sid:${sid}; rev:1;)\n`)
pathname = url.pathname
splunk.write(`"${hostname}","${pathname}","urlhaus-filter malicious website detected","${process.env.CURRENT_TIME}"\n`)
sid++
}
snort2.close()
snort3.close()
suricata.close()
splunk.close()

50
src/script.sh
View File

@ -441,60 +441,14 @@ sed "1i $COMMENT" | \
sed "1s/Blocklist/Wildcard Asterisk Blocklist/" > "../public/urlhaus-filter-wildcard-online.txt"
## Temporarily disable command print
set +x
# Snort, Suricata, Splunk
rm "../public/urlhaus-filter-snort2-online.rules" \
"../public/urlhaus-filter-snort3-online.rules" \
"../public/urlhaus-filter-suricata-online.rules" \
"../public/urlhaus-filter-splunk-online.csv"
SID="100000001"
while read DOMAIN; do
SN_RULE="alert tcp \$HOME_NET any -> \$EXTERNAL_NET [80,443] (msg:\"urlhaus-filter malicious website detected\"; flow:established,from_client; content:\"GET\"; http_method; content:\"$DOMAIN\"; content:\"Host\"; http_header; classtype:trojan-activity; sid:$SID; rev:1;)"
SN3_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"urlhaus-filter malicious website detected\"; http_header:field host; content:\"$DOMAIN\",nocase; classtype:trojan-activity; sid:$SID; rev:1;)"
SR_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"urlhaus-filter malicious website detected\"; flow:established,from_client; http.method; content:\"GET\"; http.host; content:\"$DOMAIN\"; classtype:trojan-activity; sid:$SID; rev:1;)"
SP_RULE="\"$DOMAIN\",\"\",\"urlhaus-filter malicious website detected\",\"$CURRENT_TIME\""
echo "$SN_RULE" >> "../public/urlhaus-filter-snort2-online.rules"
echo "$SN3_RULE" >> "../public/urlhaus-filter-snort3-online.rules"
echo "$SR_RULE" >> "../public/urlhaus-filter-suricata-online.rules"
echo "$SP_RULE" >> "../public/urlhaus-filter-splunk-online.csv"
SID=$(( $SID + 1 ))
done < "malware-domains-online.txt"
while read URL; do
DOMAIN=$(echo "$URL" | cut -d"/" -f1)
# escape ";"
PATHNAME=$(echo "$URL" | sed -e "s/^$DOMAIN//" -e "s/;/\\\;/g")
# Snort2 only supports <=2047 characters of `content`
SN_RULE="alert tcp \$HOME_NET any -> \$EXTERNAL_NET [80,443] (msg:\"urlhaus-filter malicious website detected\"; flow:established,from_client; content:\"GET\"; http_method; content:\"$(echo $PATHNAME | cut -c -2047)\"; http_uri; nocase; content:\"$DOMAIN\"; content:\"Host\"; http_header; classtype:trojan-activity; sid:$SID; rev:1;)"
SN3_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"urlhaus-filter malicious website detected\"; http_header:field host; content:\"$DOMAIN\",nocase; http_uri; content:\"$PATHNAME\",nocase; classtype:trojan-activity; sid:$SID; rev:1;)"
SR_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"urlhaus-filter malicious website detected\"; flow:established,from_client; http.method; content:\"GET\"; http.uri; content:\"$PATHNAME\"; endswith; nocase; http.host; content:\"$DOMAIN\"; classtype:trojan-activity; sid:$SID; rev:1;)"
PATHNAME=$(echo "$URL" | sed "s/^$DOMAIN//")
SP_RULE="\"$DOMAIN\",\"$PATHNAME\",\"urlhaus-filter malicious website detected\",\"$CURRENT_TIME\""
echo "$SN_RULE" >> "../public/urlhaus-filter-snort2-online.rules"
echo "$SN3_RULE" >> "../public/urlhaus-filter-snort3-online.rules"
echo "$SR_RULE" >> "../public/urlhaus-filter-suricata-online.rules"
echo "$SP_RULE" >> "../public/urlhaus-filter-splunk-online.csv"
SID=$(( $SID + 1 ))
done < "malware-url-top-domains-raw-online.txt"
## Re-enable command print
set -x
export CURRENT_TIME
node "../src/ids.js"
sed -i "1i $COMMENT_ONLINE" "../public/urlhaus-filter-snort2-online.rules"
sed -i "1s/Domains Blocklist/URL Snort2 Ruleset/" "../public/urlhaus-filter-snort2-online.rules"