From 368fa19db8fa494eaf061134feaa1be1ced27061 Mon Sep 17 00:00:00 2001 From: Ming Di Leom <2809763-curben@users.noreply.gitlab.com> Date: Fri, 10 Feb 2023 20:24:03 +0000 Subject: [PATCH] feat: download mirrors --- README.md | 14 ++++++++++++++ bin/getbotnetfilter.py | 11 ++++++++--- bin/getphishingfilter.py | 11 ++++++++--- bin/getpupfilter.py | 11 +++++++++-- bin/geturlhausfilter.py | 11 ++++++++--- bin/getvnbadsitefilter.py | 11 ++++++++--- bin/utils.py | 23 ++++++++++++++++++++--- 7 files changed, 75 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 9743756..45e1131 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ - [getopendbl](#getopendbl) - [Disable individual commands](#disable-individual-commands) - [Build](#build) +- [Download failover](#download-failover) Provide custom search commands to update [malware-filter](https://gitlab.com/malware-filter) lookups. Each command downloads from a source CSV and emit rows as events which can then be piped to a lookup file or used as a subsearch. Each command is exported globally and can be used in any app. This add-on currently does not have any UI. @@ -162,6 +163,19 @@ cd splunk-malware-filter python build.py ``` +## Download failover + +For `get*filter` search commands, the script will attempt to download from the following domains in sequence (check out the `DOWNLOAD_URLS` constant in each script): + +- malware-filter.gitlab.io +- curbengh.github.io +- curbengh.github.io +- malware-filter.gitlab.io +- malware-filter.pages.dev +- \*-filter.pages.dev + +If your corporate proxy admin balks at having to allow >1 domains, allowing any of them will do. Since the script wouldn't know the proxy ruleset, it will still attempt those domains in sequence until it found a reachable one. + ## Disclaimer `getbotnetip.py` and `getopendbl.py` are included simply for convenience, their upstream sources are not affiliated with malware-filter. diff --git a/bin/getbotnetfilter.py b/bin/getbotnetfilter.py index 073c10f..8641912 100644 --- a/bin/getbotnetfilter.py +++ b/bin/getbotnetfilter.py @@ -12,8 +12,13 @@ from utils import Utility sys.path.insert(0, path.join(path.dirname(__file__), "..", "lib")) from splunklib.searchcommands import Configuration, GeneratingCommand, Option, dispatch -DOWNLOAD_URL = ( - "https://malware-filter.gitlab.io/malware-filter/botnet-filter-splunk.csv" +DOWNLOAD_URLS = ( + "https://malware-filter.gitlab.io/malware-filter/botnet-filter-splunk.csv", + "https://curbengh.github.io/malware-filter/botnet-filter-splunk.csv", + "https://curbengh.github.io/botnet-filter/botnet-filter-splunk.csv", + "https://malware-filter.gitlab.io/botnet-filter/botnet-filter-splunk.csv", + "https://malware-filter.pages.dev/botnet-filter-splunk.csv", + "https://botnet-filter.pages.dev/botnet-filter-splunk.csv", ) @@ -24,7 +29,7 @@ class GetBotnetFilter(Utility, GeneratingCommand): custom_message = Option(name="message") def generate(self): - dl_csv = self.download(DOWNLOAD_URL) + dl_csv = self.download(DOWNLOAD_URLS) for row in self.csv_reader(dl_csv): if isinstance(self.custom_message, str) and len(self.custom_message) >= 1: row["custom_message"] = self.custom_message diff --git a/bin/getphishingfilter.py b/bin/getphishingfilter.py index 97d3cfb..8ed66b0 100644 --- a/bin/getphishingfilter.py +++ b/bin/getphishingfilter.py @@ -12,8 +12,13 @@ from utils import Utility sys.path.insert(0, path.join(path.dirname(__file__), "..", "lib")) from splunklib.searchcommands import Configuration, GeneratingCommand, Option, dispatch -DOWNLOAD_URL = ( - "https://malware-filter.gitlab.io/malware-filter/phishing-filter-splunk.csv" +DOWNLOAD_URLS = ( + "https://malware-filter.gitlab.io/malware-filter/phishing-filter-splunk.csv", + "https://curbengh.github.io/malware-filter/phishing-filter-splunk.csv", + "https://curbengh.github.io/phishing-filter/phishing-filter-splunk.csv", + "https://malware-filter.gitlab.io/phishing-filter/phishing-filter-splunk.csv", + "https://malware-filter.pages.dev/phishing-filter-splunk.csv", + "https://phishing-filter.pages.dev/phishing-filter-splunk.csv", ) @@ -27,7 +32,7 @@ class GetPhishingFilter(Utility, GeneratingCommand): custom_message = Option(name="message") def generate(self): - dl_csv = self.download(DOWNLOAD_URL) + dl_csv = self.download(DOWNLOAD_URLS) for row in self.csv_reader(dl_csv): if isinstance(self.custom_message, str) and len(self.custom_message) >= 1: row["custom_message"] = self.custom_message diff --git a/bin/getpupfilter.py b/bin/getpupfilter.py index 90f7bab..07a526c 100644 --- a/bin/getpupfilter.py +++ b/bin/getpupfilter.py @@ -12,7 +12,14 @@ from utils import Utility sys.path.insert(0, path.join(path.dirname(__file__), "..", "lib")) from splunklib.searchcommands import Configuration, GeneratingCommand, Option, dispatch -DOWNLOAD_URL = "https://malware-filter.gitlab.io/malware-filter/pup-filter-splunk.csv" +DOWNLOAD_URLS = ( + "https://malware-filter.gitlab.io/malware-filter/pup-filter-splunk.csv", + "https://curbengh.github.io/malware-filter/pup-filter-splunk.csv", + "https://curbengh.github.io/pup-filter/pup-filter-splunk.csv", + "https://malware-filter.gitlab.io/pup-filter/pup-filter-splunk.csv", + "https://malware-filter.pages.dev/pup-filter-splunk.csv", + "https://pup-filter.pages.dev/pup-filter-splunk.csv", +) @Configuration() @@ -25,7 +32,7 @@ class GetPupFilter(Utility, GeneratingCommand): custom_message = Option(name="message") def generate(self): - dl_csv = self.download(DOWNLOAD_URL) + dl_csv = self.download(DOWNLOAD_URLS) for row in self.csv_reader(dl_csv): if isinstance(self.custom_message, str) and len(self.custom_message) >= 1: row["custom_message"] = self.custom_message diff --git a/bin/geturlhausfilter.py b/bin/geturlhausfilter.py index 1f48224..2487a9c 100644 --- a/bin/geturlhausfilter.py +++ b/bin/geturlhausfilter.py @@ -12,8 +12,13 @@ from utils import Utility sys.path.insert(0, path.join(path.dirname(__file__), "..", "lib")) from splunklib.searchcommands import Configuration, GeneratingCommand, Option, dispatch -DOWNLOAD_URL = ( - "https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-splunk-online.csv" +DOWNLOAD_URLS = ( + "https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-splunk-online.csv", + "https://curbengh.github.io/malware-filter/urlhaus-filter-splunk-online.csv", + "https://curbengh.github.io/urlhaus-filter/urlhaus-filter-splunk-online.csv", + "https://malware-filter.gitlab.io/urlhaus-filter/urlhaus-filter-splunk-online.csv", + "https://malware-filter.pages.dev/urlhaus-filter-splunk-online.csv", + "https://urlhaus-filter.pages.dev/urlhaus-filter-splunk-online.csv", ) @@ -27,7 +32,7 @@ class GetUrlhausFilter(Utility, GeneratingCommand): custom_message = Option(name="message") def generate(self): - dl_csv = self.download(DOWNLOAD_URL) + dl_csv = self.download(DOWNLOAD_URLS) for row in self.csv_reader(dl_csv): if isinstance(self.custom_message, str) and len(self.custom_message) >= 1: row["custom_message"] = self.custom_message diff --git a/bin/getvnbadsitefilter.py b/bin/getvnbadsitefilter.py index 4638817..5d335e2 100644 --- a/bin/getvnbadsitefilter.py +++ b/bin/getvnbadsitefilter.py @@ -12,8 +12,13 @@ from utils import Utility sys.path.insert(0, path.join(path.dirname(__file__), "..", "lib")) from splunklib.searchcommands import Configuration, GeneratingCommand, Option, dispatch -DOWNLOAD_URL = ( - "https://malware-filter.gitlab.io/malware-filter/vn-badsite-filter-splunk.csv" +DOWNLOAD_URLS = ( + "https://malware-filter.gitlab.io/malware-filter/vn-badsite-filter-splunk.csv", + "https://curbengh.github.io/malware-filter/vn-badsite-filter-splunk.csv", + "https://curbengh.github.io/vn-badsite-filter/vn-badsite-filter-splunk.csv", + "https://malware-filter.gitlab.io/vn-badsite-filter/vn-badsite-filter-splunk.csv", + "https://malware-filter.pages.dev/vn-badsite-filter-splunk.csv", + "https://vn-badsite-filter.pages.dev/vn-badsite-filter-splunk.csv", ) @@ -27,7 +32,7 @@ class GetVNBadsiteFilter(Utility, GeneratingCommand): custom_message = Option(name="message") def generate(self): - dl_csv = self.download(DOWNLOAD_URL) + dl_csv = self.download(DOWNLOAD_URLS) for row in self.csv_reader(dl_csv): if isinstance(self.custom_message, str) and len(self.custom_message) >= 1: row["custom_message"] = self.custom_message diff --git a/bin/utils.py b/bin/utils.py index d7c931a..f5cc00e 100644 --- a/bin/utils.py +++ b/bin/utils.py @@ -58,13 +58,30 @@ class Utility: return {} - def download(self, url): - """Send a GET request to the URL and return content of the response.""" + # pylint: disable=inconsistent-return-statements + def download(self, urls, index=0): + """ + Send a GET request to the URL and return content of the response. + + Arguments: + urls {list/tuple/string} -- A list of URLs to try in sequence + index -- List's index to start + """ + if isinstance(urls, str): + urls = (urls,) + + url = urls[index] proxy_config = self.__get_proxy(url) try: res = requests.get(url, timeout=5, **proxy_config) + # pylint: disable=no-member + if res.status_code == requests.codes.ok: + return res.text + + if index < len(urls) - 1: + return self.download(urls, index + 1) + res.raise_for_status() - return res.text except requests.exceptions.HTTPError as errh: raise errh except requests.exceptions.ConnectionError as errc: