Initial commit
This commit is contained in:
Ming Di Leom
commit
22d1b7e8f5
|
|
@ -0,0 +1,6 @@
|
|||
__pycache__
|
||||
.venv*/
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
lib/
|
||||
malware_filter-*.tar.gz
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
image: python:slim
|
||||
|
||||
variables:
|
||||
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
|
||||
|
||||
lint:
|
||||
stage: test
|
||||
|
||||
cache:
|
||||
paths:
|
||||
- .cache/pip
|
||||
- .venv/
|
||||
|
||||
before_script:
|
||||
- python --version
|
||||
- python -m venv .venv
|
||||
- source .venv/bin/activate
|
||||
- pip install -r requirements-dev.txt -U
|
||||
|
||||
script:
|
||||
- pylint $(find -type f -name "*.py" ! -path "./.venv/**" ! -path "./lib/**")
|
||||
|
||||
build:
|
||||
stage: build
|
||||
|
||||
script:
|
||||
- python --version
|
||||
- python build.py
|
||||
|
||||
artifacts:
|
||||
paths:
|
||||
- malware_filter-*.tar.gz
|
||||
expire_in: 30 days
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
default_install_hook_types: [pre-push]
|
||||
repos:
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: standard-python-shebang
|
||||
name: Standard python shebang
|
||||
entry: sed
|
||||
language: system
|
||||
types: [python]
|
||||
args: [
|
||||
"-i", # modify in-place
|
||||
"-E", # extended regex
|
||||
"s|^#\\!.*|#\\!/usr/bin/env python|",
|
||||
]
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: "v4.4.0"
|
||||
hooks:
|
||||
- id: end-of-file-fixer
|
||||
- id: trailing-whitespace
|
||||
- repo: https://github.com/PyCQA/isort
|
||||
rev: "5.11.4"
|
||||
hooks:
|
||||
- id: isort
|
||||
types: [python]
|
||||
args: [
|
||||
".", # sort all Python files recursively
|
||||
]
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 22.12.0
|
||||
hooks:
|
||||
- id: black
|
||||
- repo: https://github.com/PyCQA/pylint
|
||||
rev: "v2.15.10"
|
||||
hooks:
|
||||
- id: pylint
|
||||
language: system
|
||||
types: [python]
|
||||
args: [
|
||||
"-rn", # Only display messages
|
||||
"-sn", # Don't display the score
|
||||
]
|
||||
- repo: https://github.com/pre-commit/mirrors-prettier
|
||||
rev: "v2.7.1"
|
||||
hooks:
|
||||
- id: prettier
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
{
|
||||
"files.eol": "\n",
|
||||
"files.trimTrailingWhitespace": true,
|
||||
"files.insertFinalNewline": true,
|
||||
"files.trimFinalNewlines": true,
|
||||
"python.formatting.provider": "none",
|
||||
"[python]": {
|
||||
"editor.defaultFormatter": "ms-python.black-formatter",
|
||||
"editor.formatOnSave": true,
|
||||
"editor.tabSize": 4,
|
||||
"editor.codeActionsOnSave": {
|
||||
"source.organizeImports": true
|
||||
}
|
||||
},
|
||||
"isort.args": ["--profile", "black"],
|
||||
"[javascript]": {
|
||||
"editor.formatOnSave": true,
|
||||
"editor.defaultFormatter": "esbenp.prettier-vscode"
|
||||
},
|
||||
"[markdown]": {
|
||||
"editor.formatOnSave": true,
|
||||
"editor.defaultFormatter": "esbenp.prettier-vscode"
|
||||
},
|
||||
"[yaml]": {
|
||||
"editor.formatOnSave": true,
|
||||
"editor.defaultFormatter": "esbenp.prettier-vscode"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
# CC0 1.0 Universal
|
||||
|
||||
## Statement of Purpose
|
||||
|
||||
The laws of most jurisdictions throughout the world automatically confer exclusive Copyright and Related Rights (defined below) upon the creator and subsequent owner(s) (each and all, an "owner") of an original work of authorship and/or a database (each, a "Work").
|
||||
|
||||
Certain owners wish to permanently relinquish those rights to a Work for the purpose of contributing to a commons of creative, cultural and scientific works ("Commons") that the public can reliably and without fear of later claims of infringement build upon, modify, incorporate in other works, reuse and redistribute as freely as possible in any form whatsoever and for any purposes, including without limitation commercial purposes. These owners may contribute to the Commons to promote the ideal of a free culture and the further production of creative, cultural and scientific works, or to gain reputation or greater distribution for their Work in part through the use and efforts of others.
|
||||
|
||||
For these and/or other purposes and motivations, and without any expectation of additional consideration or compensation, the person associating CC0 with a Work (the "Affirmer"), to the extent that he or she is an owner of Copyright and Related Rights in the Work, voluntarily elects to apply CC0 to the Work and publicly distribute the Work under its terms, with knowledge of his or her Copyright and Related Rights in the Work and the meaning and intended legal effect of CC0 on those rights.
|
||||
|
||||
1. Copyright and Related Rights.
|
||||
|
||||
---
|
||||
|
||||
A Work made available under CC0 may be protected by copyright and related or neighboring rights ("Copyright and Related Rights"). Copyright and Related Rights include, but are not limited to, the following:
|
||||
|
||||
i. the right to reproduce, adapt, distribute, perform, display, communicate, and translate a Work;
|
||||
ii. moral rights retained by the original author(s) and/or performer(s);
|
||||
iii. publicity and privacy rights pertaining to a person's image or likeness depicted in a Work;
|
||||
iv. rights protecting against unfair competition in regards to a Work, subject to the limitations in paragraph 4(a), below;
|
||||
v. rights protecting the extraction, dissemination, use and reuse of data in a Work;
|
||||
vi. database rights (such as those arising under Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, and under any national implementation thereof, including any amended or successor version of such directive); and
|
||||
vii. other similar, equivalent or corresponding rights throughout the world based on applicable law or treaty, and any national implementations thereof.
|
||||
|
||||
2. Waiver.
|
||||
|
||||
---
|
||||
|
||||
To the greatest extent permitted by, but not in contravention of, applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and unconditionally waives, abandons, and surrenders all of Affirmer's Copyright and Related Rights and associated claims and causes of action, whether now known or unknown (including existing as well as future claims and causes of action), in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each member of the public at large and to the detriment of Affirmer's heirs and successors, fully intending that such Waiver shall not be subject to revocation, rescission, cancellation, termination, or any other legal or equitable action to disrupt the quiet enjoyment of the Work by the public as contemplated by Affirmer's express Statement of Purpose.
|
||||
|
||||
3. Public License Fallback.
|
||||
|
||||
---
|
||||
|
||||
Should any part of the Waiver for any reason be judged legally invalid or ineffective under applicable law, then the Waiver shall be preserved to the maximum extent permitted taking into account Affirmer's express Statement of Purpose. In addition, to the extent the Waiver is so judged Affirmer hereby grants to each affected person a royalty-free, non transferable, non sublicensable, non exclusive, irrevocable and unconditional license to exercise Affirmer's Copyright and Related Rights in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "License"). The License shall be deemed effective as of the date CC0 was applied by Affirmer to the Work. Should any part of the License for any reason be judged legally invalid or ineffective under applicable law, such partial invalidity or ineffectiveness shall not invalidate the remainder of the License, and in such case Affirmer hereby affirms that he or she will not (i) exercise any of his or her remaining Copyright and Related Rights in the Work or (ii) assert any associated claims and causes of action with respect to the Work, in either case contrary to Affirmer's express Statement of Purpose.
|
||||
|
||||
4. Limitations and Disclaimers.
|
||||
|
||||
---
|
||||
|
||||
a. No trademark or patent rights held by Affirmer are waived, abandoned, surrendered, licensed or otherwise affected by this document.
|
||||
b. Affirmer offers the Work as-is and makes no representations or warranties of any kind concerning the Work, express, implied, statutory or otherwise, including without limitation warranties of title, merchantability, fitness for a particular purpose, non infringement, or the absence of latent or other defects, accuracy, or the present or absence of errors, whether or not discoverable, all to the greatest extent permissible under applicable law.
|
||||
c. Affirmer disclaims responsibility for clearing rights of other persons that may apply to the Work or any use thereof, including without limitation any person's Copyright and Related Rights in the Work. Further, Affirmer disclaims responsibility for obtaining any necessary consents, permissions or other rights required for any use of the Work.
|
||||
d. Affirmer understands and acknowledges that Creative Commons is not a party to this document and has no duty or obligation with respect to this CC0 or use of the Work.
|
||||
|
||||
For more information, please see
|
||||
https://creativecommons.org/publicdomain/zero/1.0/
|
||||
|
|
@ -0,0 +1,141 @@
|
|||
# Splunk Add-on for malware-filter
|
||||
|
||||
Provide custom search commands to update [malware-filter](https://gitlab.com/malware-filter) lookups. Each command downloads from a source CSV and emit rows as events which can then be piped to a lookup file or used as a subsearch. Each command is exported globally and can be used in any app. This add-on currently does not have any UI.
|
||||
|
||||
## Usage
|
||||
|
||||
```
|
||||
| geturlhausfilter wildcard_prefix=<string> wildcard_suffix=<string> wildcard_affix=<string> message=<string>
|
||||
| outputlookup override_if_empty=false urlhaus-filter-splunk-online.csv
|
||||
```
|
||||
|
||||
Optional arguments:
|
||||
|
||||
- **wildcard_prefix** `<string>`: list of column names to have wildcard "\*" prefixed to their _non-empty_ value. New column(s) named "{column_name}\_wildcard_prefix" will be created. Non-existant column will be silently ignored. Accepted values: `"column_name"`, `"columnA,columnB"`.
|
||||
- **wildcard_suffix** `<string>`: Same as wildcard_prefix but have the wildcard suffixed instead.
|
||||
- **wildcard_affix** `<string>`: Same as wildcard_prefix but have the wildcard prefixed and suffixed.
|
||||
- **message** `<string>`: Add custom message column. New column "custom_message" will be created.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
| geturlhausfilter
|
||||
| outputlookup override_if_empty=false urlhaus-filter-splunk-online.csv
|
||||
```
|
||||
|
||||
| host | path | message | updated |
|
||||
| ------------ | ---------- | ----------------------------------------- | -------------------- |
|
||||
| example2.com | /some-path | urlhaus-filter malicious website detected | 2022-12-21T12:34:56Z |
|
||||
|
||||
```
|
||||
| geturlhausfilter wildcard_prefix=path message="lorem ipsum"
|
||||
| outputlookup override_if_empty=false urlhaus-filter-splunk-online.csv
|
||||
```
|
||||
|
||||
| host | path | message | updated | path_wildcard_prefix | message |
|
||||
| ------------ | ---------- | ----------------------------------------- | -------------------- | -------------------- | ----------- |
|
||||
| example2.com | /some-path | urlhaus-filter malicious website detected | 2022-12-21T12:34:56Z | \*/some-path | lorem ipsum |
|
||||
| example.com | | urlhaus-filter malicious website detected | 2022-12-21T12:34:56Z | | lorem ipsum |
|
||||
|
||||
## Lookup files
|
||||
|
||||
Lookup files are bundled but they are empty, run the relevant `| getsomething | outputlookup some-filter.csv` to get the latest lookup before using any of them.
|
||||
|
||||
- urlhaus-filter-splunk-online.csv
|
||||
- phishing-filter-splunk.csv
|
||||
- pup-filter-splunk.csv
|
||||
- vn-badsite-filter-splunk.csv
|
||||
- botnet-filter-splunk.csv
|
||||
- botnet_ip.csv
|
||||
- opendbl_ip.csv
|
||||
|
||||
## geturlhausfilter
|
||||
|
||||
```
|
||||
| geturlhausfilter wildcard_prefix=<string> wildcard_suffix=<string> wildcard_affix=<string> message=<string>
|
||||
| outputlookup override_if_empty=false urlhaus-filter-splunk-online.csv
|
||||
```
|
||||
|
||||
Output columns are listed here https://gitlab.com/malware-filter/urlhaus-filter#splunk
|
||||
|
||||
## getphishingfilter
|
||||
|
||||
```
|
||||
| getphishingfilter wildcard_prefix=<string> wildcard_suffix=<string> wildcard_affix=<string> message=<string>
|
||||
| outputlookup override_if_empty=false phishing-filter-splunk.csv
|
||||
```
|
||||
|
||||
Output columns are listed here https://gitlab.com/malware-filter/phishing-filter#splunk
|
||||
|
||||
## getpupfilter
|
||||
|
||||
```
|
||||
| getphishingfilter message=<string>
|
||||
| outputlookup override_if_empty=false pup-filter-splunk.csv
|
||||
```
|
||||
|
||||
Output columns are listed here https://gitlab.com/malware-filter/pup-filter#splunk
|
||||
|
||||
## getvnbadsitefilter
|
||||
|
||||
```
|
||||
| getphishingfilter wildcard_prefix=<string> wildcard_suffix=<string> wildcard_affix=<string> message=<string>
|
||||
| outputlookup override_if_empty=false vn-badsite-filter-splunk.csv
|
||||
```
|
||||
|
||||
Output columns are listed here https://gitlab.com/malware-filter/vn-badsite-filter#splunk
|
||||
|
||||
## getbotnetfilter
|
||||
|
||||
Highly recommend to use [`getbotnetip`](#getbotnetip) instead.
|
||||
|
||||
```
|
||||
| getphishingfilter message=<string>
|
||||
| outputlookup override_if_empty=false botnet-filter-splunk.csv
|
||||
```
|
||||
|
||||
Output columns are listed here https://gitlab.com/malware-filter/botnet-filter#splunk
|
||||
|
||||
## getbotnetip
|
||||
|
||||
Recommend to update the lookup file "botnet_ip.csv" every 5 minutes (cron `*/5 * * * *`).
|
||||
|
||||
```
|
||||
| getphishingfilter wildcard_prefix=<string> wildcard_suffix=<string> wildcard_affix=<string> message=<string>
|
||||
| outputlookup override_if_empty=false botnet_ip.csv
|
||||
```
|
||||
|
||||
Source: https://feodotracker.abuse.ch/downloads/ipblocklist.csv
|
||||
|
||||
Columns:
|
||||
|
||||
| first_seen_utc | dst_ip | dst_port | c2_status | last_online | malware | last_updated_utc |
|
||||
| ------------------- | ------------- | -------- | --------- | ----------- | ------- | ------------------- |
|
||||
| 2021-01-17 07:44:46 | 51.178.161.32 | 4643 | online | 2023-01-26 | Dridex | 2023-01-25 17:41:16 |
|
||||
|
||||
## getopendbl
|
||||
|
||||
Recommend to update the lookup file "opendbl_ip.csv" every 15 minutes (cron `*/15 * * * *`).
|
||||
|
||||
```
|
||||
| getopendbl wildcard_prefix=<string> wildcard_suffix=<string> wildcard_affix=<string> message=<string>
|
||||
| outputlookup override_if_empty=false opendbl_ip.csv
|
||||
```
|
||||
|
||||
Source: https://opendbl.net/
|
||||
|
||||
## Disabling individual commands
|
||||
|
||||
Settings -> All configurations -> filter by "malware_filter" app
|
||||
|
||||
## Build
|
||||
|
||||
`python build.py`
|
||||
|
||||
## Disclaimer
|
||||
|
||||
`getbotnetip.py` and `getopendbl.py` are included simply for convenience, their upstream sources are not affiliated with malware-filter.
|
||||
|
||||
## License
|
||||
|
||||
[Creative Commons Zero v1.0 Universal](LICENSE.md)
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
Get lookup csv from botnet-filter
|
||||
Usage: "| getbotnetfilter | outputlookup override_if_empty=false botnet-filter-splunk.csv"
|
||||
"""
|
||||
|
||||
import sys
|
||||
from os import path
|
||||
|
||||
sys.path.insert(0, path.join(path.dirname(__file__), "..", "lib"))
|
||||
from splunklib.searchcommands import Configuration, GeneratingCommand, Option, dispatch
|
||||
from utils import Utility
|
||||
|
||||
DOWNLOAD_URL = (
|
||||
"https://malware-filter.gitlab.io/malware-filter/botnet-filter-splunk.csv"
|
||||
)
|
||||
|
||||
|
||||
@Configuration()
|
||||
class GetBotnetFilter(Utility, GeneratingCommand):
|
||||
"""Defines a search command that generates event records"""
|
||||
|
||||
custom_message = Option(name="message")
|
||||
|
||||
def generate(self):
|
||||
dl_csv = self.download(DOWNLOAD_URL)
|
||||
for row in self.csv_reader(dl_csv):
|
||||
if isinstance(self.custom_message, str) and len(self.custom_message) >= 1:
|
||||
row["custom_message"] = self.custom_message
|
||||
|
||||
yield self.gen_record(**row)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
dispatch(GetBotnetFilter, sys.argv, sys.stdin, sys.stdout, __name__)
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
Get botnet IPs from feodo tracker
|
||||
Usage: "| getbotnetip | outputlookup override_if_empty=false botnet_ip.csv"
|
||||
Recommend to update the lookup file every 5 minutes (cron "*/5 * * * *")
|
||||
"""
|
||||
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from os import path
|
||||
from re import search
|
||||
|
||||
sys.path.insert(0, path.join(path.dirname(__file__), "..", "lib"))
|
||||
from splunklib.searchcommands import Configuration, GeneratingCommand, Option, dispatch
|
||||
from utils import Utility
|
||||
|
||||
DOWNLOAD_URL = "https://feodotracker.abuse.ch/downloads/ipblocklist.csv"
|
||||
|
||||
|
||||
@Configuration()
|
||||
class GetBotnetIP(Utility, GeneratingCommand):
|
||||
"""Defines a search command that generates event records"""
|
||||
|
||||
custom_message = Option(name="message")
|
||||
|
||||
def generate(self):
|
||||
feodo_csv = self.download(DOWNLOAD_URL)
|
||||
last_updated_utc = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
# parse updated time from header comment
|
||||
for line in filter(lambda row: row[0] == "#", feodo_csv.splitlines()):
|
||||
if line.startswith("# Last updated:"):
|
||||
last_updated_utc = search(
|
||||
r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}", line
|
||||
).group()
|
||||
break
|
||||
# parse input csv, remove '#' comments and output as events
|
||||
for row in self.csv_reader(feodo_csv):
|
||||
row["last_updated_utc"] = last_updated_utc
|
||||
if isinstance(self.custom_message, str) and len(self.custom_message) >= 1:
|
||||
row["custom_message"] = self.custom_message
|
||||
|
||||
yield self.gen_record(**row)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
dispatch(GetBotnetIP, sys.argv, sys.stdin, sys.stdout, __name__)
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
Get IP blocklists from OpenDBL
|
||||
Usage: "| getopendbl | outputlookup override_if_empty=false opendbl_ip.csv"
|
||||
Recommend to update the lookup file every 15 minutes (cron "*/15 * * * *")
|
||||
"""
|
||||
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from os import path
|
||||
from re import search
|
||||
|
||||
sys.path.insert(0, path.join(path.dirname(__file__), "..", "lib"))
|
||||
|
||||
from splunklib.searchcommands import Configuration, GeneratingCommand, Option, dispatch
|
||||
from utils import Utility
|
||||
|
||||
OPENDBL_LIST = {
|
||||
"Emerging Threats: Known Compromised Hosts": "etknown.list",
|
||||
"TOR exit nodes": "tor-exit.list",
|
||||
"BruteforceBlocker": "bruteforce.list",
|
||||
"Blocklist.de All": "blocklistde-all.list",
|
||||
"Talos": "talos.list",
|
||||
"Dshield": "dshield.list",
|
||||
"SSL Abuse IP list": "sslblock.list",
|
||||
}
|
||||
|
||||
|
||||
@Configuration()
|
||||
class GetOpenDBL(Utility, GeneratingCommand):
|
||||
"""Defines a search command that generates event records"""
|
||||
|
||||
custom_message = Option(name="message")
|
||||
|
||||
def generate(self):
|
||||
for name, dl_path in OPENDBL_LIST.items():
|
||||
blocklist = self.download(f"https://opendbl.net/lists/{dl_path}")
|
||||
last_updated_utc = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
# parse updated time from header comment
|
||||
for line in filter(lambda row: row[0] == "#", blocklist.splitlines()):
|
||||
if "Last updated" in line:
|
||||
last_updated = search(
|
||||
r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}", line
|
||||
).group()
|
||||
# Assume UTC timezone
|
||||
last_updated_utc = (
|
||||
datetime.strptime(last_updated, "%Y-%m-%d %H:%M")
|
||||
.replace(tzinfo=timezone.utc)
|
||||
.isoformat()
|
||||
)
|
||||
break
|
||||
|
||||
for line in filter(lambda row: row[0] != "#", blocklist.splitlines()):
|
||||
row = {
|
||||
"start": line,
|
||||
"end": line,
|
||||
"netmask": "32",
|
||||
"cidr": f"{line}/32",
|
||||
"name": name,
|
||||
"last_updated_utc": last_updated_utc,
|
||||
}
|
||||
|
||||
if "-" in line:
|
||||
row["start"], row["end"] = line.split("-")
|
||||
row["netmask"] = 24
|
||||
row["cidr"] = f"{row['start']}/{row['netmask']}"
|
||||
|
||||
if (
|
||||
isinstance(self.custom_message, str)
|
||||
and len(self.custom_message) >= 1
|
||||
):
|
||||
row["custom_message"] = self.custom_message
|
||||
|
||||
yield self.gen_record(**row)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
dispatch(GetOpenDBL, sys.argv, sys.stdin, sys.stdout, __name__)
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
Get lookup csv from phishing-filter
|
||||
Usage: "| getphishingfilter | outputlookup override_if_empty=false phishing-filter-splunk.csv"
|
||||
"""
|
||||
|
||||
import sys
|
||||
from os import path
|
||||
|
||||
sys.path.insert(0, path.join(path.dirname(__file__), "..", "lib"))
|
||||
from splunklib.searchcommands import Configuration, GeneratingCommand, Option, dispatch
|
||||
from utils import Utility
|
||||
|
||||
DOWNLOAD_URL = (
|
||||
"https://malware-filter.gitlab.io/malware-filter/phishing-filter-splunk.csv"
|
||||
)
|
||||
|
||||
|
||||
@Configuration()
|
||||
class GetPhishingFilter(Utility, GeneratingCommand):
|
||||
"""Defines a search command that generates event records"""
|
||||
|
||||
wildcard_prefix = Option(name="wildcard_prefix")
|
||||
wildcard_suffix = Option(name="wildcard_suffix")
|
||||
wildcard_affix = Option(name="wildcard_affix")
|
||||
custom_message = Option(name="message")
|
||||
|
||||
def generate(self):
|
||||
dl_csv = self.download(DOWNLOAD_URL)
|
||||
for row in self.csv_reader(dl_csv):
|
||||
if isinstance(self.custom_message, str) and len(self.custom_message) >= 1:
|
||||
row["custom_message"] = self.custom_message
|
||||
|
||||
affixed_row = self.insert_affix(
|
||||
row, self.wildcard_prefix, self.wildcard_suffix, self.wildcard_affix
|
||||
)
|
||||
|
||||
yield self.gen_record(**affixed_row)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
dispatch(GetPhishingFilter, sys.argv, sys.stdin, sys.stdout, __name__)
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
Get lookup csv from pup-filter
|
||||
Usage: "| getpupfilter | outputlookup override_if_empty=false pup-filter-splunk.csv"
|
||||
"""
|
||||
|
||||
import sys
|
||||
from os import path
|
||||
|
||||
sys.path.insert(0, path.join(path.dirname(__file__), "..", "lib"))
|
||||
from splunklib.searchcommands import Configuration, GeneratingCommand, Option, dispatch
|
||||
from utils import Utility
|
||||
|
||||
DOWNLOAD_URL = "https://malware-filter.gitlab.io/malware-filter/pup-filter-splunk.csv"
|
||||
|
||||
|
||||
@Configuration()
|
||||
class GetPupFilter(Utility, GeneratingCommand):
|
||||
"""Defines a search command that generates event records"""
|
||||
|
||||
wildcard_prefix = Option(name="wildcard_prefix")
|
||||
wildcard_suffix = Option(name="wildcard_suffix")
|
||||
wildcard_affix = Option(name="wildcard_affix")
|
||||
custom_message = Option(name="message")
|
||||
|
||||
def generate(self):
|
||||
dl_csv = self.download(DOWNLOAD_URL)
|
||||
for row in self.csv_reader(dl_csv):
|
||||
if isinstance(self.custom_message, str) and len(self.custom_message) >= 1:
|
||||
row["custom_message"] = self.custom_message
|
||||
affixed_row = self.insert_affix(
|
||||
row, self.wildcard_prefix, self.wildcard_suffix, self.wildcard_affix
|
||||
)
|
||||
yield self.gen_record(**affixed_row)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
dispatch(GetPupFilter, sys.argv, sys.stdin, sys.stdout, __name__)
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
Get lookup csv from urlhaus-filter
|
||||
Usage: "| geturlhausfilter | outputlookup override_if_empty=false urlhaus-filter-splunk-online.csv"
|
||||
"""
|
||||
|
||||
import sys
|
||||
from os import path
|
||||
|
||||
sys.path.insert(0, path.join(path.dirname(__file__), "..", "lib"))
|
||||
from splunklib.searchcommands import Configuration, GeneratingCommand, Option, dispatch
|
||||
from utils import Utility
|
||||
|
||||
DOWNLOAD_URL = (
|
||||
"https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-splunk-online.csv"
|
||||
)
|
||||
|
||||
|
||||
@Configuration()
|
||||
class GetUrlhausFilter(Utility, GeneratingCommand):
|
||||
"""Defines a search command that generates event records"""
|
||||
|
||||
wildcard_prefix = Option(name="wildcard_prefix")
|
||||
wildcard_suffix = Option(name="wildcard_suffix")
|
||||
wildcard_affix = Option(name="wildcard_affix")
|
||||
custom_message = Option(name="message")
|
||||
|
||||
def generate(self):
|
||||
dl_csv = self.download(DOWNLOAD_URL)
|
||||
for row in self.csv_reader(dl_csv):
|
||||
if isinstance(self.custom_message, str) and len(self.custom_message) >= 1:
|
||||
row["custom_message"] = self.custom_message
|
||||
|
||||
affixed_row = self.insert_affix(
|
||||
row, self.wildcard_prefix, self.wildcard_suffix, self.wildcard_affix
|
||||
)
|
||||
yield self.gen_record(**affixed_row)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
dispatch(GetUrlhausFilter, sys.argv, sys.stdin, sys.stdout, __name__)
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
Get lookup csv from vn-badsite-filter
|
||||
Usage: "| getvnbadsitefilter | outputlookup override_if_empty=false vn-badsite-filter-splunk.csv"
|
||||
"""
|
||||
|
||||
import sys
|
||||
from os import path
|
||||
|
||||
sys.path.insert(0, path.join(path.dirname(__file__), "..", "lib"))
|
||||
from splunklib.searchcommands import Configuration, GeneratingCommand, Option, dispatch
|
||||
from utils import Utility
|
||||
|
||||
DOWNLOAD_URL = (
|
||||
"https://malware-filter.gitlab.io/malware-filter/vn-badsite-filter-splunk.csv"
|
||||
)
|
||||
|
||||
|
||||
@Configuration()
|
||||
class GetVNBadsiteFilter(Utility, GeneratingCommand):
|
||||
"""Defines a search command that generates event records"""
|
||||
|
||||
wildcard_prefix = Option(name="wildcard_prefix")
|
||||
wildcard_suffix = Option(name="wildcard_suffix")
|
||||
wildcard_affix = Option(name="wildcard_affix")
|
||||
custom_message = Option(name="message")
|
||||
|
||||
def generate(self):
|
||||
dl_csv = self.download(DOWNLOAD_URL)
|
||||
for row in self.csv_reader(dl_csv):
|
||||
if isinstance(self.custom_message, str) and len(self.custom_message) >= 1:
|
||||
row["custom_message"] = self.custom_message
|
||||
|
||||
affixed_row = self.insert_affix(
|
||||
row, self.wildcard_prefix, self.wildcard_suffix, self.wildcard_affix
|
||||
)
|
||||
yield self.gen_record(**affixed_row)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
dispatch(GetVNBadsiteFilter, sys.argv, sys.stdin, sys.stdout, __name__)
|
||||
|
|
@ -0,0 +1,127 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
Common functions used in this add-on
|
||||
"""
|
||||
|
||||
from configparser import ConfigParser
|
||||
from csv import QUOTE_ALL, DictReader
|
||||
from os import environ, path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
class Utility:
|
||||
"""Provide common functions"""
|
||||
|
||||
def __get_proxy(self, url):
|
||||
"""
|
||||
Determine http proxy setting of a URL according to Splunk server configuration.
|
||||
Return {dict} of http/https proxy value if a URL should be proxied.
|
||||
"""
|
||||
hostname = urlparse(url).hostname
|
||||
|
||||
server_conf_path = path.join(
|
||||
environ.get("SPLUNK_HOME", path.join("opt", "splunk")),
|
||||
"etc",
|
||||
"system",
|
||||
"local",
|
||||
"server.conf",
|
||||
)
|
||||
server_conf = ConfigParser()
|
||||
server_conf.read(server_conf_path)
|
||||
proxy_config = (
|
||||
server_conf["proxyConfig"]
|
||||
if "proxyConfig" in server_conf.sections()
|
||||
else {}
|
||||
)
|
||||
proxy_rules = proxy_config.get("proxy_rules", "")
|
||||
no_proxy_rules = proxy_config.get("no_proxy", "")
|
||||
http_proxy = proxy_config.get("http_proxy", "")
|
||||
https_proxy = proxy_config.get("https_proxy", "")
|
||||
|
||||
# https://docs.splunk.com/Documentation/Splunk/9.0.3/Admin/Serverconf#Splunkd_http_proxy_configuration
|
||||
# pylint: disable=too-many-boolean-expressions
|
||||
if (
|
||||
# either configs should not be empty
|
||||
(len(http_proxy) >= 1 or len(https_proxy) >= 1)
|
||||
# hostname should not be excluded by no_proxy
|
||||
and hostname not in no_proxy_rules
|
||||
# if proxy_rules is set, should include hostname
|
||||
and (
|
||||
len(proxy_rules) == 0
|
||||
or (len(proxy_rules) >= 1 and hostname in proxy_rules)
|
||||
)
|
||||
):
|
||||
return {"proxies": {"http": http_proxy, "https": https_proxy}}
|
||||
|
||||
return {}
|
||||
|
||||
def download(self, url):
|
||||
"""Send a GET request to the URL and return content of the response."""
|
||||
proxy_config = self.__get_proxy(url)
|
||||
try:
|
||||
res = requests.get(url, timeout=5, **proxy_config)
|
||||
res.raise_for_status()
|
||||
return res.text
|
||||
except requests.exceptions.HTTPError as errh:
|
||||
raise errh
|
||||
except requests.exceptions.ConnectionError as errc:
|
||||
raise errc
|
||||
except requests.exceptions.Timeout as errt:
|
||||
raise errt
|
||||
except requests.exceptions.RequestException as err:
|
||||
raise err
|
||||
|
||||
def __split_column(self, input_str=None):
|
||||
"""Split {string} into {list} using comma separator"""
|
||||
if isinstance(input_str, str):
|
||||
return [x.strip() for x in input_str.split(",")]
|
||||
if isinstance(input_str, list):
|
||||
return input_str
|
||||
return []
|
||||
|
||||
def insert_affix(self, row, prefix_opt=None, suffix_opt=None, affix_opt=None):
|
||||
"""
|
||||
Affix wildcard "*" character to existing values
|
||||
|
||||
Arguments:
|
||||
row {dict} -- A row of an array-parsed CSV
|
||||
prefix_opt {string/list} -- A column name or a comma-separated list of column names to have wildcard prefixed to their non-empty value.
|
||||
suffix_opt {string/list} -- Same as prefix_opt but have the wildcard suffixed instead.
|
||||
affix_opt {string/list} -- Same as prefix_opt but have the wildcard prefixed and suffixed.
|
||||
|
||||
Return:
|
||||
A new row with prefix/suffix columns appended
|
||||
"""
|
||||
prefix_opt_list = self.__split_column(prefix_opt)
|
||||
suffix_opt_list = self.__split_column(suffix_opt)
|
||||
affix_opt_list = self.__split_column(affix_opt)
|
||||
new_column = {}
|
||||
for column in prefix_opt_list:
|
||||
if column in row and len(row[column]) >= 1:
|
||||
new_column = {
|
||||
**new_column,
|
||||
**{f"{column}_wildcard_prefix": f"*{row[column]}"},
|
||||
}
|
||||
for column in suffix_opt_list:
|
||||
if column in row and len(row[column]) >= 1:
|
||||
new_column = {
|
||||
**new_column,
|
||||
**{f"{column}_wildcard_suffix": f"{row[column]}*"},
|
||||
}
|
||||
for column in affix_opt_list:
|
||||
if column in row and len(row[column]) >= 1:
|
||||
new_column = {
|
||||
**new_column,
|
||||
**{f"{column}_wildcard_affix": f"*{row[column]}*"},
|
||||
}
|
||||
|
||||
return {**row, **new_column}
|
||||
|
||||
def csv_reader(self, csv_str):
|
||||
"""Parse an CSV input string into an interable of {dict} rows whose keys correspond to column names"""
|
||||
return DictReader(
|
||||
filter(lambda row: row[0] != "#", csv_str.splitlines()), quoting=QUOTE_ALL
|
||||
)
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
"""Build Splunk app package"""
|
||||
|
||||
import tarfile
|
||||
from configparser import ConfigParser
|
||||
from os import environ, path
|
||||
from re import search, sub
|
||||
from subprocess import check_call
|
||||
from sys import executable
|
||||
|
||||
|
||||
def version():
|
||||
"""
|
||||
Return version number from app.conf or commit hash if in CI
|
||||
"""
|
||||
|
||||
commit_sha = (
|
||||
# gitlab
|
||||
environ.get("CI_COMMIT_SHORT_SHA")
|
||||
# github
|
||||
or environ.get("GITHUB_SHA", "")[0:8]
|
||||
)
|
||||
if commit_sha:
|
||||
return commit_sha
|
||||
|
||||
app_conf_path = path.join(
|
||||
"default",
|
||||
"app.conf",
|
||||
)
|
||||
app_conf = ConfigParser()
|
||||
app_conf.read(app_conf_path)
|
||||
launcher = app_conf["launcher"] if "launcher" in app_conf.sections() else {}
|
||||
return launcher.get("version", "")
|
||||
|
||||
|
||||
def exclusion(tarinfo):
|
||||
"""Exclude dev files and cache, and reset file stats"""
|
||||
|
||||
# exclude certain folders/files
|
||||
pathname = tarinfo.name
|
||||
if search(
|
||||
r"/\.|\\\.|__pycache__|pyproject.toml|requirements-dev.txt|build.py", pathname
|
||||
):
|
||||
return None
|
||||
|
||||
# rename parent folder as "malware_filter"
|
||||
tarinfo.name = sub(r"^.", "malware_filter", pathname)
|
||||
|
||||
# reset file stats
|
||||
# based on https://splunkbase.splunk.com/app/833
|
||||
tarinfo.uid = 1001
|
||||
tarinfo.gid = 123
|
||||
tarinfo.uname = tarinfo.gname = ""
|
||||
|
||||
return tarinfo
|
||||
|
||||
|
||||
print("Installing dependencies into './lib/'...")
|
||||
check_call(
|
||||
[
|
||||
executable,
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--quiet",
|
||||
"-r",
|
||||
"requirements.txt",
|
||||
"-t",
|
||||
"lib",
|
||||
"--upgrade",
|
||||
]
|
||||
)
|
||||
|
||||
pkg_file = f"malware_filter-{version()}.tar.gz"
|
||||
print(f"Creating {pkg_file}...")
|
||||
with tarfile.open(pkg_file, "w:gz") as tar:
|
||||
tar.add(".", filter=exclusion)
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
#
|
||||
# App configuration file
|
||||
#
|
||||
[install]
|
||||
is_configured = false
|
||||
|
||||
[package]
|
||||
id = malware_filter
|
||||
check_for_updates = false
|
||||
|
||||
[ui]
|
||||
is_visible = false
|
||||
label = malware-filter
|
||||
|
||||
[launcher]
|
||||
author = Ming Di Leom
|
||||
description = Update malware-filter lookups. https://gitlab.com/malware-filter
|
||||
version = 0.0.1
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
#
|
||||
# Custom search command
|
||||
#
|
||||
[default]
|
||||
chunked = true
|
||||
python.version = python3
|
||||
generating = true
|
||||
|
||||
[geturlhausfilter]
|
||||
filename = geturlhausfilter.py
|
||||
|
||||
[getphishingfilter]
|
||||
filename = getphishingfilter.py
|
||||
|
||||
[getpupfilter]
|
||||
filename = getpupfilter.py
|
||||
|
||||
[getvnbadsitefilter]
|
||||
filename = getvnbadsitefilter.py
|
||||
|
||||
[getbotnetfilter]
|
||||
filename = getbotnetfilter.py
|
||||
|
||||
[getbotnetip]
|
||||
filename = getbotnetip.py
|
||||
|
||||
[getopendbl]
|
||||
filename = getopendbl.py
|
||||
|
||||
[updategeoipdb]
|
||||
filename = updategeoipdb.py
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
#
|
||||
# Search assistant text for custom search command
|
||||
#
|
||||
|
||||
[options]
|
||||
syntax = (wildcard_prefix=<column_names>)? (wildcard_suffix=<column_names>)? (wildcard_affix=<column_names>)? (message=<string>)?
|
||||
description = 'wildcard_*' controls which columns to have their value\
|
||||
affixed with wildcard character. Affixed value will be added to a\
|
||||
new column. 'message' adds a custom message to new column custom_message.
|
||||
|
||||
[message-option]
|
||||
syntax = (message=<string>)?
|
||||
description = 'message' adds a custom message to new column custom_message.
|
||||
|
||||
[geturlhausfilter-command]
|
||||
syntax = geturlhausfilter <options>
|
||||
description = Get urlhaus-filter from malware-filter.
|
||||
usage = public
|
||||
example = | geturlhausfilter wildcard_prefix="path" message="lorem ipsum"
|
||||
related = getphishingfilter getpupfilter getvnbadsitefilter getbotnetfilter
|
||||
|
||||
[getphishingfilter-command]
|
||||
syntax = getphishingfilter <options>
|
||||
description = Get phishing-filter from malware-filter.
|
||||
usage = public
|
||||
example = | getphishingfilter wildcard_prefix="path" message="lorem ipsum"
|
||||
related = geturlhausfilter getpupfilter getvnbadsitefilter getbotnetfilter
|
||||
|
||||
[getpupfilter-command]
|
||||
syntax = getpupfilter <options>
|
||||
description = Get pup-filter from malware-filter.
|
||||
usage = public
|
||||
example = | getpupfilter wildcard_prefix="path" message="lorem ipsum"
|
||||
related = geturlhausfilter getphishingfilter getvnbadsitefilter getbotnetfilter
|
||||
|
||||
[getvnbadsitefilter-command]
|
||||
syntax = getvnbadsitefilter <options>
|
||||
description = Get vn-badsite-filter from malware-filter.
|
||||
usage = public
|
||||
example = | getvnbadsitefilter wildcard_prefix="path" message="lorem ipsum"
|
||||
related = geturlhausfilter getphishingfilter getpupfilter getbotnetfilter
|
||||
|
||||
[getbotnetfilter-command]
|
||||
syntax = getbotnetfilter <message-option>
|
||||
shortdesc = Get botnet-filter from malware-filter.
|
||||
description = Get botnet-filter from malware-filter.\
|
||||
Please use 'getbotnetip' whenever possible.
|
||||
usage = public
|
||||
example = | getbotnetfilter message="lorem ipsum"
|
||||
related = geturlhausfilter getphishingfilter getpupfilter getvnbadsitefilter
|
||||
|
||||
[getbotnetip-command]
|
||||
syntax = getbotnetip <message-option>
|
||||
description = Get botnet ip from Feodo Tracker.
|
||||
usage = public
|
||||
example = | getbotnetip message="lorem ipsum"
|
||||
related = getopendbl
|
||||
|
||||
[getopendbl-command]
|
||||
syntax = getopendbl <message-option>
|
||||
description = Get ip blocklists from Open Dynamic Block Lists (Opendbl).
|
||||
usage = public
|
||||
example = | getopendbl message="lorem ipsum"
|
||||
related = getbotnetip
|
||||
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -0,0 +1,10 @@
|
|||
# Application-level permissions
|
||||
[]
|
||||
access = read : [ * ], write : [ admin, power, sc_admin ]
|
||||
export = system
|
||||
|
||||
[lookups]
|
||||
export = system
|
||||
|
||||
[searchbnf]
|
||||
export = system
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
[tool.isort]
|
||||
profile = "black"
|
||||
|
||||
[tool.pylint.'MASTER']
|
||||
py-version = "3.10"
|
||||
init-hook='import sys; sys.path.append("./bin")'
|
||||
|
||||
[tool.pylint.'MESSAGES CONTROL']
|
||||
disable = [
|
||||
"raw-checker-failed",
|
||||
"bad-inline-option",
|
||||
"locally-disabled",
|
||||
"file-ignored",
|
||||
"suppressed-message",
|
||||
"useless-suppression",
|
||||
"deprecated-pragma",
|
||||
"use-symbolic-message-instead",
|
||||
"invalid-name",
|
||||
"unspecified-encoding", # assume UTF-8
|
||||
"line-too-long",
|
||||
"too-many-nested-blocks",
|
||||
"too-many-branches",
|
||||
"duplicate-code",
|
||||
"redefined-outer-name",
|
||||
"fixme",
|
||||
"wrong-import-position"
|
||||
]
|
||||
|
||||
[tool.pylint.'FORMAT']
|
||||
indent-after-paren = 4
|
||||
indent-string = " "
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
-r requirements.txt
|
||||
pylint == 2.*
|
||||
isort == 5.*
|
||||
pre-commit == 2.*
|
||||
black ~= 22.1
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
requests == 2.*
|
||||
splunk-sdk == 1.*
|
||||
Loading…
Reference in New Issue