11 changed files with 89 additions and 78 deletions
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@ -13,20 +13,20 @@ repos:
            "s|^#\\!.*|#\\!/usr/bin/env python|",
          ]
  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: "v4.6.0"
+    rev: "v4.4.0"
    hooks:
      - id: end-of-file-fixer
      - id: trailing-whitespace
  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "v0.5.5"
+    rev: "v0.0.291"
    hooks:
      - id: ruff
        args: [--fix, --exit-non-zero-on-fix]
  - repo: https://github.com/psf/black
-    rev: 24.4.2
+    rev: 23.9.1
    hooks:
      - id: black
  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v3.1.0"
+    rev: "v3.0.3"
    hooks:
      - id: prettier
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@ -3,13 +3,14 @@
  "files.trimTrailingWhitespace": true,
  "files.insertFinalNewline": true,
  "files.trimFinalNewlines": true,
+  "python.formatting.provider": "none",
  "[python]": {
    "editor.defaultFormatter": "ms-python.black-formatter",
    "editor.formatOnSave": true,
    "editor.tabSize": 4,
    "editor.codeActionsOnSave": {
-      "source.fixAll": "explicit",
-      "source.organizeImports": "explicit"
+      "source.fixAll": true,
+      "source.organizeImports": true
    }
  },
  "[markdown]": {
--- a/README.md
+++ b/README.md
@ -156,27 +156,6 @@ Recommend to update the lookup file "opendbl_ip.csv" every 15 minutes (cron `*/1

 Source: https://opendbl.net/

-## Example usage
-
-```
-| tstats summariesonly=true allow_old_summaries=true count FROM datamodel=Web WHERE Web.action="allowed"
-BY Web.user, Web.src, Web.dest, Web.site, Web.url, Web.category, Web.action, index, _time span=1s
-| rename Web.* AS *
-| lookup urlhaus-filter-splunk-online host AS site, host AS dest OUTPUT message AS description, updated
-| lookup urlhaus-filter-splunk-online path_wildcard_prefix AS vendor_url, host AS site, host AS dest OUTPUTNEW message AS description, updated
-| lookup phishing-filter-splunk host AS site, host AS dest OUTPUTNEW message AS description, updated
-| lookup phishing-filter-splunk path_wildcard_prefix AS vendor_url, host AS site, host AS dest OUTPUTNEW message AS description, updated
-| lookup pup-filter-splunk host AS site, host AS dest OUTPUTNEW message AS description, updated
-| lookup vn-badsite-filter-splunk host AS site, host AS dest OUTPUTNEW message AS description, updated
-| lookup botnet_ip dst_ip AS dest OUTPUTNEW malware AS description, updated
-| eval Description=description
-| search Description=*
-| eval updated=coalesce(updated, updated2, updated3, updated4, updated5, updated6, updated7), "Signature Last Updated"=strftime(strptime(updated." +0000","%Y-%m-%dT%H:%M:%SZ %z"),"%Y-%m-%d %H:%M:%S %z"), Time=strftime(_time, "%Y-%m-%d %H:%M:%S %z"), "Source IP"=src, Username=user, Domain=site, "Destination IP"=dest, URL=url, Action=action
-| table Time, index, "Signature Last Updated", "Source IP", Username, Domain, "Destination IP", Description, Action, URL
-```
-
-It is not recommended to use subsearch (e.g. `[| inputlookup urlhaus-filter-splunk-online.csv | fields host ]`) for these [lookup tables](./lookups/) especially [urlhaus-filter](./lookups/urlhaus-filter-splunk-online.csv) and [phishing-filter](./lookups/phishing-filter-splunk.csv) because they usually have more than 30,000 rows, which exceed the soft-limit of [10,000 rows](https://docs.splunk.com/Documentation/SplunkCloud/latest/Search/Aboutsubsearches#Subsearch_performance_considerations) returned by subsearch.
-
 ## Disable individual commands

 Settings -> All configurations -> filter by "malware_filter" app
@ -213,5 +192,3 @@ https://gitlab.com/curben/blog#repository-mirrors
 ## License

 [Creative Commons Zero v1.0 Universal](LICENSE-CC0.md) and [MIT License](LICENSE)
-
-[splunk-sdk-python](https://github.com/splunk/splunk-sdk-python) bundled with the package under lib folder: [Apache License 2.0](https://choosealicense.com/licenses/apache-2.0/)
--- a/app.manifest
+++ b/app.manifest
@ -5,7 +5,7 @@
    "id": {
      "group": null,
      "name": "TA-malware-filter",
-      "version": "0.2.0"
+      "version": "0.0.13"
    },
    "author": [
      {
--- a/bin/utils.py
+++ b/bin/utils.py
@ -4,8 +4,6 @@
 Common functions used in this add-on
 """

-from __future__ import annotations
-
 from configparser import ConfigParser
 from csv import QUOTE_ALL, DictReader
 from os import environ, path
@ -17,7 +15,7 @@ import requests
 class Utility:
    """Provide common functions"""

-    def __get_proxy(self, url: str) -> dict[str, dict[str, str]] | str:
+    def __get_proxy(self, url):
        """
        Determine http proxy setting of a URL according to Splunk server configuration.
        Return {dict} of http/https proxy value if a URL should be proxied.
@ -44,6 +42,7 @@ class Utility:
        https_proxy = proxy_config.get("https_proxy", "")

        # https://docs.splunk.com/Documentation/Splunk/9.0.3/Admin/Serverconf#Splunkd_http_proxy_configuration
+        # pylint: disable=too-many-boolean-expressions
        if (
            # either configs should not be empty
            (len(http_proxy) >= 1 or len(https_proxy) >= 1)
@ -59,12 +58,14 @@ class Utility:

        return {}

-    def download(self, urls: list | tuple | str, index: int = 0) -> str:
+    # pylint: disable=inconsistent-return-statements
+    def download(self, urls, index=0):
        """
        Send a GET request to the URL and return content of the response.

-        :param urls: A list of URLs to try in sequence
-        :param index: List's index to start
+        Arguments:
+            urls {list/tuple/string} -- A list of URLs to try in sequence
+            index -- List's index to start
        """
        if isinstance(urls, str):
            urls = (urls,)
@ -73,6 +74,7 @@ class Utility:
        proxy_config = self.__get_proxy(url)
        try:
            res = requests.get(url, timeout=5, **proxy_config)
+            # pylint: disable=no-member
            if res.status_code == requests.codes.ok:
                return res.text

@ -89,7 +91,7 @@ class Utility:
        except requests.exceptions.RequestException as err:
            raise err

-    def __split_column(self, input_str: str | list | None = None) -> list[str] | list:
+    def __split_column(self, input_str=None):
        """Split {string} into {list} using comma separator"""
        if isinstance(input_str, str):
            return [x.strip() for x in input_str.split(",")]
@ -97,30 +99,24 @@ class Utility:
            return input_str
        return []

-    def insert_affix(
-        self,
-        row: dict,
-        prefix_opt: str | list | None = None,
-        suffix_opt: str | list | None = None,
-        affix_opt: str | list | None = None,
-    ) -> dict:
+    def insert_affix(self, row, prefix_opt=None, suffix_opt=None, affix_opt=None):
        """
        Affix wildcard "*" character to existing values

-        :param row: A row of an array-parsed CSV
-        :param prefix_opt: A column name or a comma-separated list of column names to have
-        wildcard prefixed to their non-empty value.
-        :param suffix_opt: Same as prefix_opt but have the wildcard suffixed instead.
-        :param affix_opt: Same as prefix_opt but have the wildcard prefixed and suffixed.
+        Arguments:
+            row {dict} -- A row of an array-parsed CSV
+            prefix_opt {string/list} -- A column name or a comma-separated list of column names to have
+                                        wildcard prefixed to their non-empty value.
+            suffix_opt {string/list} -- Same as prefix_opt but have the wildcard suffixed instead.
+            affix_opt {string/list} -- Same as prefix_opt but have the wildcard prefixed and suffixed.

-        Return a new row with prefix/suffix columns appended
+        Return:
+            A new row with prefix/suffix columns appended
        """
-
        prefix_opt_list = self.__split_column(prefix_opt)
        suffix_opt_list = self.__split_column(suffix_opt)
        affix_opt_list = self.__split_column(affix_opt)
        new_column = {}
-
        for column in prefix_opt_list:
            if column in row and len(row[column]) >= 1:
                new_column = {
@ -142,7 +138,7 @@ class Utility:

        return {**row, **new_column}

-    def csv_reader(self, csv_str: str) -> DictReader:
+    def csv_reader(self, csv_str):
        """Parse an CSV input string into an interable of {dict} rows whose keys correspond to column names"""
        return DictReader(
            filter(lambda row: row[0] != "#", csv_str.splitlines()), quoting=QUOTE_ALL
--- a/build.py
+++ b/build.py
@ -12,7 +12,7 @@ from subprocess import check_call
 from sys import executable


-def version() -> str:
+def version():
    """
    Return version number from app.conf or commit hash if in CI
    """
@ -46,7 +46,7 @@ def version() -> str:
    return launcher.get("version", "")


-def exclusion(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo | None:
+def exclusion(tarinfo):
    """Exclude dev files and cache, and reset file stats"""

    # exclude certain folders/files
@ -89,7 +89,7 @@ check_call(
        "pip",
        "install",
        "--quiet",
-        "splunk-sdk == 2.*",
+        "splunk-sdk == 1.*",
        "-t",
        "lib",
        "--upgrade",
--- a/default/app.conf
+++ b/default/app.conf
@ -1,3 +1,6 @@
+#
+# App configuration file
+#
 [install]
 is_configured = false

@ -6,7 +9,7 @@ id = TA-malware-filter

 [id]
 name = TA-malware-filter
-version = 0.2.0
+version = 0.1.0

 [ui]
 is_visible = false
@ -15,4 +18,4 @@ label = malware-filter Add-on
 [launcher]
 author = Ming Di Leom
 description = Update malware-filter lookups. https://gitlab.com/malware-filter
-version = 0.2.0
+version = 0.1.0
--- a/default/savedsearches.conf
+++ b/default/savedsearches.conf
@ -1,63 +1,70 @@
 [malware-filter Update botnet_ip.csv]
+action.lookup = 1
+action.lookup.filename = botnet_ip.csv
 cron_schedule = */15 * * * *
 description = Update lookup every 15 minutes from 00:00
-# https://docs.splunk.com/Documentation/Splunk/latest/SearchReference/Collect#Events_without_timestamps
+# https://docs.splunk.com/Documentation/Splunk/9.1.1/SearchReference/Collect#Events_without_timestamps
 dispatch.earliest_time = 0
 enableSched = 0
-schedule_window = 5
-search = | getbotnetip\
-| outputlookup override_if_empty=false botnet_ip.csv
+schedule_window = 60
+search = | getbotnetip

 [malware-filter Update botnet-filter-splunk.csv]
+action.lookup = 1
+action.lookup.filename = botnet-filter-splunk.csv
 cron_schedule = 0 */12 * * *
 description = Update lookup every 12 hours from 00:00
 dispatch.earliest_time = 0
 enableSched = 0
 schedule_window = 60
-search = | getbotnetfilter\
-| outputlookup override_if_empty=false botnet-filter-splunk.csv
+search = | getbotnetfilter

 [malware-filter Update opendbl_ip.csv]
+action.lookup = 1
+action.lookup.filename = opendbl_ip.csv
 cron_schedule = */15 * * * *
 description = Update lookup every 15 minutes from 00:00
 dispatch.earliest_time = 0
 enableSched = 0
-schedule_window = 5
-search = | getopendbl\
-| outputlookup override_if_empty=false opendbl_ip.csv
+schedule_window = 60
+search = | getopendbl

 [malware-filter Update phishing-filter-splunk.csv]
+action.lookup = 1
+action.lookup.filename = phishing-filter-splunk.csv
 cron_schedule = 0 */12 * * *
 description = Update lookup every 12 hours from 00:00
 dispatch.earliest_time = 0
 enableSched = 0
 schedule_window = 60
-search = | getphishingfilter\
-| outputlookup override_if_empty=false phishing-filter-splunk.csv
+search = | getphishingfilter

 [malware-filter Update pup-filter-splunk.csv]
+action.lookup = 1
+action.lookup.filename = pup-filter-splunk.csv
 cron_schedule = 0 */12 * * *
 description = Update lookup every 12 hours from 00:00
 dispatch.earliest_time = 0
 enableSched = 0
 schedule_window = 60
-search = | getpupfilter\
-| outputlookup override_if_empty=false pup-filter-splunk.csv
+search = | getpupfilter

 [malware-filter Update urlhaus-filter-splunk-online.csv]
+action.lookup = 1
+action.lookup.filename = urlhaus-filter-splunk-online.csv
 cron_schedule = 0 */12 * * *
 description = Update lookup every 12 hours from 00:00
 dispatch.earliest_time = 0
 enableSched = 0
 schedule_window = 60
-search = | geturlhausfilter\
-| outputlookup override_if_empty=false urlhaus-filter-splunk-online.csv
+search = | geturlhausfilter

 [malware-filter Update vn-badsite-filter-splunk.csv]
+action.lookup = 1
+action.lookup.filename = vn-badsite-filter-splunk.csv
 cron_schedule = 0 */12 * * *
 description = Update lookup every 12 hours from 00:00
 dispatch.earliest_time = 0
 enableSched = 0
 schedule_window = 60
-search = | getvnbadsitefilter\
-| outputlookup override_if_empty=false vn-badsite-filter-splunk.csv
+search = | getvnbadsitefilter
--- a/default/transforms.conf
+++ b/default/transforms.conf
@ -1,32 +1,39 @@
 [urlhaus-filter-splunk-online]
+batch_index_query = 0
 case_sensitive_match = 1
 filename = urlhaus-filter-splunk-online.csv
 max_matches = 1

 [phishing-filter-splunk]
+batch_index_query = 0
 case_sensitive_match = 1
 filename = phishing-filter-splunk.csv
 max_matches = 1

 [pup-filter-splunk]
+batch_index_query = 0
 case_sensitive_match = 1
 filename = pup-filter-splunk.csv
 max_matches = 1

 [vn-badsite-filter-splunk]
+batch_index_query = 0
 case_sensitive_match = 1
 filename = vn-badsite-filter-splunk.csv
 max_matches = 1

 [botnet-filter-splunk]
+batch_index_query = 0
 case_sensitive_match = 1
 filename = botnet-filter-splunk.csv

 [botnet_ip]
+batch_index_query = 0
 case_sensitive_match = 1
 filename = botnet_ip.csv

 [opendbl_ip]
+batch_index_query = 0
 case_sensitive_match = 1
 filename = opendbl_ip.csv
 min_matches = 1
--- a/pyproject.toml
+++ b/pyproject.toml
@ -1,8 +1,28 @@
 [tool.pylint.'MASTER']
-# https://docs.splunk.com/Documentation/Splunk/9.3.0/Python3Migration/PythonCompatibility
-py-version = "3.7"
+py-version = "3.10"
 init-hook='import sys; sys.path.append("./bin")'

+[tool.pylint.'MESSAGES CONTROL']
+disable = [
+  "raw-checker-failed",
+  "bad-inline-option",
+  "locally-disabled",
+  "file-ignored",
+  "suppressed-message",
+  "useless-suppression",
+  "deprecated-pragma",
+  "use-symbolic-message-instead",
+  "invalid-name",
+  "unspecified-encoding", # assume UTF-8
+  "line-too-long",
+  "too-many-nested-blocks",
+  "too-many-branches",
+  "duplicate-code",
+  "redefined-outer-name",
+  "fixme",
+  "wrong-import-position"
+]
+
 [tool.pylint.'FORMAT']
 indent-after-paren = 4
 indent-string = "    "
--- a/requirements.txt
+++ b/requirements.txt
@ -1,2 +1,2 @@
 requests == 2.*
-splunk-sdk == 2.*
+splunk-sdk == 1.*