Added word filtering from grumbulon's fork. Plan to eventually make this regenerate the post instead of just replacing the text

This commit is contained in:
Amber 2022-03-22 17:03:50 -05:00
parent 50fe24c4e4
commit dd503789e0
5 changed files with 19 additions and 2 deletions

1
.gitignore vendored
View File

@ -13,3 +13,4 @@ __pycache__/
config.json config.json
venv/ venv/
*.log *.log
filter.txt

View File

@ -11,5 +11,6 @@
"length_upper_limit": 50, "length_upper_limit": 50,
"overlap_ratio_enabled": false, "overlap_ratio_enabled": false,
"overlap_ratio": 0.7, "overlap_ratio": 0.7,
"word_filter": 0,
"website": "https://git.nixnet.services/amber/amber-ebooks" "website": "https://git.nixnet.services/amber/amber-ebooks"
} }

5
filter.txt.example Normal file
View File

@ -0,0 +1,5 @@
put
bad
words
in
filter.txt

View File

@ -56,6 +56,15 @@ def make_sentence(output, cfg):
elif cfg['mention_handling'] == 0: elif cfg['mention_handling'] == 0:
sentence = re.sub(r"\S*@\u200B\S*\s?", "", sentence) sentence = re.sub(r"\S*@\u200B\S*\s?", "", sentence)
# optionally remove filtered words
if cfg['word_filter'] == 1:
try:
fp = open('./filter.txt')
for word in fp:
if word in sentence:
sentence = sentence.replace(word, "[REDACTED]")
finally:
fp.close()
output.send(sentence) output.send(sentence)
@ -64,7 +73,7 @@ def make_toot(cfg):
pin, pout = multiprocessing.Pipe(False) pin, pout = multiprocessing.Pipe(False)
p = multiprocessing.Process(target=make_sentence, args=[pout, cfg]) p = multiprocessing.Process(target=make_sentence, args=[pout, cfg])
p.start() p.start()
p.join(300) # wait 5 seconds to get something p.join(5) # wait 5 seconds to get something
if p.is_alive(): # if it's still trying to make a toot after 5 seconds if p.is_alive(): # if it's still trying to make a toot after 5 seconds
p.terminate() p.terminate()
p.join() p.join()

View File

@ -30,7 +30,8 @@ cfg = {
"length_lower_limit": 5, "length_lower_limit": 5,
"length_upper_limit": 50, "length_upper_limit": 50,
"overlap_ratio_enabled": False, "overlap_ratio_enabled": False,
"overlap_ratio": 0.7 "overlap_ratio": 0.7,
"word_filter": 0
} }
try: try: