diff --git a/.gitignore b/.gitignore index d23632f..505249e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ toots.db toots.db-journal toots.db-wal __pycache__/* +.vscode/ +.editorconfig +.*.swp diff --git a/config.json b/config.json index 7b1521d..cc02a33 100644 --- a/config.json +++ b/config.json @@ -1 +1,4 @@ -{"site":"https://botsin.space","cw":null} \ No newline at end of file +{ + "site": "https://botsin.space", + "cw": null +} \ No newline at end of file diff --git a/create.py b/create.py deleted file mode 100755 index caa62a3..0000000 --- a/create.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import markovify -import json -import re, random, multiprocessing, time, sqlite3, shutil, os - -def make_sentence(output): - class nlt_fixed(markovify.NewlineText): - def test_sentence_input(self, sentence): - return True #all sentences are valid <3 - - # with open("corpus.txt", encoding="utf-8") as fp: - # model = nlt_fixed(fp.read()) - - shutil.copyfile("toots.db", "toots-copy.db") - db = sqlite3.connect("toots-copy.db") - db.text_factory=str - c = db.cursor() - toots = c.execute("SELECT content FROM `toots`").fetchall() - toots_str = "" - for toot in toots: - toots_str += "\n{}".format(toot[0]) - model = nlt_fixed(toots_str) - toots_str = None - db.close() - os.remove("toots-copy.db") - - sentence = None - tries = 0 - while sentence is None and tries < 10: - sentence = model.make_short_sentence(500, tries=10000) - tries = tries + 1 - sentence = re.sub("^@\u202B[^ ]* ", "", sentence) - output.send(sentence) - -def make_toot(force_markov = False, args = None): - return make_toot_markov() - -def make_toot_markov(query = None): - tries = 0 - toot = None - while toot == None and tries < 25: - pin, pout = multiprocessing.Pipe(False) - p = multiprocessing.Process(target = make_sentence, args = [pout]) - p.start() - p.join(10) - if p.is_alive(): - p.terminate() - p.join() - toot = None - tries = tries + 1 - else: - toot = pin.recv() - if toot == None: - toot = "Toot generation failed! Contact Lynne for assistance." - return { - "toot":toot, - "media":None - } diff --git a/functions.py b/functions.py new file mode 100755 index 0000000..c21271a --- /dev/null +++ b/functions.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import markovify +from bs4 import BeautifulSoup +import re, multiprocessing, sqlite3, shutil, os, json + +def make_sentence(output): + class nlt_fixed(markovify.NewlineText): #modified version of NewlineText that never rejects sentences + def test_sentence_input(self, sentence): + return True #all sentences are valid <3 + + shutil.copyfile("toots.db", "toots-copy.db") #create a copy of the database because reply.py will be using the main one + db = sqlite3.connect("toots-copy.db") + db.text_factory=str + c = db.cursor() + toots = c.execute("SELECT content FROM `toots` ORDER BY RANDOM() LIMIT 10000").fetchall() + toots_str = "" + for toot in toots: + toots_str += "\n{}".format(toot[0]) + model = nlt_fixed(toots_str) + toots_str = None + db.close() + os.remove("toots-copy.db") + + sentence = None + tries = 0 + while sentence is None and tries < 10: + sentence = model.make_short_sentence(500, tries=10000) + tries = tries + 1 + + sentence = re.sub("^(?:@\u202B[^ ]* )*", "", sentence) #remove leading pings (don't say "@bob blah blah" but still say "blah @bob blah") + sentence = re.sub("^(?:@\u200B[^ ]* )*", "", sentence) + + output.send(sentence) + +def make_toot(force_markov = False, args = None): + return make_toot_markov() + +def make_toot_markov(query = None): + tries = 0 + toot = None + while toot == None and tries < 10: #try to make a toot 10 times + pin, pout = multiprocessing.Pipe(False) + p = multiprocessing.Process(target = make_sentence, args = [pout]) + p.start() + p.join(10) #wait 10 seconds to get something + if p.is_alive(): #if it's still trying to make a toot after 10 seconds + p.terminate() + p.join() + toot = None + tries = tries + 1 #give up, and increment tries by one + else: + toot = pin.recv() + if toot == None: #if we've tried and failed ten times, just give up + toot = "Toot generation failed! Contact Lynne (lynnesbian@fedi.lynnesbian.space) for assistance." + return { + "toot": toot, + "media": None + } + +def extract_toot(toot): + toot = toot.replace("'", "'") #convert HTML stuff to normal stuff + toot = toot.replace(""", '"') #ditto + soup = BeautifulSoup(toot, "html.parser") + for lb in soup.select("br"): #replace
with linebreak + lb.insert_after("\n") + lb.decompose() + + for p in soup.select("p"): #ditto for

+ p.insert_after("\n") + p.unwrap() + + for ht in soup.select("a.hashtag"): #make hashtags no longer links, just text + ht.unwrap() + + for link in soup.select("a"): #ocnvert with linebreak - for lb in soup.select("br"): - lb.insert_after("\n") - lb.decompose() - - # replace

with linebreak - for p in soup.select("p"): - p.insert_after("\n") - p.unwrap() - - # fix hashtags - for ht in soup.select("a.hashtag"): - ht.unwrap() - - # fix links - for link in soup.select("a"): - link.insert_after(link["href"]) - link.decompose() - - toot = soup.get_text() - toot = toot.rstrip("\n") #remove trailing newline + toot = functions.extract_toot(toot) toot = toot.replace("@", "@\u200B") #put a zws between @ and username to avoid mentioning return(toot) @@ -104,25 +92,12 @@ def handleCtrlC(signal, frame): signal.signal(signal.SIGINT, handleCtrlC) -def get_toots_legacy(client, id): - i = 0 - toots = client.account_statuses(id) - while toots is not None and len(toots) > 0: - for toot in toots: - if toot.spoiler_text != "": continue - if toot.reblog is not None: continue - if toot.visibility not in ["public", "unlisted"]: continue - t = extract_toot(toot.content) - if t != None: - yield { - "toot": t, - "id": toot.id, - "uri": toot.uri - } - toots = client.fetch_next(toots) - i += 1 - if i%20 == 0: - print('.', end='', flush=True) +patterns = { + "handle": re.compile(r"^.*@(.+)"), + "url": re.compile(r"https?:\/\/(.*)"), + "uri": re.compile(r'template="([^"]+)"'), + "pid": re.compile(r"[^\/]+$"), +} for f in following: last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY id DESC LIMIT 1", (f.id,)).fetchone() @@ -133,28 +108,27 @@ for f in following: print("Harvesting toots for user @{}, starting from {}".format(f.acct, last_toot)) #find the user's activitypub outbox - print("WebFingering...") - instance = re.search(r"^.*@(.+)", f.acct) + print("WebFingering... (do not laugh at this. WebFinger is a federated protocol. https://wikipedia.org/wiki/WebFinger)") + instance = patterns["handle"].search(f.acct) if instance == None: - instance = re.search(r"https?:\/\/(.*)", cfg['site']).group(1) + instance = patterns["url"].search(cfg['site']).group(1) else: instance = instance.group(1) - if instance == "bofa.lol": - print("rest in piece bofa, skipping") + if instance in cfg['instance_blacklist']: + print("skipping blacklisted instance: {}".format(instance)) continue - - # print("{} is on {}".format(f.acct, instance)) + try: r = requests.get("https://{}/.well-known/host-meta".format(instance), timeout=10) - uri = re.search(r'template="([^"]+)"', r.text).group(1) + uri = patterns["uri"].search(r.text).group(1) uri = uri.format(uri = "{}@{}".format(f.username, instance)) r = requests.get(uri, headers={"Accept": "application/json"}, timeout=10) j = r.json() - if len(j['aliases']) == 1: #TODO: this is a hack on top of a hack, fix it - uri = j['aliases'][0] - else: - uri = j['aliases'][1] + for link in j['links']: + if link['rel'] == 'self': + #this is a link formatted like "https://instan.ce/users/username", which is what we need + uri = link['href'] uri = "{}/outbox?page=true".format(uri) r = requests.get(uri, timeout=10) j = r.json() @@ -168,23 +142,23 @@ for f in following: pleroma = True j = j['first'] else: - print("Mastodon instance detected") + print("Mastodon/Misskey instance detected") uri = "{}&min_id={}".format(uri, last_toot) r = requests.get(uri) j = r.json() - print("Downloading and parsing toots", end='', flush=True) + print("Downloading and saving toots", end='', flush=True) done = False try: while not done and len(j['orderedItems']) > 0: for oi in j['orderedItems']: if oi['type'] != "Create": - continue #not a toost. fuck outta here + continue #this isn't a toot/post/status/whatever, it's a boost or a follow or some other activitypub thing. ignore # its a toost baby content = oi['object']['content'] if oi['object']['summary'] != None and oi['object']['summary'] != "": - #don't download CW'd toots + #don't download CW'd toots. if you want your bot to download and learn from CW'd toots, replace "continue" with "pass". (todo: add a config.json option for this) continue toot = extract_toot(content) # print(toot) @@ -192,11 +166,12 @@ for f in following: if pleroma: if c.execute("SELECT COUNT(*) FROM toots WHERE uri LIKE ?", (oi['object']['id'],)).fetchone()[0] > 0: #we've caught up to the notices we've already downloaded, so we can stop now + #you might be wondering, "lynne, what if the instance ratelimits you after 40 posts, and they've made 60 since main.py was last run? wouldn't the bot miss 20 posts and never be able to see them?" to which i reply, "it's called mstdn-ebooks not fediverse-ebooks. pleroma support is an afterthought" done = True break - pid = re.search(r"[^\/]+$", oi['object']['id']).group(0) - c.execute("REPLACE INTO toots (id, userid, uri, content) VALUES (?, ?, ?, ?)", - (pid, + pid = patterns["pid"].search(oi['object']['id']).group(0) + c.execute("REPLACE INTO toots (id, userid, uri, content) VALUES (?, ?, ?, ?)", ( + pid, f.id, oi['object']['id'], toot @@ -205,7 +180,6 @@ for f in following: pass except: pass #ignore any toots that don't successfully go into the DB - # sys.exit(0) if not pleroma: r = requests.get(j['prev'], timeout=15) else: @@ -215,9 +189,8 @@ for f in following: print(" Done!") db.commit() except: - print("Encountered an error! Saving toots to database and continuing.") + print("Encountered an error! Saving toots to database and moving to next followed account.") db.commit() - # db.close() print("Done!") diff --git a/reply.py b/reply.py index 36a5c0f..2e60de1 100755 --- a/reply.py +++ b/reply.py @@ -5,7 +5,7 @@ import mastodon import os, random, re, json -import create +import functions from bs4 import BeautifulSoup cfg = json.load(open('config.json', 'r')) @@ -17,40 +17,25 @@ client = mastodon.Mastodon( api_base_url=cfg['site']) def extract_toot(toot): - #copied from main.py, see there for comments - soup = BeautifulSoup(toot, "html.parser") - for lb in soup.select("br"): - lb.insert_after("\n") - lb.decompose() - for p in soup.select("p"): - p.insert_after("\n") - p.unwrap() - for ht in soup.select("a.hashtag"): - ht.unwrap() - for link in soup.select("a"): - link.insert_after(link["href"]) - link.decompose() - text = map(lambda a: a.strip(), soup.get_text().strip().split("\n")) - text = "\n".join(list(text)) - text = re.sub("https?://([^/]+)/(@[^ ]+)", r"\2@\1", text) #put mentions back in - text = re.sub("^@[^@]+@[^ ]+ *", r"", text) #...but remove the initial one - text = text.lower() #for easier matching + text = functions.extract_toot(toot) + text = re.sub(r"^@[^@]+@[^ ]+\s*", r"", text) #remove the initial mention + text = text.lower() #treat text as lowercase for easier keyword matching (if this bot uses it) return text class ReplyListener(mastodon.StreamListener): - def on_notification(self, notification): - if notification['type'] == 'mention': - acct = "@" + notification['account']['acct'] + def on_notification(self, notification): #listen for notifications + if notification['type'] == 'mention': #if we're mentioned: + acct = "@" + notification['account']['acct'] #get the account's @ post_id = notification['status']['id'] mention = extract_toot(notification['status']['content']) - toot = create.make_toot(True)['toot'] - toot = acct + " " + toot - print(acct + " says " + mention) + toot = functions.make_toot(True)['toot'] #generate a toot + toot = acct + " " + toot #prepend the @ + print(acct + " says " + mention) #logging visibility = notification['status']['visibility'] if visibility == "public": visibility = "unlisted" - client.status_post(toot, post_id, visibility=visibility, spoiler_text = cfg['cw']) - print("replied with " + toot) + client.status_post(toot, post_id, visibility=visibility, spoiler_text = cfg['cw']) #send toost + print("replied with " + toot) #logging rl = ReplyListener() -client.stream_user(rl) +client.stream_user(rl) #go!