amber
/
amber-ebooks-archived
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

precompiled regexes used inside of loop

This commit is contained in:
Lucina Wilton 2019-02-07 10:27:52 -05:00
parent 910af03aef
commit f71c4af0e6
1 changed files with 135 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

257
main.py
View File

@ -16,61 +16,61 @@ cfg = json.load(open('config.json', 'r'))
#config.json *MUST* contain the instance URL, the instance blacklist (for dead/broken instances), and the CW text. if they're not provided, we'll fall back to defaults. #config.json *MUST* contain the instance URL, the instance blacklist (for dead/broken instances), and the CW text. if they're not provided, we'll fall back to defaults.
if 'site' not in cfg: if 'site' not in cfg:
cfg['website'] = "https://botsin.space" cfg['website'] = "https://botsin.space"
if 'cw' not in cfg: if 'cw' not in cfg:
cfg['cw'] = None cfg['cw'] = None
if 'instance_blacklist' not in cfg: if 'instance_blacklist' not in cfg:
cfg["instance_blacklist"] = [ cfg["instance_blacklist"] = [
"bofa.lol", "bofa.lol",
"witches.town" "witches.town"
] ]
#if the user is using a (very!) old version that still uses the .secret files, migrate to the new method #if the user is using a (very!) old version that still uses the .secret files, migrate to the new method
if os.path.exists("clientcred.secret"): if os.path.exists("clientcred.secret"):
print("Upgrading to new storage method") print("Upgrading to new storage method")
cc = open("clientcred.secret").read().split("\n") cc = open("clientcred.secret").read().split("\n")
cfg['client'] = { cfg['client'] = {
"id": cc[0], "id": cc[0],
"secret": cc[1] "secret": cc[1]
} }
cfg['secret'] = open("usercred.secret").read().rstrip("\n") cfg['secret'] = open("usercred.secret").read().rstrip("\n")
os.remove("clientcred.secret") os.remove("clientcred.secret")
os.remove("usercred.secret") os.remove("usercred.secret")
if "client" not in cfg: if "client" not in cfg:
print("No application info -- registering application with {}".format(cfg['site'])) print("No application info -- registering application with {}".format(cfg['site']))
client_id, client_secret = Mastodon.create_app("mstdn-ebooks", client_id, client_secret = Mastodon.create_app("mstdn-ebooks",
api_base_url=cfg['site'], api_base_url=cfg['site'],
scopes=scopes, scopes=scopes,
website="https://github.com/Lynnesbian/mstdn-ebooks") website="https://github.com/Lynnesbian/mstdn-ebooks")
cfg['client'] = { cfg['client'] = {
"id": client_id, "id": client_id,
"secret": client_secret "secret": client_secret
} }
if "secret" not in cfg: if "secret" not in cfg:
print("No user credentials -- logging in to {}".format(cfg['site'])) print("No user credentials -- logging in to {}".format(cfg['site']))
client = Mastodon(client_id = cfg['client']['id'], client = Mastodon(client_id = cfg['client']['id'],
client_secret = cfg['client']['secret'], client_secret = cfg['client']['secret'],
api_base_url=cfg['site']) api_base_url=cfg['site'])
print("Open this URL and authenticate to give mstdn-ebooks access to your bot's account: {}".format(client.auth_request_url(scopes=scopes))) print("Open this URL and authenticate to give mstdn-ebooks access to your bot's account: {}".format(client.auth_request_url(scopes=scopes)))
cfg['secret'] = client.log_in(code=input("Secret: "), scopes=scopes) cfg['secret'] = client.log_in(code=input("Secret: "), scopes=scopes)
json.dump(cfg, open("config.json", "w+")) json.dump(cfg, open("config.json", "w+"))
def extract_toot(toot): def extract_toot(toot):
toot = functions.extract_toot(toot) toot = functions.extract_toot(toot)
toot = toot.replace("@", "@\u200B") #put a zws between @ and username to avoid mentioning toot = toot.replace("@", "@\u200B") #put a zws between @ and username to avoid mentioning
return(toot) return(toot)
client = Mastodon( client = Mastodon(
client_id=cfg['client']['id'], client_id=cfg['client']['id'],
client_secret = cfg['client']['secret'], client_secret = cfg['client']['secret'],
access_token=cfg['secret'], access_token=cfg['secret'],
api_base_url=cfg['site']) api_base_url=cfg['site'])
me = client.account_verify_credentials() me = client.account_verify_credentials()
following = client.account_following(me.id) following = client.account_following(me.id)
@ -82,104 +82,111 @@ c.execute("CREATE TABLE IF NOT EXISTS `toots` (id INT NOT NULL UNIQUE PRIMARY KE
db.commit() db.commit()
def handleCtrlC(signal, frame): def handleCtrlC(signal, frame):
print("\nPREMATURE EVACUATION - Saving chunks") print("\nPREMATURE EVACUATION - Saving chunks")
db.commit() db.commit()
sys.exit(1) sys.exit(1)
signal.signal(signal.SIGINT, handleCtrlC) signal.signal(signal.SIGINT, handleCtrlC)
patterns = {
"handle": re.compile(r"^.*@(.+)"),
"url": re.compile(r"https?:\/\/(.*)"),
"uri": re.compile(r'template="([^"]+)"'),
"pid": re.compile(r"[^\/]+$"),
}
for f in following: for f in following:
last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY id DESC LIMIT 1", (f.id,)).fetchone() last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY id DESC LIMIT 1", (f.id,)).fetchone()
if last_toot != None: if last_toot != None:
last_toot = last_toot[0] last_toot = last_toot[0]
else: else:
last_toot = 0 last_toot = 0
print("Harvesting toots for user @{}, starting from {}".format(f.acct, last_toot)) print("Harvesting toots for user @{}, starting from {}".format(f.acct, last_toot))
#find the user's activitypub outbox #find the user's activitypub outbox
print("WebFingering... (do not laugh at this. WebFinger is a federated protocol. https://wikipedia.org/wiki/WebFinger)") print("WebFingering... (do not laugh at this. WebFinger is a federated protocol. https://wikipedia.org/wiki/WebFinger)")
instance = re.search(r"^.*@(.+)", f.acct) instance = patterns["handle"].search(f.acct)
if instance == None: if instance == None:
instance = re.search(r"https?:\/\/(.*)", cfg['site']).group(1) instance = patterns["url"].search(cfg['site']).group(1)
else: else:
instance = instance.group(1) instance = instance.group(1)
if instance in cfg['instance_blacklist']: if instance in cfg['instance_blacklist']:
print("skipping blacklisted instance: {}".format(instance)) print("skipping blacklisted instance: {}".format(instance))
continue continue
try: try:
r = requests.get("https://{}/.well-known/host-meta".format(instance), timeout=10) r = requests.get("https://{}/.well-known/host-meta".format(instance), timeout=10)
uri = re.search(r'template="([^"]+)"', r.text).group(1) uri = patterns["uri"].search(r.text).group(1)
uri = uri.format(uri = "{}@{}".format(f.username, instance)) uri = uri.format(uri = "{}@{}".format(f.username, instance))
r = requests.get(uri, headers={"Accept": "application/json"}, timeout=10) r = requests.get(uri, headers={"Accept": "application/json"}, timeout=10)
j = r.json() j = r.json()
for link in j['links']: for link in j['links']:
if link['rel'] == 'self': if link['rel'] == 'self':
#this is a link formatted like "https://instan.ce/users/username", which is what we need #this is a link formatted like "https://instan.ce/users/username", which is what we need
uri = link['href'] uri = link['href']
uri = "{}/outbox?page=true".format(uri) uri = "{}/outbox?page=true".format(uri)
r = requests.get(uri, timeout=10) r = requests.get(uri, timeout=10)
j = r.json() j = r.json()
except Exception: except Exception:
print("oopsy woopsy!! we made a fucky wucky!!!\n(we're probably rate limited, please hang up and try again)") print("oopsy woopsy!! we made a fucky wucky!!!\n(we're probably rate limited, please hang up and try again)")
sys.exit(1) sys.exit(1)
pleroma = False pleroma = False
if 'first' in j and type(j['first']) != str: if 'first' in j and type(j['first']) != str:
print("Pleroma instance detected") print("Pleroma instance detected")
pleroma = True pleroma = True
j = j['first'] j = j['first']
else: else:
print("Mastodon/Misskey instance detected") print("Mastodon/Misskey instance detected")
uri = "{}&min_id={}".format(uri, last_toot) uri = "{}&min_id={}".format(uri, last_toot)
r = requests.get(uri) r = requests.get(uri)
j = r.json() j = r.json()
print("Downloading and saving toots", end='', flush=True) print("Downloading and saving toots", end='', flush=True)
done = False done = False
try: try:
while not done and len(j['orderedItems']) > 0: while not done and len(j['orderedItems']) > 0:
for oi in j['orderedItems']: for oi in j['orderedItems']:
if oi['type'] != "Create": if oi['type'] != "Create":
continue #this isn't a toot/post/status/whatever, it's a boost or a follow or some other activitypub thing. ignore continue #this isn't a toot/post/status/whatever, it's a boost or a follow or some other activitypub thing. ignore
# its a toost baby # its a toost baby
content = oi['object']['content'] content = oi['object']['content']
if oi['object']['summary'] != None: if oi['object']['summary'] != None:
#don't download CW'd toots. if you want your bot to download and learn from CW'd toots, replace "continue" with "pass". (todo: add a config.json option for this) #don't download CW'd toots. if you want your bot to download and learn from CW'd toots, replace "continue" with "pass". (todo: add a config.json option for this)
continue continue
toot = extract_toot(content) toot = extract_toot(content)
# print(toot) # print(toot)
try: try:
if pleroma: if pleroma:
if c.execute("SELECT COUNT(*) FROM toots WHERE id LIKE ?", (oi['object']['id'],)).fetchone()[0] > 0: if c.execute("SELECT COUNT(*) FROM toots WHERE id LIKE ?", (oi['object']['id'],)).fetchone()[0] > 0:
#we've caught up to the notices we've already downloaded, so we can stop now #we've caught up to the notices we've already downloaded, so we can stop now
#you might be wondering, "lynne, what if the instance ratelimits you after 40 posts, and they've made 60 since main.py was last run? wouldn't the bot miss 20 posts and never be able to see them?" to which i reply, "it's called mstdn-ebooks not fediverse-ebooks. pleroma support is an afterthought" #you might be wondering, "lynne, what if the instance ratelimits you after 40 posts, and they've made 60 since main.py was last run? wouldn't the bot miss 20 posts and never be able to see them?" to which i reply, "it's called mstdn-ebooks not fediverse-ebooks. pleroma support is an afterthought"
done = True done = True
break break
pid = re.search(r"[^\/]+$", oi['object']['id']).group(0) pid = patterns["pid"].search(oi['object']['id']).group(0)
c.execute("REPLACE INTO toots (id, userid, uri, content) VALUES (?, ?, ?, ?)", ( c.execute("REPLACE INTO toots (id, userid, uri, content) VALUES (?, ?, ?, ?)", (
pid, pid,
f.id, f.id,
oi['object']['id'], oi['object']['id'],
toot toot
) )
) )
pass pass
except: except:
pass #ignore any toots that don't successfully go into the DB pass #ignore any toots that don't successfully go into the DB
if not pleroma: if not pleroma:
r = requests.get(j['prev'], timeout=15) r = requests.get(j['prev'], timeout=15)
else: else:
r = requests.get(j['next'], timeout=15) r = requests.get(j['next'], timeout=15)
j = r.json() j = r.json()
print('.', end='', flush=True) print('.', end='', flush=True)
print(" Done!") print(" Done!")
db.commit() db.commit()
except: except:
print("Encountered an error! Saving toots to database and moving to next followed account.") print("Encountered an error! Saving toots to database and moving to next followed account.")
db.commit() db.commit()
print("Done!") print("Done!")