fix for pleroma

This commit is contained in:
Lynne 2018-10-27 18:28:20 +10:00
parent a1324acfba
commit eeba1c9066
1 changed files with 82 additions and 38 deletions

54
main.py
View File

@ -105,6 +105,26 @@ def handleCtrlC(signal, frame):
signal.signal(signal.SIGINT, handleCtrlC) signal.signal(signal.SIGINT, handleCtrlC)
def get_toots_legacy(client, id):
i = 0
toots = client.account_statuses(id)
while toots is not None and len(toots) > 0:
for toot in toots:
if toot.spoiler_text != "": continue
if toot.reblog is not None: continue
if toot.visibility not in ["public", "unlisted"]: continue
t = extract_toot(toot.content)
if t != None:
yield {
"toot": t,
"id": toot.id,
"uri": toot.uri
}
toots = client.fetch_next(toots)
i += 1
if i%20 == 0:
print('.', end='', flush=True)
for f in following: for f in following:
last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY id DESC LIMIT 1", (f.id,)).fetchone() last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY id DESC LIMIT 1", (f.id,)).fetchone()
if last_toot != None: if last_toot != None:
@ -114,7 +134,7 @@ for f in following:
print("Harvesting toots for user @{}, starting from {}".format(f.acct, last_toot)) print("Harvesting toots for user @{}, starting from {}".format(f.acct, last_toot))
#find the user's activitypub outbox #find the user's activitypub outbox
#print("WebFingering...") print("WebFingering...")
instance = re.search(r"^.*@(.+)", f.acct) instance = re.search(r"^.*@(.+)", f.acct)
if instance == None: if instance == None:
instance = re.search(r"https?:\/\/(.*)", cfg['site']).group(1) instance = re.search(r"https?:\/\/(.*)", cfg['site']).group(1)
@ -130,8 +150,12 @@ for f in following:
r = requests.get("https://{}/.well-known/host-meta".format(instance)) r = requests.get("https://{}/.well-known/host-meta".format(instance))
uri = re.search(r'template="([^"]+)"', r.text).group(1) uri = re.search(r'template="([^"]+)"', r.text).group(1)
uri = uri.format(uri = "{}@{}".format(f.username, instance)) uri = uri.format(uri = "{}@{}".format(f.username, instance))
r = requests.get(uri) r = requests.get(uri, headers={"Accept": "application/json"})
uri = r.json()['aliases'][1] #TODO: find out if it's safe to rely on this j = r.json()
if len(j['aliases']) == 1: #TODO: this is a hack on top of a hack, fix it
uri = j['aliases'][0]
else:
uri = j['aliases'][1]
uri = "{}/outbox?page=true&min_id={}".format(uri, last_toot) uri = "{}/outbox?page=true&min_id={}".format(uri, last_toot)
r = requests.get(uri) r = requests.get(uri)
j = r.json() j = r.json()
@ -139,12 +163,31 @@ for f in following:
print("oopsy woopsy!! we made a fucky wucky!!!\n(we're probably rate limited, please hang up and try again)") print("oopsy woopsy!! we made a fucky wucky!!!\n(we're probably rate limited, please hang up and try again)")
sys.exit(1) sys.exit(1)
pleroma = False
if 'first' in j:
print("{} is a pleroma instance -- falling back to legacy toot collection method".format(instance))
pleroma = True
print("Downloading and parsing toots", end='', flush=True) print("Downloading and parsing toots", end='', flush=True)
current = None current = None
try: try:
if pleroma:
for t in get_toots_legacy(client, f.id):
try:
c.execute("REPLACE INTO toots (id, userid, uri, content) VALUES (?, ?, ?, ?)",
(t['id'],
f.id,
t['uri'],
t['toot']
)
)
except:
pass
else:
while len(j['orderedItems']) > 0: while len(j['orderedItems']) > 0:
for oi in j['orderedItems']: for oi in j['orderedItems']:
if oi['type'] == "Create": if (not pleroma and oi['type'] == "Create") or (pleroma and oi['to']['type'] == "Create"):
# its a toost baby # its a toost baby
content = oi['object']['content'] content = oi['object']['content']
if oi['object']['summary'] != None: if oi['object']['summary'] != None:
@ -153,8 +196,9 @@ for f in following:
toot = extract_toot(content) toot = extract_toot(content)
# print(toot) # print(toot)
try: try:
pid = re.search(r"[^\/]+$", oi['object']['id']).group(0)
c.execute("REPLACE INTO toots (id, userid, uri, content) VALUES (?, ?, ?, ?)", c.execute("REPLACE INTO toots (id, userid, uri, content) VALUES (?, ?, ?, ?)",
(re.search(r"[^\/]+$", oi['object']['id']).group(0), (pid,
f.id, f.id,
oi['object']['id'], oi['object']['id'],
toot toot