okay this is a big one. fixes #27.

- fixed: compatibility with snowflake IDs
- fixed: no longer assume userids will be unique across instances
- fixed: don't use post ID for sorting
- fixed: we now store userid and id as VARCHAR instead of INT
- new: print friendly message if access token is invalid
- lynne: relieved to be done with this

i tested this with a bot that follows mastodon users. if any of the people your ebooks bot is following (and therefore learning from) use pleroma, you need to delete toots.db.
This commit is contained in:
Lynne 2019-08-15 11:56:27 +10:00
parent 6ae8586500
commit 8eb2750d86
1 changed files with 43 additions and 20 deletions

55
main.py
View File

@ -4,7 +4,7 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this # License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/. # file, You can obtain one at http://mozilla.org/MPL/2.0/.
from mastodon import Mastodon from mastodon import Mastodon, MastodonUnauthorizedError
from os import path from os import path
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
import os, sqlite3, signal, sys, json, re, shutil, argparse import os, sqlite3, signal, sys, json, re, shutil, argparse
@ -71,31 +71,54 @@ client = Mastodon(
access_token=cfg['secret'], access_token=cfg['secret'],
api_base_url=cfg['site']) api_base_url=cfg['site'])
me = client.account_verify_credentials() try:
me = client.account_verify_credentials()
except MastodonUnauthorizedError:
print("The provided access token in {} is invalid. Please delete {} and run main.py again.".format(args.cfg, args.cfg))
sys.exit(1)
following = client.account_following(me.id) following = client.account_following(me.id)
db = sqlite3.connect("toots.db") db = sqlite3.connect("toots.db")
db.text_factory=str db.text_factory=str
c = db.cursor() c = db.cursor()
c.execute("CREATE TABLE IF NOT EXISTS `toots` (sortid INT NOT NULL DEFAULT 0, id VARCHAR NOT NULL UNIQUE PRIMARY KEY, cw INT NOT NULL DEFAULT 0, userid VARCHAR NOT NULL, uri VARCHAR NOT NULL, content VARCHAR NOT NULL) WITHOUT ROWID") c.execute("CREATE TABLE IF NOT EXISTS `toots_temp` (sortid INTEGER UNIQUE PRIMARY KEY AUTOINCREMENT, id VARCHAR NOT NULL, cw INT NOT NULL DEFAULT 0, userid VARCHAR NOT NULL, uri VARCHAR NOT NULL, content VARCHAR NOT NULL)")
try:
c.execute("ALTER TABLE `toots` ADD COLUMN sortid INT NOT NULL DEFAULT 0") tableinfo = c.execute("PRAGMA table_info(`toots`)").fetchall()
found = False
columns = []
for entry in tableinfo:
if entry[1] == "sortid":
found = True
break
columns.append(entry[1])
if not found:
print("Migrating to new database format. Please wait...")
print("WARNING: If any of the accounts your bot is following are Pleroma users, please delete toots.db and run main.py again to create it anew.")
try:
c.execute("DROP TABLE `toots_temp`")
except:
pass
c.execute("CREATE TABLE `toots_temp` (sortid INTEGER UNIQUE PRIMARY KEY AUTOINCREMENT, id VARCHAR NOT NULL, cw INT NOT NULL DEFAULT 0, userid VARCHAR NOT NULL, uri VARCHAR NOT NULL, content VARCHAR NOT NULL)")
for f in following: for f in following:
last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY id DESC LIMIT 1", (f.id,)).fetchone() user_toots = c.execute("SELECT * FROM `toots` WHERE userid LIKE ? ORDER BY id", (f.id,)).fetchall()
if last_toot != None: if user_toots == None:
last_toot = last_toot[0] continue
if columns[-1] == "cw":
for toot in user_toots:
c.execute("INSERT INTO `toots_temp` (id, userid, uri, content, cw) VALUES (?, ?, ?, ?, ?)", toot)
else: else:
last_toot = 0 for toot in user_toots:
c.execute("INSERT INTO `toots_temp` (id, cw, userid, uri, content) VALUES (?, ?, ?, ?, ?)", toot)
c.execute("UPDATE `toots` SET sortid = 1 WHERE id LIKE ? AND userid LIKE ?", (last_toot, f.id)) c.execute("DROP TABLE `toots`")
c.execute("ALTER TABLE `toots_temp` RENAME TO `toots`")
except:
pass # column already exists
db.commit() db.commit()
sys.exit(0)
def handleCtrlC(signal, frame): def handleCtrlC(signal, frame):
print("\nPREMATURE EVACUATION - Saving chunks") print("\nPREMATURE EVACUATION - Saving chunks")
db.commit() db.commit()
@ -123,7 +146,7 @@ def insert_toot(oii, acc, post, cursor): # extracted to prevent duplication
for f in following: for f in following:
last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY id DESC LIMIT 1", (f.id,)).fetchone() last_toot = c.execute("SELECT id FROM `toots` WHERE userid LIKE ? ORDER BY sortid DESC LIMIT 1", (f.id,)).fetchone()
if last_toot != None: if last_toot != None:
last_toot = last_toot[0] last_toot = last_toot[0]
else: else: