partially fixed misskey scraping. Stolen from Grumbulon's fedi-books fork, commit e58c24bf0a

.gitignore

@@ -11,7 +11,3 @@ __pycache__/
 .editorconfig
 .*.swp
 config.json
-config/
-data/
-config
-data

functions.py

@@ -37,7 +37,7 @@ def make_sentence(output, cfg):
 	sentence = None
 	tries = 0
 	while sentence is None and tries < 10:
-		sentence = model.make_short_sentence(500, tries=10000)
+		sentence = model.make_short_sentence(500, 200, tries=10000)
 		tries = tries + 1
 
 	# optionally remove mentions
@@ -68,10 +68,10 @@ def extract_toot(toot):
 	toot = html.unescape(toot) # convert HTML escape codes to text
 	soup = BeautifulSoup(toot, "html.parser")
 	for lb in soup.select("br"): # replace <br> with linebreak
-		lb.replace_with("\n")
+		lb.name = "\n"
 
 	for p in soup.select("p"): # ditto for <p>
-		p.replace_with("\n")
+		lb.name = "\n"
 
 	for ht in soup.select("a.hashtag"): # convert hashtags from links to text
 		ht.unwrap()