amber
/
amber-ebooks-archived
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update extract code to match fedibooks

This commit is contained in:
Lynne 2020-05-27 22:31:16 +10:00
parent 2321f75e11
commit 8274409bf4
No known key found for this signature in database
GPG Key ID: F0A184B5213D9F90
1 changed files with 15 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
functions.py
View File

@ -68,22 +68,21 @@ def extract_toot(toot):
toot = html.unescape(toot) # convert HTML escape codes to text toot = html.unescape(toot) # convert HTML escape codes to text
soup = BeautifulSoup(toot, "html.parser") soup = BeautifulSoup(toot, "html.parser")
for lb in soup.select("br"): # replace <br> with linebreak for lb in soup.select("br"): # replace <br> with linebreak
lb.insert_after("\n") lb.replace_with("\n")
lb.decompose()
for p in soup.select("p"): # ditto for <p> for p in soup.select("p"): # ditto for <p>
p.insert_after("\n") p.replace_with("\n")
p.unwrap()
for ht in soup.select("a.hashtag"): #make hashtags no longer links, just text for ht in soup.select("a.hashtag"): # convert hashtags from links to text
ht.unwrap() ht.unwrap()
for link in soup.select("a"): #convert <a href='https://example.com>example.com</a> to just https://example.com for link in soup.select("a"): #ocnvert <a href='https://example.com>example.com</a> to just https://example.com
link.insert_after(link["href"]) if 'href' in link:
link.decompose() # apparently not all a tags have a href, which is understandable if you're doing normal web stuff, but on a social media platform??
link.replace_with(link["href"])
text = soup.get_text() text = soup.get_text()
text = re.sub("https://([^/]+)/(@[^ ]+)", r"\2@\1", text) #put mastodon-style mentions back in text = re.sub(r"https://([^/]+)/(@[^\s]+)", r"\2@\1", text) # put mastodon-style mentions back in
text = re.sub("https://([^/]+)/users/([^ ]+)", r"@\2@\1", text) #put pleroma-style mentions back in text = re.sub(r"https://([^/]+)/users/([^\s/]+)", r"@\2@\1", text) # put pleroma-style mentions back in
text = text.rstrip("\n") #remove trailing newline text = text.rstrip("\n") # remove trailing newline(s)
return text return text