BREAKING: SQL schema change

- Redo schema to improve handling of lightweight tags
- Try to clean up empty directories when untracking a project

To resolve schema conflict, run `sqlite3 willow.sqlite` and paste the
following:

ALTER TABLE releases RENAME TO releases_bak;
CREATE TABLE releases
(
    id          TEXT      NOT NULL PRIMARY KEY,
    project_url TEXT      NOT NULL,
    release_url TEXT      NOT NULL,
    tag         TEXT      NOT NULL,
    content     TEXT      NOT NULL,
    date        TIMESTAMP NOT NULL,
    created_at  TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);

If everything works as expected, you can `DROP TABLE releases_bak`.
This commit is contained in:
Amolith 2023-10-29 10:41:00 -04:00
parent 3612ffc595
commit 984d44775b
Signed by: Amolith
GPG Key ID: 8AE30347CE28D101
5 changed files with 142 additions and 66 deletions

View File

@ -4,7 +4,9 @@
package db package db
import "database/sql" import (
"database/sql"
)
// AddRelease adds a release for a project with a given URL to the database // AddRelease adds a release for a project with a given URL to the database
@ -14,16 +16,16 @@ import "database/sql"
// UpsertRelease adds or updates a release for a project with a given URL in the // UpsertRelease adds or updates a release for a project with a given URL in the
// database // database
func UpsertRelease(db *sql.DB, projectURL, releaseURL, tag, content, date string) error { func UpsertRelease(db *sql.DB, id, projectURL, releaseURL, tag, content, date string) error {
_, err := db.Exec(`INSERT INTO releases (project_url, release_url, tag, content, date) _, err := db.Exec(`INSERT INTO releases (id, project_url, release_url, tag, content, date)
VALUES (?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(release_url) DO ON CONFLICT(id) DO
UPDATE SET UPDATE SET
release_url = excluded.release_url, release_url = excluded.release_url,
content = excluded.content, content = excluded.content,
tag = excluded.tag, tag = excluded.tag,
content = excluded.content, content = excluded.content,
date = excluded.date;`, projectURL, releaseURL, tag, content, date) date = excluded.date;`, id, projectURL, releaseURL, tag, content, date)
return err return err
} }

View File

@ -4,43 +4,44 @@
-- Create table of users with username, password hash, salt, and creation -- Create table of users with username, password hash, salt, and creation
-- timestamp -- timestamp
CREATE TABLE users ( CREATE TABLE users
username VARCHAR(255) NOT NULL, (
hash VARCHAR(255) NOT NULL, username TEXT NOT NULL PRIMARY KEY,
salt VARCHAR(255) NOT NULL, hash TEXT NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, salt TEXT NOT NULL,
PRIMARY KEY (username) created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
); );
-- Create table of sessions with session GUID, username, and timestamp of when -- Create table of sessions with session GUID, username, and timestamp of when
-- the session was created -- the session was created
CREATE TABLE sessions ( CREATE TABLE sessions
token VARCHAR(255) NOT NULL, (
username VARCHAR(255) NOT NULL, token TEXT NOT NULL,
expires TIMESTAMP NOT NULL, username TEXT NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, expires TIMESTAMP NOT NULL,
PRIMARY KEY (token) created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
); );
-- Create table of tracked projects with URL, name, forge, running version, and -- Create table of tracked projects with URL, name, forge, running version, and
-- timestamp of when the project was added -- timestamp of when the project was added
CREATE TABLE projects ( CREATE TABLE projects
url VARCHAR(255) NOT NULL, (
name VARCHAR(255) NOT NULL, url TEXT NOT NULL,
forge VARCHAR(255) NOT NULL, name TEXT NOT NULL,
version VARCHAR(255) NOT NULL, forge TEXT NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, version TEXT NOT NULL,
PRIMARY KEY (url) created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
); );
-- Create table of project releases with the project URL and the release tags, -- Create table of project releases with the project URL and the release tags,
-- contents, URLs, and dates -- contents, URLs, and dates
CREATE TABLE releases ( CREATE TABLE releases
project_url VARCHAR(255) NOT NULL, (
release_url VARCHAR(255) NOT NULL, id TEXT NOT NULL PRIMARY KEY,
tag VARCHAR(255) NOT NULL, project_url TEXT NOT NULL,
content TEXT NOT NULL, release_url TEXT NOT NULL,
date TIMESTAMP NOT NULL, tag TEXT NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, content TEXT NOT NULL,
PRIMARY KEY (release_url) date TIMESTAMP NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
); );

View File

@ -7,6 +7,7 @@ package git
import ( import (
"errors" "errors"
"fmt" "fmt"
"io"
"net/url" "net/url"
"os" "os"
"sort" "sort"
@ -64,31 +65,39 @@ func GetReleases(gitURI, forge string) ([]Release, error) {
releases := make([]Release, 0) releases := make([]Release, 0)
err = tagRefs.ForEach(func(tagRef *plumbing.Reference) error { err = tagRefs.ForEach(func(tagRef *plumbing.Reference) error {
obj, err := r.TagObject(tagRef.Hash()) tagObj, err := r.TagObject(tagRef.Hash())
switch {
case errors.Is(err, plumbing.ErrObjectNotFound): var message string
// This is a lightweight tag, not an annotated tag, skip it var date time.Time
return nil if errors.Is(err, plumbing.ErrObjectNotFound) {
case err == nil: commitTag, err := r.CommitObject(tagRef.Hash())
tagURL := "" if err != nil {
tagName := bmStrict.Sanitize(tagRef.Name().Short()) return err
switch forge {
case "sourcehut":
tagURL = "https://" + httpURI + "/refs/" + tagName
case "gitlab":
tagURL = "https://" + httpURI + "/-/releases/" + tagName
default:
tagURL = ""
} }
releases = append(releases, Release{ message = commitTag.Message
Tag: tagName, date = commitTag.Committer.When
Content: bmUGC.Sanitize(obj.Message), } else {
URL: tagURL, message = tagObj.Message
Date: obj.Tagger.When, date = tagObj.Tagger.When
})
default:
return err
} }
tagURL := ""
tagName := bmStrict.Sanitize(tagRef.Name().Short())
switch forge {
case "sourcehut":
tagURL = "https://" + httpURI + "/refs/" + tagName
case "gitlab":
tagURL = "https://" + httpURI + "/-/releases/" + tagName
default:
tagURL = ""
}
releases = append(releases, Release{
Tag: tagName,
Content: bmUGC.Sanitize(message),
URL: tagURL,
Date: date,
})
return nil return nil
}) })
if err != nil { if err != nil {
@ -139,9 +148,48 @@ func RemoveRepo(url string) (err error) {
return err return err
} }
err = os.RemoveAll(path) err = os.RemoveAll(path)
if err != nil {
return err
}
// TODO: Check whether the two parent directories are empty and remove them if
// so
for i := 0; i < 2; i++ {
path = strings.TrimSuffix(path, "/")
if path == "data" {
break
}
empty, err := dirEmpty(path)
if err != nil {
return err
}
if empty {
err = os.Remove(path)
if err != nil {
return err
}
}
path = path[:strings.LastIndex(path, "/")]
}
return err return err
} }
// dirEmpty checks if a directory is empty.
func dirEmpty(name string) (empty bool, err error) {
f, err := os.Open(name)
if err != nil {
return false, err
}
defer f.Close()
_, err = f.Readdirnames(1)
if err == io.EOF {
return true, nil
}
return false, err
}
// stringifyRepo accepts a repository URI string and the corresponding local // stringifyRepo accepts a repository URI string and the corresponding local
// filesystem path, whether the URI is HTTP, HTTPS, or SSH. // filesystem path, whether the URI is HTTP, HTTPS, or SSH.
func stringifyRepo(url string) (path string, err error) { func stringifyRepo(url string) (path string, err error) {

View File

@ -5,6 +5,7 @@
package project package project
import ( import (
"crypto/sha256"
"database/sql" "database/sql"
"fmt" "fmt"
"log" "log"
@ -40,7 +41,7 @@ func GetReleases(dbConn *sql.DB, proj Project) (Project, error) {
} }
if len(ret) == 0 { if len(ret) == 0 {
return fetchReleases(proj) return fetchReleases(dbConn, proj)
} }
for _, row := range ret { for _, row := range ret {
@ -58,7 +59,7 @@ func GetReleases(dbConn *sql.DB, proj Project) (Project, error) {
} }
// fetchReleases fetches releases from a project's forge given its URI // fetchReleases fetches releases from a project's forge given its URI
func fetchReleases(p Project) (Project, error) { func fetchReleases(dbConn *sql.DB, p Project) (Project, error) {
var err error var err error
switch p.Forge { switch p.Forge {
case "github", "gitea", "forgejo": case "github", "gitea", "forgejo":
@ -74,6 +75,11 @@ func fetchReleases(p Project) (Project, error) {
URL: release.URL, URL: release.URL,
Date: release.Date, Date: release.Date,
}) })
err = upsert(dbConn, p.URL, p.Releases)
if err != nil {
log.Printf("Error upserting release: %v", err)
return p, err
}
} }
default: default:
gitReleases, err := git.GetReleases(p.URL, p.Forge) gitReleases, err := git.GetReleases(p.URL, p.Forge)
@ -87,6 +93,11 @@ func fetchReleases(p Project) (Project, error) {
URL: release.URL, URL: release.URL,
Date: release.Date, Date: release.Date,
}) })
err = upsert(dbConn, p.URL, p.Releases)
if err != nil {
log.Printf("Error upserting release: %v", err)
return p, err
}
} }
} }
sort.Slice(p.Releases, func(i, j int) bool { sort.Slice(p.Releases, func(i, j int) bool {
@ -95,6 +106,21 @@ func fetchReleases(p Project) (Project, error) {
return p, err return p, err
} }
// upsert updates or inserts a project release into the database
func upsert(dbConn *sql.DB, url string, releases []Release) error {
for _, release := range releases {
date := release.Date.Format("2006-01-02 15:04:05")
idByte := sha256.Sum256([]byte(url + release.URL + release.Tag + date))
id := fmt.Sprintf("%x", idByte)
err := db.UpsertRelease(dbConn, id, url, release.URL, release.Tag, release.Content, date)
if err != nil {
log.Printf("Error upserting release: %v", err)
return err
}
}
return nil
}
func Track(dbConn *sql.DB, manualRefresh *chan struct{}, name, url, forge, release string) { func Track(dbConn *sql.DB, manualRefresh *chan struct{}, name, url, forge, release string) {
err := db.UpsertProject(dbConn, url, name, forge, release) err := db.UpsertProject(dbConn, url, name, forge, release)
if err != nil { if err != nil {
@ -126,7 +152,7 @@ func RefreshLoop(dbConn *sql.DB, interval int, manualRefresh, req *chan struct{}
fmt.Println("Error getting projects:", err) fmt.Println("Error getting projects:", err)
} }
for i, p := range projectsList { for i, p := range projectsList {
p, err := fetchReleases(p) p, err := fetchReleases(dbConn, p)
if err != nil { if err != nil {
fmt.Println(err) fmt.Println(err)
continue continue
@ -137,12 +163,10 @@ func RefreshLoop(dbConn *sql.DB, interval int, manualRefresh, req *chan struct{}
return strings.ToLower(projectsList[i].Name) < strings.ToLower(projectsList[j].Name) return strings.ToLower(projectsList[i].Name) < strings.ToLower(projectsList[j].Name)
}) })
for i := range projectsList { for i := range projectsList {
for j := range projectsList[i].Releases { err = upsert(dbConn, projectsList[i].URL, projectsList[i].Releases)
err = db.UpsertRelease(dbConn, projectsList[i].URL, projectsList[i].Releases[j].URL, projectsList[i].Releases[j].Tag, projectsList[i].Releases[j].Content, projectsList[i].Releases[j].Date.Format("2006-01-02 15:04:05")) if err != nil {
if err != nil { fmt.Println("Error upserting release:", err)
fmt.Println("Error upserting release:", err) continue
continue
}
} }
} }
return projectsList return projectsList

View File

@ -8,7 +8,6 @@ import (
"database/sql" "database/sql"
"embed" "embed"
"fmt" "fmt"
"git.sr.ht/~amolith/willow/users"
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
@ -17,6 +16,8 @@ import (
"text/template" "text/template"
"time" "time"
"git.sr.ht/~amolith/willow/users"
"git.sr.ht/~amolith/willow/project" "git.sr.ht/~amolith/willow/project"
"github.com/microcosm-cc/bluemonday" "github.com/microcosm-cc/bluemonday"
) )