From 984d44775bcd1331eb73e4a7812d768fd8c485bb Mon Sep 17 00:00:00 2001 From: Amolith Date: Sun, 29 Oct 2023 10:41:00 -0400 Subject: [PATCH] BREAKING: SQL schema change - Redo schema to improve handling of lightweight tags - Try to clean up empty directories when untracking a project To resolve schema conflict, run `sqlite3 willow.sqlite` and paste the following: ALTER TABLE releases RENAME TO releases_bak; CREATE TABLE releases ( id TEXT NOT NULL PRIMARY KEY, project_url TEXT NOT NULL, release_url TEXT NOT NULL, tag TEXT NOT NULL, content TEXT NOT NULL, date TIMESTAMP NOT NULL, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); If everything works as expected, you can `DROP TABLE releases_bak`. --- db/release.go | 14 ++++--- db/sql/schema.sql | 55 ++++++++++++++------------- git/git.go | 94 ++++++++++++++++++++++++++++++++++------------ project/project.go | 42 ++++++++++++++++----- ws/ws.go | 3 +- 5 files changed, 142 insertions(+), 66 deletions(-) diff --git a/db/release.go b/db/release.go index 86bf399..3166195 100644 --- a/db/release.go +++ b/db/release.go @@ -4,7 +4,9 @@ package db -import "database/sql" +import ( + "database/sql" +) // AddRelease adds a release for a project with a given URL to the database @@ -14,16 +16,16 @@ import "database/sql" // UpsertRelease adds or updates a release for a project with a given URL in the // database -func UpsertRelease(db *sql.DB, projectURL, releaseURL, tag, content, date string) error { - _, err := db.Exec(`INSERT INTO releases (project_url, release_url, tag, content, date) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(release_url) DO +func UpsertRelease(db *sql.DB, id, projectURL, releaseURL, tag, content, date string) error { + _, err := db.Exec(`INSERT INTO releases (id, project_url, release_url, tag, content, date) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET release_url = excluded.release_url, content = excluded.content, tag = excluded.tag, content = excluded.content, - date = excluded.date;`, projectURL, releaseURL, tag, content, date) + date = excluded.date;`, id, projectURL, releaseURL, tag, content, date) return err } diff --git a/db/sql/schema.sql b/db/sql/schema.sql index 30ae5d1..a98f64d 100644 --- a/db/sql/schema.sql +++ b/db/sql/schema.sql @@ -4,43 +4,44 @@ -- Create table of users with username, password hash, salt, and creation -- timestamp -CREATE TABLE users ( - username VARCHAR(255) NOT NULL, - hash VARCHAR(255) NOT NULL, - salt VARCHAR(255) NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (username) +CREATE TABLE users +( + username TEXT NOT NULL PRIMARY KEY, + hash TEXT NOT NULL, + salt TEXT NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); -- Create table of sessions with session GUID, username, and timestamp of when -- the session was created -CREATE TABLE sessions ( - token VARCHAR(255) NOT NULL, - username VARCHAR(255) NOT NULL, - expires TIMESTAMP NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (token) +CREATE TABLE sessions +( + token TEXT NOT NULL, + username TEXT NOT NULL, + expires TIMESTAMP NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); -- Create table of tracked projects with URL, name, forge, running version, and -- timestamp of when the project was added -CREATE TABLE projects ( - url VARCHAR(255) NOT NULL, - name VARCHAR(255) NOT NULL, - forge VARCHAR(255) NOT NULL, - version VARCHAR(255) NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (url) +CREATE TABLE projects +( + url TEXT NOT NULL, + name TEXT NOT NULL, + forge TEXT NOT NULL, + version TEXT NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); -- Create table of project releases with the project URL and the release tags, -- contents, URLs, and dates -CREATE TABLE releases ( - project_url VARCHAR(255) NOT NULL, - release_url VARCHAR(255) NOT NULL, - tag VARCHAR(255) NOT NULL, - content TEXT NOT NULL, - date TIMESTAMP NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (release_url) +CREATE TABLE releases +( + id TEXT NOT NULL PRIMARY KEY, + project_url TEXT NOT NULL, + release_url TEXT NOT NULL, + tag TEXT NOT NULL, + content TEXT NOT NULL, + date TIMESTAMP NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); \ No newline at end of file diff --git a/git/git.go b/git/git.go index 40ca6b9..892ba50 100644 --- a/git/git.go +++ b/git/git.go @@ -7,6 +7,7 @@ package git import ( "errors" "fmt" + "io" "net/url" "os" "sort" @@ -64,31 +65,39 @@ func GetReleases(gitURI, forge string) ([]Release, error) { releases := make([]Release, 0) err = tagRefs.ForEach(func(tagRef *plumbing.Reference) error { - obj, err := r.TagObject(tagRef.Hash()) - switch { - case errors.Is(err, plumbing.ErrObjectNotFound): - // This is a lightweight tag, not an annotated tag, skip it - return nil - case err == nil: - tagURL := "" - tagName := bmStrict.Sanitize(tagRef.Name().Short()) - switch forge { - case "sourcehut": - tagURL = "https://" + httpURI + "/refs/" + tagName - case "gitlab": - tagURL = "https://" + httpURI + "/-/releases/" + tagName - default: - tagURL = "" + tagObj, err := r.TagObject(tagRef.Hash()) + + var message string + var date time.Time + if errors.Is(err, plumbing.ErrObjectNotFound) { + commitTag, err := r.CommitObject(tagRef.Hash()) + if err != nil { + return err } - releases = append(releases, Release{ - Tag: tagName, - Content: bmUGC.Sanitize(obj.Message), - URL: tagURL, - Date: obj.Tagger.When, - }) - default: - return err + message = commitTag.Message + date = commitTag.Committer.When + } else { + message = tagObj.Message + date = tagObj.Tagger.When } + + tagURL := "" + tagName := bmStrict.Sanitize(tagRef.Name().Short()) + switch forge { + case "sourcehut": + tagURL = "https://" + httpURI + "/refs/" + tagName + case "gitlab": + tagURL = "https://" + httpURI + "/-/releases/" + tagName + default: + tagURL = "" + } + + releases = append(releases, Release{ + Tag: tagName, + Content: bmUGC.Sanitize(message), + URL: tagURL, + Date: date, + }) return nil }) if err != nil { @@ -139,9 +148,48 @@ func RemoveRepo(url string) (err error) { return err } err = os.RemoveAll(path) + if err != nil { + return err + } + + // TODO: Check whether the two parent directories are empty and remove them if + // so + for i := 0; i < 2; i++ { + path = strings.TrimSuffix(path, "/") + if path == "data" { + break + } + empty, err := dirEmpty(path) + if err != nil { + return err + } + if empty { + err = os.Remove(path) + if err != nil { + return err + } + } + path = path[:strings.LastIndex(path, "/")] + } + return err } +// dirEmpty checks if a directory is empty. +func dirEmpty(name string) (empty bool, err error) { + f, err := os.Open(name) + if err != nil { + return false, err + } + defer f.Close() + + _, err = f.Readdirnames(1) + if err == io.EOF { + return true, nil + } + return false, err +} + // stringifyRepo accepts a repository URI string and the corresponding local // filesystem path, whether the URI is HTTP, HTTPS, or SSH. func stringifyRepo(url string) (path string, err error) { diff --git a/project/project.go b/project/project.go index c040bf9..99fed90 100644 --- a/project/project.go +++ b/project/project.go @@ -5,6 +5,7 @@ package project import ( + "crypto/sha256" "database/sql" "fmt" "log" @@ -40,7 +41,7 @@ func GetReleases(dbConn *sql.DB, proj Project) (Project, error) { } if len(ret) == 0 { - return fetchReleases(proj) + return fetchReleases(dbConn, proj) } for _, row := range ret { @@ -58,7 +59,7 @@ func GetReleases(dbConn *sql.DB, proj Project) (Project, error) { } // fetchReleases fetches releases from a project's forge given its URI -func fetchReleases(p Project) (Project, error) { +func fetchReleases(dbConn *sql.DB, p Project) (Project, error) { var err error switch p.Forge { case "github", "gitea", "forgejo": @@ -74,6 +75,11 @@ func fetchReleases(p Project) (Project, error) { URL: release.URL, Date: release.Date, }) + err = upsert(dbConn, p.URL, p.Releases) + if err != nil { + log.Printf("Error upserting release: %v", err) + return p, err + } } default: gitReleases, err := git.GetReleases(p.URL, p.Forge) @@ -87,6 +93,11 @@ func fetchReleases(p Project) (Project, error) { URL: release.URL, Date: release.Date, }) + err = upsert(dbConn, p.URL, p.Releases) + if err != nil { + log.Printf("Error upserting release: %v", err) + return p, err + } } } sort.Slice(p.Releases, func(i, j int) bool { @@ -95,6 +106,21 @@ func fetchReleases(p Project) (Project, error) { return p, err } +// upsert updates or inserts a project release into the database +func upsert(dbConn *sql.DB, url string, releases []Release) error { + for _, release := range releases { + date := release.Date.Format("2006-01-02 15:04:05") + idByte := sha256.Sum256([]byte(url + release.URL + release.Tag + date)) + id := fmt.Sprintf("%x", idByte) + err := db.UpsertRelease(dbConn, id, url, release.URL, release.Tag, release.Content, date) + if err != nil { + log.Printf("Error upserting release: %v", err) + return err + } + } + return nil +} + func Track(dbConn *sql.DB, manualRefresh *chan struct{}, name, url, forge, release string) { err := db.UpsertProject(dbConn, url, name, forge, release) if err != nil { @@ -126,7 +152,7 @@ func RefreshLoop(dbConn *sql.DB, interval int, manualRefresh, req *chan struct{} fmt.Println("Error getting projects:", err) } for i, p := range projectsList { - p, err := fetchReleases(p) + p, err := fetchReleases(dbConn, p) if err != nil { fmt.Println(err) continue @@ -137,12 +163,10 @@ func RefreshLoop(dbConn *sql.DB, interval int, manualRefresh, req *chan struct{} return strings.ToLower(projectsList[i].Name) < strings.ToLower(projectsList[j].Name) }) for i := range projectsList { - for j := range projectsList[i].Releases { - err = db.UpsertRelease(dbConn, projectsList[i].URL, projectsList[i].Releases[j].URL, projectsList[i].Releases[j].Tag, projectsList[i].Releases[j].Content, projectsList[i].Releases[j].Date.Format("2006-01-02 15:04:05")) - if err != nil { - fmt.Println("Error upserting release:", err) - continue - } + err = upsert(dbConn, projectsList[i].URL, projectsList[i].Releases) + if err != nil { + fmt.Println("Error upserting release:", err) + continue } } return projectsList diff --git a/ws/ws.go b/ws/ws.go index 8213129..de755ce 100644 --- a/ws/ws.go +++ b/ws/ws.go @@ -8,7 +8,6 @@ import ( "database/sql" "embed" "fmt" - "git.sr.ht/~amolith/willow/users" "io" "net/http" "net/url" @@ -17,6 +16,8 @@ import ( "text/template" "time" + "git.sr.ht/~amolith/willow/users" + "git.sr.ht/~amolith/willow/project" "github.com/microcosm-cc/bluemonday" )