BREAKING: SQL schema change

- Redo schema to improve handling of lightweight tags
- Try to clean up empty directories when untracking a project

To resolve schema conflict, run `sqlite3 willow.sqlite` and paste the
following:

ALTER TABLE releases RENAME TO releases_bak;
CREATE TABLE releases
(
    id          TEXT      NOT NULL PRIMARY KEY,
    project_url TEXT      NOT NULL,
    release_url TEXT      NOT NULL,
    tag         TEXT      NOT NULL,
    content     TEXT      NOT NULL,
    date        TIMESTAMP NOT NULL,
    created_at  TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);

If everything works as expected, you can `DROP TABLE releases_bak`.

db/release.go

@@ -4,7 +4,9 @@
 
 package db
 
-import "database/sql"
+import (
+	"database/sql"
+)
 
 // AddRelease adds a release for a project with a given URL to the database
 
@@ -14,16 +16,16 @@ import "database/sql"
 
 // UpsertRelease adds or updates a release for a project with a given URL in the
 // database
-func UpsertRelease(db *sql.DB, projectURL, releaseURL, tag, content, date string) error {
+func UpsertRelease(db *sql.DB, id, projectURL, releaseURL, tag, content, date string) error {
-	_, err := db.Exec(`INSERT INTO releases (project_url, release_url, tag, content, date)
+	_, err := db.Exec(`INSERT INTO releases (id, project_url, release_url, tag, content, date)
-		VALUES (?, ?, ?, ?, ?)
+		VALUES (?, ?, ?, ?, ?, ?)
-		ON CONFLICT(release_url) DO 
+		ON CONFLICT(id) DO 
 			UPDATE SET
 				release_url = excluded.release_url,
 				content = excluded.content,
 				tag = excluded.tag,
 				content = excluded.content,
-				date = excluded.date;`, projectURL, releaseURL, tag, content, date)
+				date = excluded.date;`, id, projectURL, releaseURL, tag, content, date)
 	return err
 }
 

db/sql/schema.sql

@@ -4,43 +4,44 @@
 
 -- Create table of users with username, password hash, salt, and creation
 -- timestamp
-CREATE TABLE users (
+CREATE TABLE users
-    username       VARCHAR(255)  NOT         NULL,
+(
-    hash           VARCHAR(255)  NOT         NULL,
+    username   TEXT      NOT NULL PRIMARY KEY,
-    salt           VARCHAR(255)  NOT         NULL,
+    hash       TEXT      NOT NULL,
-    created_at     TIMESTAMP     NOT         NULL   DEFAULT  CURRENT_TIMESTAMP,
+    salt       TEXT      NOT NULL,
-    PRIMARY        KEY           (username)
+    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
 );
 
 -- Create table of sessions with session GUID, username, and timestamp of when
 -- the session was created
-CREATE TABLE sessions (
+CREATE TABLE sessions
-    token        VARCHAR(255)  NOT          NULL,
+(
-    username     VARCHAR(255)  NOT          NULL,
+    token      TEXT      NOT NULL,
-    expires      TIMESTAMP     NOT          NULL,
+    username   TEXT      NOT NULL,
-    created_at   TIMESTAMP     NOT          NULL   DEFAULT  CURRENT_TIMESTAMP,
+    expires    TIMESTAMP NOT NULL,
-    PRIMARY      KEY           (token)
+    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
 );
 
 -- Create table of tracked projects with URL, name, forge, running version, and
 -- timestamp of when the project was added
-CREATE TABLE projects (
+CREATE TABLE projects
-    url         VARCHAR(255)  NOT    NULL,
+(
-    name        VARCHAR(255)  NOT    NULL,
+    url        TEXT      NOT NULL,
-    forge       VARCHAR(255)  NOT    NULL,
+    name       TEXT      NOT NULL,
-    version     VARCHAR(255)  NOT    NULL,
+    forge      TEXT      NOT NULL,
-    created_at  TIMESTAMP     NOT    NULL   DEFAULT  CURRENT_TIMESTAMP,
+    version    TEXT      NOT NULL,
-    PRIMARY     KEY           (url)
+    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
 );
 
 -- Create table of project releases with the project URL and the release tags,
 -- contents, URLs, and dates
-CREATE TABLE releases (
+CREATE TABLE releases
-    project_url  VARCHAR(255)  NOT    NULL,
+(
-    release_url  VARCHAR(255)  NOT    NULL,
+    id          TEXT      NOT NULL PRIMARY KEY,
-    tag          VARCHAR(255)  NOT    NULL,
+    project_url TEXT      NOT NULL,
-    content      TEXT          NOT    NULL,
+    release_url TEXT      NOT NULL,
-    date         TIMESTAMP     NOT    NULL,
+    tag         TEXT      NOT NULL,
-    created_at   TIMESTAMP     NOT    NULL   DEFAULT  CURRENT_TIMESTAMP,
+    content     TEXT      NOT NULL,
-    PRIMARY      KEY           (release_url)
+    date        TIMESTAMP NOT NULL,
+    created_at  TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
 );

git/git.go

@@ -7,6 +7,7 @@ package git
 import (
 	"errors"
 	"fmt"
+	"io"
 	"net/url"
 	"os"
 	"sort"
@@ -64,31 +65,39 @@ func GetReleases(gitURI, forge string) ([]Release, error) {
 	releases := make([]Release, 0)
 
 	err = tagRefs.ForEach(func(tagRef *plumbing.Reference) error {
-		obj, err := r.TagObject(tagRef.Hash())
+		tagObj, err := r.TagObject(tagRef.Hash())
-		switch {
+
-		case errors.Is(err, plumbing.ErrObjectNotFound):
+		var message string
-			// This is a lightweight tag, not an annotated tag, skip it
+		var date time.Time
-			return nil
+		if errors.Is(err, plumbing.ErrObjectNotFound) {
-		case err == nil:
+			commitTag, err := r.CommitObject(tagRef.Hash())
-			tagURL := ""
+			if err != nil {
-			tagName := bmStrict.Sanitize(tagRef.Name().Short())
+				return err
-			switch forge {
-			case "sourcehut":
-				tagURL = "https://" + httpURI + "/refs/" + tagName
-			case "gitlab":
-				tagURL = "https://" + httpURI + "/-/releases/" + tagName
-			default:
-				tagURL = ""
 			}
-			releases = append(releases, Release{
+			message = commitTag.Message
-				Tag:     tagName,
+			date = commitTag.Committer.When
-				Content: bmUGC.Sanitize(obj.Message),
+		} else {
-				URL:     tagURL,
+			message = tagObj.Message
-				Date:    obj.Tagger.When,
+			date = tagObj.Tagger.When
-			})
-		default:
-			return err
 		}
+
+		tagURL := ""
+		tagName := bmStrict.Sanitize(tagRef.Name().Short())
+		switch forge {
+		case "sourcehut":
+			tagURL = "https://" + httpURI + "/refs/" + tagName
+		case "gitlab":
+			tagURL = "https://" + httpURI + "/-/releases/" + tagName
+		default:
+			tagURL = ""
+		}
+
+		releases = append(releases, Release{
+			Tag:     tagName,
+			Content: bmUGC.Sanitize(message),
+			URL:     tagURL,
+			Date:    date,
+		})
 		return nil
 	})
 	if err != nil {
@@ -139,9 +148,48 @@ func RemoveRepo(url string) (err error) {
 		return err
 	}
 	err = os.RemoveAll(path)
+	if err != nil {
+		return err
+	}
+
+	// TODO: Check whether the two parent directories are empty and remove them if
+	// so
+	for i := 0; i < 2; i++ {
+		path = strings.TrimSuffix(path, "/")
+		if path == "data" {
+			break
+		}
+		empty, err := dirEmpty(path)
+		if err != nil {
+			return err
+		}
+		if empty {
+			err = os.Remove(path)
+			if err != nil {
+				return err
+			}
+		}
+		path = path[:strings.LastIndex(path, "/")]
+	}
+
 	return err
 }
 
+// dirEmpty checks if a directory is empty.
+func dirEmpty(name string) (empty bool, err error) {
+	f, err := os.Open(name)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	_, err = f.Readdirnames(1)
+	if err == io.EOF {
+		return true, nil
+	}
+	return false, err
+}
+
 // stringifyRepo accepts a repository URI string and the corresponding local
 // filesystem path, whether the URI is HTTP, HTTPS, or SSH.
 func stringifyRepo(url string) (path string, err error) {

project/project.go

@@ -5,6 +5,7 @@
 package project
 
 import (
+	"crypto/sha256"
 	"database/sql"
 	"fmt"
 	"log"
@@ -40,7 +41,7 @@ func GetReleases(dbConn *sql.DB, proj Project) (Project, error) {
 	}
 
 	if len(ret) == 0 {
-		return fetchReleases(proj)
+		return fetchReleases(dbConn, proj)
 	}
 
 	for _, row := range ret {
@@ -58,7 +59,7 @@ func GetReleases(dbConn *sql.DB, proj Project) (Project, error) {
 }
 
 // fetchReleases fetches releases from a project's forge given its URI
-func fetchReleases(p Project) (Project, error) {
+func fetchReleases(dbConn *sql.DB, p Project) (Project, error) {
 	var err error
 	switch p.Forge {
 	case "github", "gitea", "forgejo":
@@ -74,6 +75,11 @@ func fetchReleases(p Project) (Project, error) {
 				URL:     release.URL,
 				Date:    release.Date,
 			})
+			err = upsert(dbConn, p.URL, p.Releases)
+			if err != nil {
+				log.Printf("Error upserting release: %v", err)
+				return p, err
+			}
 		}
 	default:
 		gitReleases, err := git.GetReleases(p.URL, p.Forge)
@@ -87,6 +93,11 @@ func fetchReleases(p Project) (Project, error) {
 				URL:     release.URL,
 				Date:    release.Date,
 			})
+			err = upsert(dbConn, p.URL, p.Releases)
+			if err != nil {
+				log.Printf("Error upserting release: %v", err)
+				return p, err
+			}
 		}
 	}
 	sort.Slice(p.Releases, func(i, j int) bool {
@@ -95,6 +106,21 @@ func fetchReleases(p Project) (Project, error) {
 	return p, err
 }
 
+// upsert updates or inserts a project release into the database
+func upsert(dbConn *sql.DB, url string, releases []Release) error {
+	for _, release := range releases {
+		date := release.Date.Format("2006-01-02 15:04:05")
+		idByte := sha256.Sum256([]byte(url + release.URL + release.Tag + date))
+		id := fmt.Sprintf("%x", idByte)
+		err := db.UpsertRelease(dbConn, id, url, release.URL, release.Tag, release.Content, date)
+		if err != nil {
+			log.Printf("Error upserting release: %v", err)
+			return err
+		}
+	}
+	return nil
+}
+
 func Track(dbConn *sql.DB, manualRefresh *chan struct{}, name, url, forge, release string) {
 	err := db.UpsertProject(dbConn, url, name, forge, release)
 	if err != nil {
@@ -126,7 +152,7 @@ func RefreshLoop(dbConn *sql.DB, interval int, manualRefresh, req *chan struct{}
 			fmt.Println("Error getting projects:", err)
 		}
 		for i, p := range projectsList {
-			p, err := fetchReleases(p)
+			p, err := fetchReleases(dbConn, p)
 			if err != nil {
 				fmt.Println(err)
 				continue
@@ -137,12 +163,10 @@ func RefreshLoop(dbConn *sql.DB, interval int, manualRefresh, req *chan struct{}
 			return strings.ToLower(projectsList[i].Name) < strings.ToLower(projectsList[j].Name)
 		})
 		for i := range projectsList {
-			for j := range projectsList[i].Releases {
+			err = upsert(dbConn, projectsList[i].URL, projectsList[i].Releases)
-				err = db.UpsertRelease(dbConn, projectsList[i].URL, projectsList[i].Releases[j].URL, projectsList[i].Releases[j].Tag, projectsList[i].Releases[j].Content, projectsList[i].Releases[j].Date.Format("2006-01-02 15:04:05"))
+			if err != nil {
-				if err != nil {
+				fmt.Println("Error upserting release:", err)
-					fmt.Println("Error upserting release:", err)
+				continue
-					continue
-				}
 			}
 		}
 		return projectsList

ws/ws.go

@@ -8,7 +8,6 @@ import (
 	"database/sql"
 	"embed"
 	"fmt"
-	"git.sr.ht/~amolith/willow/users"
 	"io"
 	"net/http"
 	"net/url"
@@ -17,6 +16,8 @@ import (
 	"text/template"
 	"time"
 
+	"git.sr.ht/~amolith/willow/users"
+
 	"git.sr.ht/~amolith/willow/project"
 	"github.com/microcosm-cc/bluemonday"
 )