Amolith
/
willow
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

BREAKING: SQL schema change 

- Redo schema to improve handling of lightweight tags
- Try to clean up empty directories when untracking a project

To resolve schema conflict, run `sqlite3 willow.sqlite` and paste the
following:

ALTER TABLE releases RENAME TO releases_bak;
CREATE TABLE releases
(
    id          TEXT      NOT NULL PRIMARY KEY,
    project_url TEXT      NOT NULL,
    release_url TEXT      NOT NULL,
    tag         TEXT      NOT NULL,
    content     TEXT      NOT NULL,
    date        TIMESTAMP NOT NULL,
    created_at  TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);

If everything works as expected, you can `DROP TABLE releases_bak`.
This commit is contained in:
Amolith 2023-10-29 10:41:00 -04:00
parent 3612ffc595
commit 984d44775b
Signed by: Amolith
GPG Key ID: 8AE30347CE28D101
5 changed files with 142 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
db/release.go
View File

@ -4,7 +4,9 @@
package db
import "database/sql"
import (
"database/sql"
)
// AddRelease adds a release for a project with a given URL to the database
@ -14,16 +16,16 @@ import "database/sql"
// UpsertRelease adds or updates a release for a project with a given URL in the
// database
func UpsertRelease(db *sql.DB, projectURL, releaseURL, tag, content, date string) error {
_, err := db.Exec(`INSERT INTO releases (project_url, release_url, tag, content, date)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(release_url) DO
func UpsertRelease(db *sql.DB, id, projectURL, releaseURL, tag, content, date string) error {
_, err := db.Exec(`INSERT INTO releases (id, project_url, release_url, tag, content, date)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO
UPDATE SET
release_url = excluded.release_url,
content = excluded.content,
tag = excluded.tag,
content = excluded.content,
date = excluded.date;`, projectURL, releaseURL, tag, content, date)
date = excluded.date;`, id, projectURL, releaseURL, tag, content, date)
return err
}

55
db/sql/schema.sql
View File

@ -4,43 +4,44 @@
-- Create table of users with username, password hash, salt, and creation
-- timestamp
CREATE TABLE users (
username VARCHAR(255) NOT NULL,
hash VARCHAR(255) NOT NULL,
salt VARCHAR(255) NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (username)
CREATE TABLE users
(
username TEXT NOT NULL PRIMARY KEY,
hash TEXT NOT NULL,
salt TEXT NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- Create table of sessions with session GUID, username, and timestamp of when
-- the session was created
CREATE TABLE sessions (
token VARCHAR(255) NOT NULL,
username VARCHAR(255) NOT NULL,
expires TIMESTAMP NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (token)
CREATE TABLE sessions
(
token TEXT NOT NULL,
username TEXT NOT NULL,
expires TIMESTAMP NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- Create table of tracked projects with URL, name, forge, running version, and
-- timestamp of when the project was added
CREATE TABLE projects (
url VARCHAR(255) NOT NULL,
name VARCHAR(255) NOT NULL,
forge VARCHAR(255) NOT NULL,
version VARCHAR(255) NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (url)
CREATE TABLE projects
(
url TEXT NOT NULL,
name TEXT NOT NULL,
forge TEXT NOT NULL,
version TEXT NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
-- Create table of project releases with the project URL and the release tags,
-- contents, URLs, and dates
CREATE TABLE releases (
project_url VARCHAR(255) NOT NULL,
release_url VARCHAR(255) NOT NULL,
tag VARCHAR(255) NOT NULL,
content TEXT NOT NULL,
date TIMESTAMP NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (release_url)
CREATE TABLE releases
(
id TEXT NOT NULL PRIMARY KEY,
project_url TEXT NOT NULL,
release_url TEXT NOT NULL,
tag TEXT NOT NULL,
content TEXT NOT NULL,
date TIMESTAMP NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);

94
git/git.go
View File

@ -7,6 +7,7 @@ package git
import (
"errors"
"fmt"
"io"
"net/url"
"os"
"sort"
@ -64,31 +65,39 @@ func GetReleases(gitURI, forge string) ([]Release, error) {
releases := make([]Release, 0)
err = tagRefs.ForEach(func(tagRef *plumbing.Reference) error {
obj, err := r.TagObject(tagRef.Hash())
switch {
case errors.Is(err, plumbing.ErrObjectNotFound):
// This is a lightweight tag, not an annotated tag, skip it
return nil
case err == nil:
tagURL := ""
tagName := bmStrict.Sanitize(tagRef.Name().Short())
switch forge {
case "sourcehut":
tagURL = "https://" + httpURI + "/refs/" + tagName
case "gitlab":
tagURL = "https://" + httpURI + "/-/releases/" + tagName
default:
tagURL = ""
tagObj, err := r.TagObject(tagRef.Hash())
var message string
var date time.Time
if errors.Is(err, plumbing.ErrObjectNotFound) {
commitTag, err := r.CommitObject(tagRef.Hash())
if err != nil {
return err
}
releases = append(releases, Release{
Tag: tagName,
Content: bmUGC.Sanitize(obj.Message),
URL: tagURL,
Date: obj.Tagger.When,
})
default:
return err
message = commitTag.Message
date = commitTag.Committer.When
} else {
message = tagObj.Message
date = tagObj.Tagger.When
}
tagURL := ""
tagName := bmStrict.Sanitize(tagRef.Name().Short())
switch forge {
case "sourcehut":
tagURL = "https://" + httpURI + "/refs/" + tagName
case "gitlab":
tagURL = "https://" + httpURI + "/-/releases/" + tagName
default:
tagURL = ""
}
releases = append(releases, Release{
Tag: tagName,
Content: bmUGC.Sanitize(message),
URL: tagURL,
Date: date,
})
return nil
})
if err != nil {
@ -139,9 +148,48 @@ func RemoveRepo(url string) (err error) {
return err
}
err = os.RemoveAll(path)
if err != nil {
return err
}
// TODO: Check whether the two parent directories are empty and remove them if
// so
for i := 0; i < 2; i++ {
path = strings.TrimSuffix(path, "/")
if path == "data" {
break
}
empty, err := dirEmpty(path)
if err != nil {
return err
}
if empty {
err = os.Remove(path)
if err != nil {
return err
}
}
path = path[:strings.LastIndex(path, "/")]
}
return err
}
// dirEmpty checks if a directory is empty.
func dirEmpty(name string) (empty bool, err error) {
f, err := os.Open(name)
if err != nil {
return false, err
}
defer f.Close()
_, err = f.Readdirnames(1)
if err == io.EOF {
return true, nil
}
return false, err
}
// stringifyRepo accepts a repository URI string and the corresponding local
// filesystem path, whether the URI is HTTP, HTTPS, or SSH.
func stringifyRepo(url string) (path string, err error) {

42
project/project.go
View File

@ -5,6 +5,7 @@
package project
import (
"crypto/sha256"
"database/sql"
"fmt"
"log"
@ -40,7 +41,7 @@ func GetReleases(dbConn *sql.DB, proj Project) (Project, error) {
}
if len(ret) == 0 {
return fetchReleases(proj)
return fetchReleases(dbConn, proj)
}
for _, row := range ret {
@ -58,7 +59,7 @@ func GetReleases(dbConn *sql.DB, proj Project) (Project, error) {
}
// fetchReleases fetches releases from a project's forge given its URI
func fetchReleases(p Project) (Project, error) {
func fetchReleases(dbConn *sql.DB, p Project) (Project, error) {
var err error
switch p.Forge {
case "github", "gitea", "forgejo":
@ -74,6 +75,11 @@ func fetchReleases(p Project) (Project, error) {
URL: release.URL,
Date: release.Date,
})
err = upsert(dbConn, p.URL, p.Releases)
if err != nil {
log.Printf("Error upserting release: %v", err)
return p, err
}
}
default:
gitReleases, err := git.GetReleases(p.URL, p.Forge)
@ -87,6 +93,11 @@ func fetchReleases(p Project) (Project, error) {
URL: release.URL,
Date: release.Date,
})
err = upsert(dbConn, p.URL, p.Releases)
if err != nil {
log.Printf("Error upserting release: %v", err)
return p, err
}
}
}
sort.Slice(p.Releases, func(i, j int) bool {
@ -95,6 +106,21 @@ func fetchReleases(p Project) (Project, error) {
return p, err
}
// upsert updates or inserts a project release into the database
func upsert(dbConn *sql.DB, url string, releases []Release) error {
for _, release := range releases {
date := release.Date.Format("2006-01-02 15:04:05")
idByte := sha256.Sum256([]byte(url + release.URL + release.Tag + date))
id := fmt.Sprintf("%x", idByte)
err := db.UpsertRelease(dbConn, id, url, release.URL, release.Tag, release.Content, date)
if err != nil {
log.Printf("Error upserting release: %v", err)
return err
}
}
return nil
}
func Track(dbConn *sql.DB, manualRefresh *chan struct{}, name, url, forge, release string) {
err := db.UpsertProject(dbConn, url, name, forge, release)
if err != nil {
@ -126,7 +152,7 @@ func RefreshLoop(dbConn *sql.DB, interval int, manualRefresh, req *chan struct{}
fmt.Println("Error getting projects:", err)
}
for i, p := range projectsList {
p, err := fetchReleases(p)
p, err := fetchReleases(dbConn, p)
if err != nil {
fmt.Println(err)
continue
@ -137,12 +163,10 @@ func RefreshLoop(dbConn *sql.DB, interval int, manualRefresh, req *chan struct{}
return strings.ToLower(projectsList[i].Name) < strings.ToLower(projectsList[j].Name)
})
for i := range projectsList {
for j := range projectsList[i].Releases {
err = db.UpsertRelease(dbConn, projectsList[i].URL, projectsList[i].Releases[j].URL, projectsList[i].Releases[j].Tag, projectsList[i].Releases[j].Content, projectsList[i].Releases[j].Date.Format("2006-01-02 15:04:05"))
if err != nil {
fmt.Println("Error upserting release:", err)
continue
}
err = upsert(dbConn, projectsList[i].URL, projectsList[i].Releases)
if err != nil {
fmt.Println("Error upserting release:", err)
continue
}
}
return projectsList

3
ws/ws.go
View File

@ -8,7 +8,6 @@ import (
"database/sql"
"embed"
"fmt"
"git.sr.ht/~amolith/willow/users"
"io"
"net/http"
"net/url"
@ -17,6 +16,8 @@ import (
"text/template"
"time"
"git.sr.ht/~amolith/willow/users"
"git.sr.ht/~amolith/willow/project"
"github.com/microcosm-cc/bluemonday"
)