Update dependencies

storage: streamline ETag calculation
This commit introduces some helpers so that ETags can be calculated at the same time that files get read or written. Besides looking nicer, it should also help reduce lock contention around file access, as files do not need to be opened twice anymore.
2025-12-12 06:07:22 +01:00 · 2024-11-07 17:55:56 +01:00 · 2024-11-07 17:55:56 +01:00 · 2024-11-07 17:55:56 +01:00 · 2024-11-07 17:55:56 +01:00 · 2024-11-07 17:55:42 +01:00
8 changed files with 218 additions and 81 deletions
--- a/12
+++ b/12
@ -0,0 +1,12 @@
+
+build:
+	go build ./cmd/tokidoki
+
+debug:
+	go build -tags "pam nullauth" ./cmd/tokidoki
+
+fmt:
+	go fmt ./...
+
+lint:
+	golangci-lint run
--- a/cmd/tokidoki/main.go
+++ b/cmd/tokidoki/main.go
@ -207,7 +207,10 @@ func main() {
 	for sig := range sigCh {
 		switch sig {
 		case syscall.SIGINT, syscall.SIGTERM:
-			server.Shutdown(context.Background())
+			err := server.Shutdown(context.Background())
+			if err != nil {
+				log.Fatal().Err(err).Msg("Shutdown() error")
+			}
 			return
 		}
 	}
--- a/go.mod
+++ b/go.mod
@ -3,22 +3,22 @@ module git.sr.ht/~sircmpwn/tokidoki
 go 1.18

 require (
-	git.sr.ht/~emersion/go-oauth2 v0.0.0-20240217160856-2e0d6e20b088
+	git.sr.ht/~emersion/go-oauth2 v0.0.0-20240226120011-78f10ffd1d51
 	github.com/emersion/go-ical v0.0.0-20240127095438-fc1c9d8fb2b6
-	github.com/emersion/go-imap/v2 v2.0.0-beta.2.0.20240417100641-a587a14d3f01
-	github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9
-	github.com/emersion/go-webdav v0.5.1-0.20240419143909-21f251fa1de2
-	github.com/go-chi/chi/v5 v5.0.12
+	github.com/emersion/go-imap/v2 v2.0.0-beta.4
+	github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff
+	github.com/emersion/go-webdav v0.5.1-0.20240713135526-7f8c17ad7135
+	github.com/go-chi/chi/v5 v5.1.0
 	github.com/msteinert/pam/v2 v2.0.0
-	github.com/rs/zerolog v1.32.0
-	golang.org/x/crypto v0.18.0
+	github.com/rs/zerolog v1.33.0
+	golang.org/x/crypto v0.28.0
 )

 require (
 	github.com/emersion/go-message v0.18.1 // indirect
-	github.com/emersion/go-sasl v0.0.0-20231106173351-e73c9f7bad43 // indirect
+	github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/teambition/rrule-go v1.8.2 // indirect
-	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
 )
--- a/go.sum
+++ b/go.sum
@ -1,20 +1,21 @@
-git.sr.ht/~emersion/go-oauth2 v0.0.0-20240217160856-2e0d6e20b088 h1:KuPliLD8CQM1WbCHdjHR6mhadIzLaAJCNENmvB1y9gs=
-git.sr.ht/~emersion/go-oauth2 v0.0.0-20240217160856-2e0d6e20b088/go.mod h1:VHj0jSCLIkrfEwmOvJ4+ykpoVbD/YLN7BM523oKKBHc=
+git.sr.ht/~emersion/go-oauth2 v0.0.0-20240226120011-78f10ffd1d51 h1:iz8Tm7obSouGC0atCd+NtFSmCgfxDizXD1Rm+0Jw75w=
+git.sr.ht/~emersion/go-oauth2 v0.0.0-20240226120011-78f10ffd1d51/go.mod h1:VHj0jSCLIkrfEwmOvJ4+ykpoVbD/YLN7BM523oKKBHc=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/emersion/go-ical v0.0.0-20240127095438-fc1c9d8fb2b6 h1:kHoSgklT8weIDl6R6xFpBJ5IioRdBU1v2X2aCZRVCcM=
 github.com/emersion/go-ical v0.0.0-20240127095438-fc1c9d8fb2b6/go.mod h1:BEksegNspIkjCQfmzWgsgbu6KdeJ/4LwUZs7DMBzjzw=
-github.com/emersion/go-imap/v2 v2.0.0-beta.2.0.20240417100641-a587a14d3f01 h1:dq/06hDbCT+/DpbKWSrfrTeiJW97ION78N6J6Mktp2w=
-github.com/emersion/go-imap/v2 v2.0.0-beta.2.0.20240417100641-a587a14d3f01/go.mod h1:BZTFHsS1hmgBkFlHqbxGLXk2hnRqTItUgwjSSCsYNAk=
+github.com/emersion/go-imap/v2 v2.0.0-beta.4 h1:BS7+kUVhe/jfuFWgn8li0AbCKBIDoNvqJWsRJppltcc=
+github.com/emersion/go-imap/v2 v2.0.0-beta.4/go.mod h1:BZTFHsS1hmgBkFlHqbxGLXk2hnRqTItUgwjSSCsYNAk=
 github.com/emersion/go-message v0.18.1 h1:tfTxIoXFSFRwWaZsgnqS1DSZuGpYGzSmCZD8SK3QA2E=
 github.com/emersion/go-message v0.18.1/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA=
-github.com/emersion/go-sasl v0.0.0-20231106173351-e73c9f7bad43 h1:hH4PQfOndHDlpzYfLAAfl63E8Le6F2+EL/cdhlkyRJY=
-github.com/emersion/go-sasl v0.0.0-20231106173351-e73c9f7bad43/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
-github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 h1:ATgqloALX6cHCranzkLb8/zjivwQ9DWWDCQRnxTPfaA=
+github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3DfFi4EXdT+B+gTtzx8GC9xsc26Znk=
+github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
 github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM=
-github.com/emersion/go-webdav v0.5.1-0.20240419143909-21f251fa1de2 h1:k/NO/RfeXFuKGcpHDkspYoE8u6tWoHs03tH5DXg22To=
-github.com/emersion/go-webdav v0.5.1-0.20240419143909-21f251fa1de2/go.mod h1:mI8iBx3RAODwX7PJJ7qzsKAKs/vY429YfS2/9wKnDbQ=
-github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s=
-github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
+github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff h1:4N8wnS3f1hNHSmFD5zgFkWCyA4L1kCDkImPAtK7D6tg=
+github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM=
+github.com/emersion/go-webdav v0.5.1-0.20240713135526-7f8c17ad7135 h1:Ssk00uh7jhctJ23eclGxhhGqplSQB+wCt6fmbjhnOS8=
+github.com/emersion/go-webdav v0.5.1-0.20240713135526-7f8c17ad7135/go.mod h1:mI8iBx3RAODwX7PJJ7qzsKAKs/vY429YfS2/9wKnDbQ=
+github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
+github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
@ -26,15 +27,15 @@ github.com/msteinert/pam/v2 v2.0.0 h1:jnObb8MT6jvMbmrUQO5J/puTUjxy7Av+55zVJRJsCy
 github.com/msteinert/pam/v2 v2.0.0/go.mod h1:KT28NNIcDFf3PcBmNI2mIGO4zZJ+9RSs/At2PB3IDVc=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0=
-github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
+github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/teambition/rrule-go v1.8.2 h1:lIjpjvWTj9fFUZCmuoVDrKVOtdiyzbzc93qTmRVe/J8=
 github.com/teambition/rrule-go v1.8.2/go.mod h1:Ieq5AbrKGciP1V//Wq8ktsTXwSwJHDD5mD/wLBGl3p4=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
-golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
+golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
+golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@ -53,12 +54,12 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
+golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
--- a/storage/etag.go
+++ b/storage/etag.go
@ -0,0 +1,58 @@
+package storage
+
+import (
+	"crypto/sha1"
+	"encoding/base64"
+	"hash"
+	"io"
+)
+
+type ETagWriter struct {
+	io.Writer
+	output io.Writer
+	csum   hash.Hash
+}
+
+// NewETagWriter creates a new io.Writer that pipes writes to the provided
+// output while also calculating the contents ETag.
+func NewETagWriter(output io.Writer) *ETagWriter {
+	csum := sha1.New()
+	return &ETagWriter{
+		output: io.MultiWriter(output, csum),
+		csum:   csum,
+	}
+}
+
+func (e *ETagWriter) Write(p []byte) (n int, err error) {
+	return e.output.Write(p)
+}
+
+func (e *ETagWriter) ETag() string {
+	csum := e.csum.Sum(nil)
+	return base64.StdEncoding.EncodeToString(csum[:])
+}
+
+type ETagReader struct {
+	io.Reader
+	input io.Reader
+	csum  hash.Hash
+}
+
+// NewETagReader creates a new io.Reader that pipes reads from the provided
+// input while also calculating the contents ETag.
+func NewETagReader(input io.Reader) *ETagReader {
+	csum := sha1.New()
+	return &ETagReader{
+		input: io.TeeReader(input, csum),
+		csum:  csum,
+	}
+}
+
+func (e *ETagReader) Read(p []byte) (n int, err error) {
+	return e.input.Read(p)
+}
+
+func (e *ETagReader) ETag() string {
+	csum := e.csum.Sum(nil)
+	return base64.StdEncoding.EncodeToString(csum[:])
+}
--- a/storage/filesystem.go
+++ b/storage/filesystem.go
@ -1,8 +1,6 @@
 package storage

 import (
-	"crypto/sha1"
-	"encoding/base64"
 	"fmt"
 	"io"
 	"os"
@ -10,6 +8,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"

 	"github.com/rs/zerolog/log"

@ -20,9 +19,13 @@ import (

 type filesystemBackend struct {
 	webdav.UserPrincipalBackend
+
 	path          string
 	caldavPrefix  string
 	carddavPrefix string
+
+	// maps file path to *sync.RWMutex
+	locks sync.Map
 }

 var (
@ -47,6 +50,36 @@ func NewFilesystem(fsPath, caldavPrefix, carddavPrefix string, userPrincipalBack
 	return backend, backend, nil
 }

+func (b *filesystemBackend) RLock(filename string) {
+	lock_, _ := b.locks.LoadOrStore(filename, &sync.RWMutex{})
+	lock := lock_.(*sync.RWMutex)
+	lock.RLock()
+}
+
+func (b *filesystemBackend) RUnlock(filename string) {
+	lock_, ok := b.locks.Load(filename)
+	if !ok {
+		panic("attempt to unlock non-existing lock")
+	}
+	lock := lock_.(*sync.RWMutex)
+	lock.RUnlock()
+}
+
+func (b *filesystemBackend) Lock(filename string) {
+	lock_, _ := b.locks.LoadOrStore(filename, &sync.RWMutex{})
+	lock := lock_.(*sync.RWMutex)
+	lock.Lock()
+}
+
+func (b *filesystemBackend) Unlock(filename string) {
+	lock_, ok := b.locks.Load(filename)
+	if !ok {
+		panic("attempt to unlock non-existing lock")
+	}
+	lock := lock_.(*sync.RWMutex)
+	lock.Unlock()
+}
+
 func ensureLocalDir(path string) error {
 	if _, err := os.Stat(path); os.IsNotExist(err) {
 		err = os.MkdirAll(path, 0755)
@ -109,11 +142,10 @@ func etagForFile(path string) (string, error) {
 	}
 	defer f.Close()

-	h := sha1.New()
-	if _, err := io.Copy(h, f); err != nil {
+	src := NewETagReader(f)
+	if _, err := io.ReadAll(src); err != nil {
 		return "", err
 	}
-	csum := h.Sum(nil)

-	return base64.StdEncoding.EncodeToString(csum[:]), nil
+	return src.ETag(), nil
 }
--- a/storage/filesystem_caldav.go
+++ b/storage/filesystem_caldav.go
@ -47,22 +47,23 @@ func (b *filesystemBackend) safeLocalCalDAVPath(ctx context.Context, urlPath str
 	return b.safeLocalPath(homeSetPath, urlPath)
 }

-func calendarFromFile(path string, propFilter []string) (*ical.Calendar, error) {
+func calendarAndEtagFromFile(path string, propFilter []string) (*ical.Calendar, string, error) {
 	f, err := os.Open(path)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 	defer f.Close()

-	dec := ical.NewDecoder(f)
+	src := NewETagReader(f)
+	dec := ical.NewDecoder(src)
 	cal, err := dec.Decode()
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}

-	return cal, nil
 	// TODO implement
 	//return icalPropFilter(cal, propFilter), nil
+	return cal, src.ETag(), nil
 }

 func (b *filesystemBackend) loadAllCalendarObjects(ctx context.Context, urlPath string, propFilter []string) ([]caldav.CalendarObject, error) {
@ -85,17 +86,15 @@ func (b *filesystemBackend) loadAllCalendarObjects(ctx context.Context, urlPath
 			return nil
 		}

-		cal, err := calendarFromFile(filename, propFilter)
+		b.RLock(filename)
+		defer b.RUnlock(filename)
+
+		cal, etag, err := calendarAndEtagFromFile(filename, propFilter)
 		if err != nil {
 			fmt.Printf("load calendar error for %s: %v\n", filename, err)
 			return err
 		}

-		etag, err := etagForFile(filename)
-		if err != nil {
-			return err
-		}
-
 		// TODO can this potentially be called on a calendar object resource?
 		// Would work (as Walk() includes root), except for the path construction below
 		obj := caldav.CalendarObject{
@ -105,6 +104,7 @@ func (b *filesystemBackend) loadAllCalendarObjects(ctx context.Context, urlPath
 			ETag:          etag,
 			Data:          cal,
 		}
+		log.Debug().Str("path", obj.Path).Str("etag", etag).Int64("size", info.Size()).Msg("calendar object loaded")
 		result = append(result, obj)
 		return nil
 	})
@ -138,7 +138,12 @@ func (b *filesystemBackend) createDefaultCalendar(ctx context.Context) (*caldav.
 	if err != nil {
 		return nil, fmt.Errorf("error creating default calendar: %s", err.Error())
 	}
-	err = os.WriteFile(path.Join(localPath, calendarFileName), blob, 0644)
+
+	filename := path.Join(localPath, calendarFileName)
+	b.Lock(filename)
+	defer b.Unlock(filename)
+
+	err = os.WriteFile(filename, blob, 0644)
 	if err != nil {
 		return nil, fmt.Errorf("error writing default calendar: %s", err.Error())
 	}
@ -167,6 +172,10 @@ func (b *filesystemBackend) ListCalendars(ctx context.Context) ([]caldav.Calenda
 		}

 		calPath := path.Join(filename, calendarFileName)
+
+		b.RLock(calPath)
+		defer b.RUnlock(calPath)
+
 		data, err := os.ReadFile(calPath)
 		if err != nil {
 			if os.IsNotExist(err) {
@ -209,6 +218,9 @@ func (b *filesystemBackend) GetCalendar(ctx context.Context, urlPath string) (*c

 	log.Debug().Str("path", localPath).Msg("loading calendar")

+	b.RLock(localPath)
+	defer b.RUnlock(localPath)
+
 	data, err := os.ReadFile(localPath)
 	if err != nil {
 		if os.IsNotExist(err) {
@ -237,6 +249,9 @@ func (b *filesystemBackend) GetCalendarObject(ctx context.Context, objPath strin
 		return nil, err
 	}

+	b.RLock(localPath)
+	defer b.RUnlock(localPath)
+
 	info, err := os.Stat(localPath)
 	if err != nil {
 		if errors.Is(err, fs.ErrNotExist) {
@ -250,17 +265,12 @@ func (b *filesystemBackend) GetCalendarObject(ctx context.Context, objPath strin
 		propFilter = req.Props
 	}

-	calendar, err := calendarFromFile(localPath, propFilter)
+	calendar, etag, err := calendarAndEtagFromFile(localPath, propFilter)
 	if err != nil {
 		log.Debug().Str("path", localPath).Err(err).Msg("error reading calendar")
 		return nil, err
 	}

-	etag, err := etagForFile(localPath)
-	if err != nil {
-		return nil, err
-	}
-
 	obj := caldav.CalendarObject{
 		Path:          objPath,
 		ModTime:       info.ModTime(),
@ -268,6 +278,7 @@ func (b *filesystemBackend) GetCalendarObject(ctx context.Context, objPath strin
 		ETag:          etag,
 		Data:          calendar,
 	}
+	log.Debug().Str("path", objPath).Str("etag", etag).Int64("size", info.Size()).Msg("returning calendar object")
 	return &obj, nil
 }

@ -320,6 +331,9 @@ func (b *filesystemBackend) PutCalendarObject(ctx context.Context, objPath strin
 		return nil, err
 	}

+	b.Lock(localPath)
+	defer b.Unlock(localPath)
+
 	flags := os.O_RDWR | os.O_CREATE | os.O_TRUNC
 	// TODO handle IfNoneMatch == ETag
 	if opts.IfNoneMatch.IsWildcard() {
@ -352,16 +366,13 @@ func (b *filesystemBackend) PutCalendarObject(ctx context.Context, objPath strin
 	}
 	defer f.Close()

-	enc := ical.NewEncoder(f)
+	out := NewETagWriter(f)
+	enc := ical.NewEncoder(out)
 	err = enc.Encode(calendar)
 	if err != nil {
 		return nil, err
 	}

-	etag, err := etagForFile(localPath)
-	if err != nil {
-		return nil, err
-	}
 	info, err := f.Stat()
 	if err != nil {
 		return nil, err
@ -371,9 +382,10 @@ func (b *filesystemBackend) PutCalendarObject(ctx context.Context, objPath strin
 		Path:          objPath,
 		ModTime:       info.ModTime(),
 		ContentLength: info.Size(),
-		ETag:          etag,
+		ETag:          out.ETag(),
 		Data:          calendar,
 	}
+	log.Debug().Str("path", r.Path).Str("etag", r.ETag).Int64("size", info.Size()).Msg("calendar object updated")
 	return &r, nil
 }

@ -384,6 +396,10 @@ func (b *filesystemBackend) DeleteCalendarObject(ctx context.Context, path strin
 	if err != nil {
 		return err
 	}
+
+	b.Lock(localPath)
+	defer b.Unlock(localPath)
+
 	err = os.Remove(localPath)
 	if err != nil {
 		return err
--- a/storage/filesystem_carddav.go
+++ b/storage/filesystem_carddav.go
@ -68,20 +68,21 @@ func vcardPropFilter(card vcard.Card, props []string) vcard.Card {
 	return result
 }

-func vcardFromFile(path string, propFilter []string) (vcard.Card, error) {
+func vcardAndEtagFromFile(path string, propFilter []string) (vcard.Card, string, error) {
 	f, err := os.Open(path)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 	defer f.Close()

-	dec := vcard.NewDecoder(f)
+	src := NewETagReader(f)
+	dec := vcard.NewDecoder(src)
 	card, err := dec.Decode()
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}

-	return vcardPropFilter(card, propFilter), nil
+	return vcardPropFilter(card, propFilter), src.ETag(), nil
 }

 func (b *filesystemBackend) loadAllAddressObjects(ctx context.Context, urlPath string, propFilter []string) ([]carddav.AddressObject, error) {
@ -103,12 +104,10 @@ func (b *filesystemBackend) loadAllAddressObjects(ctx context.Context, urlPath s
 			return nil
 		}

-		card, err := vcardFromFile(filename, propFilter)
-		if err != nil {
-			return err
-		}
+		b.RLock(filename)
+		defer b.RUnlock(filename)

-		etag, err := etagForFile(filename)
+		card, etag, err := vcardAndEtagFromFile(filename, propFilter)
 		if err != nil {
 			return err
 		}
@ -122,6 +121,7 @@ func (b *filesystemBackend) loadAllAddressObjects(ctx context.Context, urlPath s
 			ETag:          etag,
 			Card:          card,
 		}
+		log.Debug().Str("path", obj.Path).Str("etag", etag).Int64("size", info.Size()).Msg("address object loaded")
 		result = append(result, obj)
 		return nil
 	})
@ -141,7 +141,12 @@ func (b *filesystemBackend) writeAddressBook(ctx context.Context, ab *carddav.Ad
 	if err != nil {
 		return err
 	}
-	return os.WriteFile(path.Join(localPath, addressBookFileName), blob, 0644)
+
+	filename := path.Join(localPath, addressBookFileName)
+	b.Lock(filename)
+	defer b.Unlock(filename)
+
+	err = os.WriteFile(filename, blob, 0644)
 	if err != nil {
 		return fmt.Errorf("error writing address book: %s", err.Error())
 	}
@ -208,6 +213,9 @@ func (b *filesystemBackend) ListAddressBooks(ctx context.Context) ([]carddav.Add
 		}

 		abPath := path.Join(filename, addressBookFileName)
+		b.RLock(abPath)
+		defer b.RUnlock(abPath)
+
 		data, err := os.ReadFile(abPath)
 		if err != nil {
 			if os.IsNotExist(err) {
@ -250,6 +258,9 @@ func (b *filesystemBackend) GetAddressBook(ctx context.Context, urlPath string)

 	log.Debug().Str("path", localPath).Msg("loading addressbook")

+	b.RLock(localPath)
+	defer b.RUnlock(localPath)
+
 	data, err := os.ReadFile(localPath)
 	if err != nil {
 		if os.IsNotExist(err) {
@ -303,6 +314,9 @@ func (b *filesystemBackend) GetAddressObject(ctx context.Context, objPath string
 		return nil, err
 	}

+	b.RLock(localPath)
+	defer b.RUnlock(localPath)
+
 	info, err := os.Stat(localPath)
 	if err != nil {
 		if errors.Is(err, fs.ErrNotExist) {
@ -316,17 +330,12 @@ func (b *filesystemBackend) GetAddressObject(ctx context.Context, objPath string
 		propFilter = req.Props
 	}

-	card, err := vcardFromFile(localPath, propFilter)
+	card, etag, err := vcardAndEtagFromFile(localPath, propFilter)
 	if err != nil {
 		log.Debug().Str("path", localPath).Err(err).Msg("error reading calendar")
 		return nil, err
 	}

-	etag, err := etagForFile(localPath)
-	if err != nil {
-		return nil, err
-	}
-
 	obj := carddav.AddressObject{
 		Path:          objPath,
 		ModTime:       info.ModTime(),
@ -334,6 +343,7 @@ func (b *filesystemBackend) GetAddressObject(ctx context.Context, objPath string
 		ETag:          etag,
 		Card:          card,
 	}
+	log.Debug().Str("path", objPath).Str("etag", etag).Int64("size", info.Size()).Msg("returning address object")
 	return &obj, nil
 }

@ -381,6 +391,9 @@ func (b *filesystemBackend) PutAddressObject(ctx context.Context, objPath string
 		return nil, err
 	}

+	b.Lock(localPath)
+	defer b.Unlock(localPath)
+
 	flags := os.O_RDWR | os.O_CREATE | os.O_TRUNC
 	// TODO handle IfNoneMatch == ETag
 	if opts.IfNoneMatch.IsWildcard() {
@ -413,15 +426,12 @@ func (b *filesystemBackend) PutAddressObject(ctx context.Context, objPath string
 	}
 	defer f.Close()

-	enc := vcard.NewEncoder(f)
+	out := NewETagWriter(f)
+	enc := vcard.NewEncoder(out)
 	if err := enc.Encode(card); err != nil {
 		return nil, err
 	}

-	etag, err := etagForFile(localPath)
-	if err != nil {
-		return nil, err
-	}
 	info, err := f.Stat()
 	if err != nil {
 		return nil, err
@ -431,9 +441,10 @@ func (b *filesystemBackend) PutAddressObject(ctx context.Context, objPath string
 		Path:          objPath,
 		ModTime:       info.ModTime(),
 		ContentLength: info.Size(),
-		ETag:          etag,
+		ETag:          out.ETag(),
 		Card:          card,
 	}
+	log.Debug().Str("path", r.Path).Str("etag", r.ETag).Int64("size", info.Size()).Msg("address object updated")
 	return &r, nil
 }

@ -444,6 +455,10 @@ func (b *filesystemBackend) DeleteAddressObject(ctx context.Context, path string
 	if err != nil {
 		return err
 	}
+
+	b.Lock(localPath)
+	defer b.Unlock(localPath)
+
 	err = os.Remove(localPath)
 	if err != nil {
 		return err
Author	SHA1	Message	Date
Conrad Hoffmann	c6fdff75d0	Update dependencies	2024-11-07 17:55:56 +01:00
Conrad Hoffmann	665b206709	storage: streamline ETag calculation This commit introduces some helpers so that ETags can be calculated at the same time that files get read or written. Besides looking nicer, it should also help reduce lock contention around file access, as files do not need to be opened twice anymore.	2024-11-07 17:55:56 +01:00
Conrad Hoffmann	fe0a0d0d00	Add simple Makefile	2024-11-07 17:55:56 +01:00
Conrad Hoffmann	7e03188c5a	cmd/tokidoki: fix unused error	2024-11-07 17:55:56 +01:00
Conrad Hoffmann	6eeea854be	storage/filesystem: add R/W locking This commit adds read/write locking for individual files, so that concurrent requests (e.g. to read and write the same file) cannot interfere with one another. The locking is not very fine-grained at the moment, and can probably be improved upon. But it does ensure consistency.	2024-11-07 17:55:42 +01:00