`f915f4d53531bb3d8b92b13d36e51a6d5dd8e981`

Author: Ayman Bagabas <ayman.bagabas@gmail.com>
Committer: GitHub <noreply@github.com>
Date: 2023-10-31 16:26 UTC

Message

refactor,fix: add rename lfs objects migration (#409)

* refactor,fix: add rename lfs objects migration

Rename lfs objects from OID[:2]/OID[2:4]/OID[4:] to OID[:2]/OID[2:4]/OID
to follow specs https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git

* fix: disable pure ssh lfs by default

Diff

This diff is truncated to protect this page.

  1diff --git a/README.md b/README.md
  2index a8544f051a1a2e6bb4c533be05263eda074cb2f0..c02744c5b7b7393eac2404b49f3bc27e4717f7d5 100644
  3--- a/README.md
  4+++ b/README.md
  5@@ -208,7 +208,7 @@ lfs:
  6   # Enable Git LFS.
  7   enabled: true
  8   # Enable Git SSH transfer.
  9-  ssh_enabled: true
 10+  ssh_enabled: false
 11 
 12 # Cron job configuration
 13 jobs:
 14@@ -268,6 +268,8 @@ Soft Serve supports both Git LFS [HTTP](https://github.com/git-lfs/git-lfs/blob/
 15 
 16 Use the `lfs` config section to customize your Git LFS server.
 17 
 18+> **Note**: The pure-SSH transfer is disabled by default.
 19+
 20 ## Server Access
 21 
 22 Soft Serve at its core manages your server authentication and authorization. Authentication verifies the identity of a user, while authorization determines their access rights to a repository.
 23diff --git a/go.mod b/go.mod
 24index 47b614d846d61fb1e9bcf5459dee4851d744a42e..af62532317367f438780c1b5e3f37307e16b652b 100644
 25--- a/go.mod
 26+++ b/go.mod
 27@@ -23,7 +23,7 @@ require (
 28 	github.com/caarlos0/duration v0.0.0-20220103233809-8df7c22fe305
 29 	github.com/caarlos0/env/v8 v8.0.0
 30 	github.com/caarlos0/tablewriter v0.1.0
 31-	github.com/charmbracelet/git-lfs-transfer v0.1.1-0.20230725143853-5dd0632f9245
 32+	github.com/charmbracelet/git-lfs-transfer v0.1.1-0.20231027181609-f7ff6baf2ed0
 33 	github.com/charmbracelet/keygen v0.5.0
 34 	github.com/charmbracelet/log v0.2.5
 35 	github.com/charmbracelet/ssh v0.0.0-20230822194956-1a051f898e09
 36diff --git a/go.sum b/go.sum
 37index b5da0e353bd7b26964c987ffb2d27b85d4935cde..da62882e720536a2f41cd56f71224ef323009428 100644
 38--- a/go.sum
 39+++ b/go.sum
 40@@ -25,8 +25,8 @@ github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5
 41 github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc=
 42 github.com/charmbracelet/bubbletea v0.24.2 h1:uaQIKx9Ai6Gdh5zpTbGiWpytMU+CfsPp06RaW2cx/SY=
 43 github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FDaJmO3l2nejekbsgg=
 44-github.com/charmbracelet/git-lfs-transfer v0.1.1-0.20230725143853-5dd0632f9245 h1:PeGKqKX84IAFhFSWjTyPGiLzzEPcv94C9qKsYBk2nbQ=
 45-github.com/charmbracelet/git-lfs-transfer v0.1.1-0.20230725143853-5dd0632f9245/go.mod h1:eXJuVicxnjRgRMokmutZdistxoMRjBjjfqvrYq7bCIU=
 46+github.com/charmbracelet/git-lfs-transfer v0.1.1-0.20231027181609-f7ff6baf2ed0 h1:Wi80d2xNAqvi6r8Udlc9UgPMdrJ+ld5ylKH7d8SQ7gE=
 47+github.com/charmbracelet/git-lfs-transfer v0.1.1-0.20231027181609-f7ff6baf2ed0/go.mod h1:AHLgIZ2TXQCgt3pDNXR6FgmpGHLH1wPM9cEJPHSVaYg=
 48 github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc=
 49 github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc=
 50 github.com/charmbracelet/keygen v0.5.0 h1:XY0fsoYiCSM9axkrU+2ziE6u6YjJulo/b9Dghnw6MZc=
 51diff --git a/pkg/config/config.go b/pkg/config/config.go
 52index e1124204faebef4e42f5ca53a1d965641dc48c9b..aed4a9d1f0432e9dee73e69dcc62e1734ec33b73 100644
 53--- a/pkg/config/config.go
 54+++ b/pkg/config/config.go
 55@@ -339,7 +339,7 @@ func DefaultConfig() *Config {
 56 		},
 57 		LFS: LFSConfig{
 58 			Enabled:    true,
 59-			SSHEnabled: true,
 60+			SSHEnabled: false,
 61 		},
 62 	}
 63 }
 64diff --git a/pkg/db/migrate/0003_migrate_lfs_objects.go b/pkg/db/migrate/0003_migrate_lfs_objects.go
 65new file mode 100644
 66index 0000000000000000000000000000000000000000..70aaa79e8ce2b339dbedd207d0b996ef0f257a14
 67--- /dev/null
 68+++ b/pkg/db/migrate/0003_migrate_lfs_objects.go
 69@@ -0,0 +1,70 @@
 70+package migrate
 71+
 72+import (
 73+	"context"
 74+	"os"
 75+	"path/filepath"
 76+	"strconv"
 77+
 78+	"github.com/charmbracelet/log"
 79+	"github.com/charmbracelet/soft-serve/pkg/config"
 80+	"github.com/charmbracelet/soft-serve/pkg/db"
 81+	"github.com/charmbracelet/soft-serve/pkg/db/models"
 82+)
 83+
 84+const (
 85+	migrateLfsObjectsName    = "migrate_lfs_objects"
 86+	migrateLfsObjectsVersion = 3
 87+)
 88+
 89+// Correct LFS objects relative path.
 90+// From OID[:2]/OID[2:4]/OID[4:] to OID[:2]/OID[2:4]/OID
 91+// See: https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git
 92+var migrateLfsObjects = Migration{
 93+	Name:    migrateLfsObjectsName,
 94+	Version: migrateLfsObjectsVersion,
 95+	Migrate: func(ctx context.Context, tx *db.Tx) error {
 96+		cfg := config.FromContext(ctx)
 97+		logger := log.FromContext(ctx).WithPrefix("migrate_lfs_objects")
 98+
 99+		var repoIds []int64
100+		if err := tx.Select(&repoIds, "SELECT id FROM repos"); err != nil {
101+			return err
102+		}
103+		for _, r := range repoIds {
104+			var objs []models.LFSObject
105+			if err := tx.Select(&objs, "SELECT * FROM lfs_objects WHERE repo_id = ?", r); err != nil {
106+				return err
107+			}
108+			objsp := filepath.Join(cfg.DataPath, "lfs", strconv.FormatInt(r, 10), "objects")
109+			for _, obj := range objs {
110+				oldpath := filepath.Join(objsp, badRelativePath(obj.Oid))
111+				newpath := filepath.Join(objsp, goodRelativePath(obj.Oid))
112+				if _, err := os.Stat(oldpath); err == nil {
113+					if err := os.Rename(oldpath, newpath); err != nil {
114+						logger.Error("rename lfs object", "oldpath", oldpath, "newpath", newpath, "err", err)
115+						continue
116+					}
117+				}
118+			}
119+		}
120+		return nil
121+	},
122+	Rollback: func(ctx context.Context, tx *db.Tx) error {
123+		return nil
124+	},
125+}
126+
127+func goodRelativePath(oid string) string {
128+	if len(oid) < 5 {
129+		return oid
130+	}
131+	return filepath.Join(oid[:2], oid[2:4], oid)
132+}
133+
134+func badRelativePath(oid string) string {
135+	if len(oid) < 5 {
136+		return oid
137+	}
138+	return filepath.Join(oid[:2], oid[2:4], oid[4:])
139+}
140diff --git a/pkg/db/migrate/migrations.go b/pkg/db/migrate/migrations.go
141index 8a7c375c0b0db290ef1ae4193b3ea47f3cacc516..e2598b414b2cb0326affbf4f86ab3341c71e791a 100644
142--- a/pkg/db/migrate/migrations.go
143+++ b/pkg/db/migrate/migrations.go
144@@ -17,6 +17,7 @@ var sqls embed.FS
145 var migrations = []Migration{
146 	createTables,
147 	webhooks,
148+	migrateLfsObjects,
149 }
150 
151 func execMigration(ctx context.Context, tx *db.Tx, version int, name string, down bool) error {
152diff --git a/pkg/git/lfs.go b/pkg/git/lfs.go
153index b767c30c542360ad089b4bfca92b4780006f6e57..5aae027e52acf27d2503193cd0ce773e255c23b0 100644
154--- a/pkg/git/lfs.go
155+++ b/pkg/git/lfs.go
156@@ -10,7 +10,6 @@ import (
157 	"path"
158 	"path/filepath"
159 	"strconv"
160-	"strings"
161 	"time"
162 
163 	"github.com/charmbracelet/git-lfs-transfer/transfer"
164@@ -62,7 +61,7 @@ func LFSTransfer(ctx context.Context, cmd ServiceCommand) error {
165 	}
166 
167 	logger := log.FromContext(ctx).WithPrefix("lfs-transfer")
168-	handler := transfer.NewPktline(cmd.Stdin, cmd.Stdout)
169+	handler := transfer.NewPktline(cmd.Stdin, cmd.Stdout, &lfsLogger{logger})
170 	repo := proto.RepositoryFromContext(ctx)
171 	if repo == nil {
172 		logger.Error("no repository in context")
173@@ -95,43 +94,36 @@ func LFSTransfer(ctx context.Context, cmd ServiceCommand) error {
174 		logger:  logger,
175 		storage: storage.NewLocalStorage(filepath.Join(cfg.DataPath, "lfs", repoID)),
176 		repo:    repo,
177-	})
178+	}, &lfsLogger{logger})
179 
180 	return processor.ProcessCommands(op)
181 }
182 
183 // Batch implements transfer.Backend.
184-func (t *lfsTransfer) Batch(_ string, pointers []transfer.Pointer, _ map[string]string) ([]transfer.BatchItem, error) {
185-	items := make([]transfer.BatchItem, 0)
186-	for _, p := range pointers {
187-		obj, err := t.store.GetLFSObjectByOid(t.ctx, t.dbx, t.repo.ID(), p.Oid)
188+func (t *lfsTransfer) Batch(_ string, pointers []transfer.BatchItem, _ transfer.Args) ([]transfer.BatchItem, error) {
189+	for i := range pointers {
190+		obj, err := t.store.GetLFSObjectByOid(t.ctx, t.dbx, t.repo.ID(), pointers[i].Oid)
191 		if err != nil && !errors.Is(err, db.ErrRecordNotFound) {
192-			return items, db.WrapError(err)
193+			return pointers, db.WrapError(err)
194 		}
195 
196-		exist, err := t.storage.Exists(path.Join("objects", p.RelativePath()))
197+		pointers[i].Present, err = t.storage.Exists(path.Join("objects", pointers[i].RelativePath()))
198 		if err != nil {
199-			return items, err
200+			return pointers, err
201 		}
202 
203-		if exist && obj.ID == 0 {
204-			if err := t.store.CreateLFSObject(t.ctx, t.dbx, t.repo.ID(), p.Oid, p.Size); err != nil {
205-				return items, db.WrapError(err)
206+		if pointers[i].Present && obj.ID == 0 {
207+			if err := t.store.CreateLFSObject(t.ctx, t.dbx, t.repo.ID(), pointers[i].Oid, pointers[i].Size); err != nil {
208+				return pointers, db.WrapError(err)
209 			}
210 		}
211-
212-		item := transfer.BatchItem{
213-			Pointer: p,
214-			Present: exist,
215-		}
216-		items = append(items, item)
217 	}
218 
219-	return items, nil
220+	return pointers, nil
221 }
222 
223 // Download implements transfer.Backend.
224-func (t *lfsTransfer) Download(oid string, _ map[string]string) (fs.File, error) {
225+func (t *lfsTransfer) Download(oid string, _ transfer.Args) (fs.File, error) {
226 	cfg := config.FromContext(t.ctx)
227 	repoID := strconv.FormatInt(t.repo.ID(), 10)
228 	strg := storage.NewLocalStorage(filepath.Join(cfg.DataPath, "lfs", repoID))
229@@ -145,8 +137,12 @@ type uploadObject struct {
230 	object storage.Object
231 }
232 
233+func (u *uploadObject) Close() error {
234+	return u.object.Close()
235+}
236+
237 // StartUpload implements transfer.Backend.
238-func (t *lfsTransfer) StartUpload(oid string, r io.Reader, _ map[string]string) (interface{}, error) {
239+func (t *lfsTransfer) StartUpload(oid string, r io.Reader, _ transfer.Args) (io.Closer, error) {
240 	if r == nil {
241 		return nil, fmt.Errorf("no reader: %w", transfer.ErrMissingData)
242 	}
243@@ -174,7 +170,7 @@ func (t *lfsTransfer) StartUpload(oid string, r io.Reader, _ map[string]string)
244 
245 	t.logger.Infof("Object name: %s", obj.Name())
246 
247-	return uploadObject{
248+	return &uploadObject{
249 		oid:    oid,
250 		size:   written,
251 		object: obj,
252@@ -182,20 +178,13 @@ func (t *lfsTransfer) StartUpload(oid string, r io.Reader, _ map[string]string)
253 }
254 
255 // FinishUpload implements transfer.Backend.
256diff --git a/pkg/git/lfs_log.go b/pkg/git/lfs_log.go
257new file mode 100644
258index 0000000000000000000000000000000000000000..979932d51df6a5cfd9700c12f7d103d2d2a8aed3
259--- /dev/null
260+++ b/pkg/git/lfs_log.go
261@@ -0,0 +1,17 @@
262+package git
263+
264+import (
265+	"github.com/charmbracelet/git-lfs-transfer/transfer"
266+	"github.com/charmbracelet/log"
267+)
268+
269+type lfsLogger struct {
270+	l *log.Logger
271+}
272+
273+var _ transfer.Logger = &lfsLogger{}
274+
275+// Log implements transfer.Logger.
276+func (l *lfsLogger) Log(msg string, kv ...interface{}) {
277+	l.l.Debug(msg, kv...)
278+}
279diff --git a/pkg/lfs/pointer.go b/pkg/lfs/pointer.go
280index b38d04ce59b67ac7e5021a0d8bd33a5cc04b077e..69bceca1da4ab857a895fee44e2b85dec4258d55 100644
281--- a/pkg/lfs/pointer.go
282+++ b/pkg/lfs/pointer.go
283@@ -102,12 +102,13 @@ func (p Pointer) String() string {
284 }
285 
286 // RelativePath returns the relative storage path of the pointer
287+// https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git
288 func (p Pointer) RelativePath() string {
289 	if len(p.Oid) < 5 {
290 		return p.Oid
291 	}
292 
293-	return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid[4:])
294+	return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid)
295 }
296 
297 // GeneratePointer generates a pointer for arbitrary content
298diff --git a/pkg/lfs/pointer_test.go b/pkg/lfs/pointer_test.go
299index df2f2daa2fb5e62d8cee71226d894a23676787c4..fac9c35c6a850c097c349eff1fe1c8a3e43d0ec8 100644
300--- a/pkg/lfs/pointer_test.go
301+++ b/pkg/lfs/pointer_test.go
302@@ -2,8 +2,8 @@ package lfs
303 
304 import (
305 	"errors"
306+	"path"
307 	"strconv"
308-	"strings"
309 	"testing"
310 )
311 
312@@ -78,8 +78,8 @@ size abc
313 					t.Errorf("Expected a valid pointer")
314 					return
315 				}
316-				if p.Oid != strings.ReplaceAll(p.RelativePath(), "/", "") {
317-					t.Errorf("Expected oid to be the relative path without slashes")
318+				if path.Join(p.Oid[:2], p.Oid[2:4], p.Oid) != p.RelativePath() {
319+					t.Errorf("Expected a valid relative path")
320 					return
321 				}
322 			}