lfs.go
12492 bytes
1package git
2
3import (
4 "context"
5 "crypto/rand"
6 "errors"
7 "fmt"
8 "io"
9 "path"
10 "path/filepath"
11 "strconv"
12 "time"
13
14 "charm.land/log/v2"
15 "github.com/charmbracelet/git-lfs-transfer/transfer"
16 "github.com/charmbracelet/soft-serve/pkg/config"
17 "github.com/charmbracelet/soft-serve/pkg/db"
18 "github.com/charmbracelet/soft-serve/pkg/db/models"
19 "github.com/charmbracelet/soft-serve/pkg/lfs"
20 "github.com/charmbracelet/soft-serve/pkg/proto"
21 "github.com/charmbracelet/soft-serve/pkg/storage"
22 "github.com/charmbracelet/soft-serve/pkg/store"
23)
24
25// lfsTransfer implements transfer.Backend.
26type lfsTransfer struct {
27 ctx context.Context
28 cfg *config.Config
29 dbx *db.DB
30 store store.Store
31 logger *log.Logger
32 storage storage.Storage
33 repo proto.Repository
34}
35
36var _ transfer.Backend = &lfsTransfer{}
37
38// errInvalidOid is returned when a client sends a malformed object ID. Object
39// IDs are interpolated into storage paths, so anything outside the SHA-256
40// alphabet is a path traversal attempt.
41//
42// It wraps transfer.ErrParseError so the processor answers with a 400 instead
43// of falling through to a generic internal error, and lfs.ErrInvalidOIDFormat
44// so callers can still test for the cause.
45var errInvalidOid = fmt.Errorf("%w: %w", transfer.ErrParseError, lfs.ErrInvalidOIDFormat)
46
47// LFSTransfer is a Git LFS transfer service handler.
48// ctx is expected to have proto.User, *backend.Backend, *log.Logger,
49// *config.Config, *db.DB, and store.Store.
50// The first arg in cmd.Args should be the repo path.
51// The second arg in cmd.Args should be the LFS operation (download or upload).
52func LFSTransfer(ctx context.Context, cmd ServiceCommand) error {
53 if len(cmd.Args) < 2 {
54 return errors.New("missing args")
55 }
56
57 op := cmd.Args[1]
58 if op != lfs.OperationDownload && op != lfs.OperationUpload {
59 return errors.New("invalid operation")
60 }
61
62 logger := log.FromContext(ctx).WithPrefix("lfs-transfer")
63 handler := transfer.NewPktline(cmd.Stdin, cmd.Stdout, &lfsLogger{logger})
64 repo := proto.RepositoryFromContext(ctx)
65 if repo == nil {
66 logger.Error("no repository in context")
67 return proto.ErrRepoNotFound
68 }
69
70 // Advertise capabilities.
71 for _, cap := range transfer.Capabilities {
72 if err := handler.WritePacketText(cap); err != nil {
73 logger.Errorf("error sending capability: %s: %v", cap, err)
74 return err
75 }
76 }
77
78 if err := handler.WriteFlush(); err != nil {
79 logger.Error("error sending flush", "err", err)
80 return err
81 }
82
83 repoID := strconv.FormatInt(repo.ID(), 10)
84 cfg := config.FromContext(ctx)
85 processor := transfer.NewProcessor(handler, &lfsTransfer{
86 ctx: ctx,
87 cfg: cfg,
88 dbx: db.FromContext(ctx),
89 store: store.FromContext(ctx),
90 logger: logger,
91 storage: storage.NewLocalStorage(filepath.Join(cfg.DataPath, "lfs", repoID)),
92 repo: repo,
93 }, &lfsLogger{logger})
94
95 return processor.ProcessCommands(op)
96}
97
98// Batch implements transfer.Backend.
99func (t *lfsTransfer) Batch(_ string, pointers []transfer.BatchItem, _ transfer.Args) ([]transfer.BatchItem, error) {
100 for i := range pointers {
101 p := transfer.Pointer{Oid: pointers[i].Oid, Size: pointers[i].Size}
102 if !p.IsValid() {
103 return pointers, errInvalidOid
104 }
105 }
106
107 for i := range pointers {
108 obj, err := t.store.GetLFSObjectByOid(t.ctx, t.dbx, t.repo.ID(), pointers[i].Oid)
109 if err != nil && !errors.Is(err, db.ErrRecordNotFound) {
110 return pointers, db.WrapError(err)
111 }
112
113 pointers[i].Present, err = t.storage.Exists(path.Join("objects", pointers[i].RelativePath()))
114 if err != nil {
115 return pointers, err
116 }
117
118 if pointers[i].Present && obj.ID == 0 {
119 if err := t.store.CreateLFSObject(t.ctx, t.dbx, t.repo.ID(), pointers[i].Oid, pointers[i].Size); err != nil {
120 return pointers, db.WrapError(err)
121 }
122 }
123 }
124
125 return pointers, nil
126}
127
128// Download implements transfer.Backend.
129func (t *lfsTransfer) Download(oid string, _ transfer.Args) (io.ReadCloser, int64, error) {
130 p := transfer.Pointer{Oid: oid}
131 if !p.IsValid() {
132 return nil, 0, errInvalidOid
133 }
134
135 cfg := config.FromContext(t.ctx)
136 repoID := strconv.FormatInt(t.repo.ID(), 10)
137 strg := storage.NewLocalStorage(filepath.Join(cfg.DataPath, "lfs", repoID))
138 pointer := transfer.Pointer{Oid: oid}
139 obj, err := strg.Open(path.Join("objects", pointer.RelativePath()))
140 if err != nil {
141 return nil, 0, err
142 }
143 stat, err := obj.Stat()
144 if err != nil {
145 return nil, 0, err
146 }
147 return obj, stat.Size(), nil
148}
149
150// Upload implements transfer.Backend.
151func (t *lfsTransfer) Upload(oid string, size int64, r io.Reader, _ transfer.Args) error {
152 p := transfer.Pointer{Oid: oid}
153 if !p.IsValid() {
154 return errInvalidOid
155 }
156
157 if r == nil {
158 return fmt.Errorf("no reader: %w", transfer.ErrMissingData)
159 }
160
161 tempDir := "incomplete"
162 randBytes := make([]byte, 12)
163 if _, err := rand.Read(randBytes); err != nil {
164 return err
165 }
166
167 tempName := fmt.Sprintf("%s%x", oid, randBytes)
168 tempName = path.Join(tempDir, tempName)
169
170 written, err := t.storage.Put(tempName, r)
171 if err != nil {
172 t.logger.Errorf("error putting object: %v", err)
173 return err
174 }
175
176 pointer := transfer.Pointer{
177 Oid: oid,
178 }
179 if size > 0 {
180 pointer.Size = size
181 } else {
182 pointer.Size = written
183 }
184
185 if err := t.store.CreateLFSObject(t.ctx, t.dbx, t.repo.ID(), pointer.Oid, pointer.Size); err != nil {
186 return db.WrapError(err)
187 }
188
189 // Rename takes names relative to the storage root, not the absolute path
190 // the temp file happens to live at.
191 expectedPath := path.Join("objects", pointer.RelativePath())
192 if err := t.storage.Rename(tempName, expectedPath); err != nil {
193 t.logger.Errorf("error renaming object: %v", err)
194 _ = t.store.DeleteLFSObjectByOid(t.ctx, t.dbx, t.repo.ID(), pointer.Oid)
195 return err
196 }
197
198 return nil
199}
200
201// Verify implements transfer.Backend.
202func (t *lfsTransfer) Verify(oid string, size int64, _ transfer.Args) (transfer.Status, error) {
203 p := transfer.Pointer{Oid: oid}
204 if !p.IsValid() {
205 return transfer.NewStatus(transfer.StatusConflict, "invalid OID format"), nil
206 }
207
208 obj, err := t.store.GetLFSObjectByOid(t.ctx, t.dbx, t.repo.ID(), oid)
209 if err != nil {
210 if errors.Is(err, db.ErrRecordNotFound) {
211 return transfer.NewStatus(transfer.StatusNotFound, "object not found"), nil
212 }
213 t.logger.Errorf("error getting object: %v", err)
214 return nil, err
215 }
216
217 if obj.Size != size {
218 t.logger.Errorf("size mismatch: %d != %d", obj.Size, size)
219 return transfer.NewStatus(transfer.StatusConflict, "size mismatch"), nil
220 }
221
222 return transfer.SuccessStatus(), nil
223}
224
225type lfsLockBackend struct {
226 *lfsTransfer
227 args map[string]string
228 user proto.User
229}
230
231var _ transfer.LockBackend = (*lfsLockBackend)(nil)
232
233// LockBackend implements transfer.Backend.
234func (t *lfsTransfer) LockBackend(args transfer.Args) transfer.LockBackend {
235 user := proto.UserFromContext(t.ctx)
236 if user == nil {
237 t.logger.Errorf("no user in context while creating lock backend, repo %s", t.repo.Name())
238 return nil
239 }
240
241 return &lfsLockBackend{t, args, user}
242}
243
244// Create implements transfer.LockBackend.
245func (l *lfsLockBackend) Create(path string, refname string) (transfer.Lock, error) {
246 var lock LFSLock
247 if err := l.dbx.TransactionContext(l.ctx, func(tx *db.Tx) error {
248 if err := l.store.CreateLFSLockForUser(l.ctx, tx, l.repo.ID(), l.user.ID(), path, refname); err != nil {
249 return db.WrapError(err)
250 }
251
252 var err error
253 lock.lock, err = l.store.GetLFSLockForUserPath(l.ctx, tx, l.repo.ID(), l.user.ID(), path)
254 if err != nil {
255 return db.WrapError(err)
256 }
257
258 lock.owner, err = l.store.GetUserByID(l.ctx, tx, lock.lock.UserID)
259 return db.WrapError(err)
260 }); err != nil {
261 // Return conflict (409) if the lock already exists.
262 if errors.Is(err, db.ErrDuplicateKey) {
263 return nil, transfer.ErrConflict
264 }
265 l.logger.Errorf("error creating lock: %v", err)
266 return nil, err
267 }
268
269 lock.backend = l
270
271 return &lock, nil
272}
273
274// FromID implements transfer.LockBackend.
275func (l *lfsLockBackend) FromID(id string) (transfer.Lock, error) {
276 var lock LFSLock
277 iid, err := strconv.ParseInt(id, 10, 64)
278 if err != nil {
279 return nil, err
280 }
281
282 if err := l.dbx.TransactionContext(l.ctx, func(tx *db.Tx) error {
283 var err error
284 lock.lock, err = l.store.GetLFSLockForUserByID(l.ctx, tx, l.repo.ID(), l.user.ID(), iid)
285 if err != nil {
286 return db.WrapError(err)
287 }
288
289 lock.owner, err = l.store.GetUserByID(l.ctx, tx, lock.lock.UserID)
290 return db.WrapError(err)
291 }); err != nil {
292 if errors.Is(err, db.ErrRecordNotFound) {
293 return nil, transfer.ErrNotFound
294 }
295 l.logger.Errorf("error getting lock: %v", err)
296 return nil, err
297 }
298
299 lock.backend = l
300
301 return &lock, nil
302}
303
304// FromPath implements transfer.LockBackend.
305func (l *lfsLockBackend) FromPath(path string) (transfer.Lock, error) {
306 var lock LFSLock
307
308 if err := l.dbx.TransactionContext(l.ctx, func(tx *db.Tx) error {
309 var err error
310 lock.lock, err = l.store.GetLFSLockForUserPath(l.ctx, tx, l.repo.ID(), l.user.ID(), path)
311 if err != nil {
312 return db.WrapError(err)
313 }
314
315 lock.owner, err = l.store.GetUserByID(l.ctx, tx, lock.lock.UserID)
316 return db.WrapError(err)
317 }); err != nil {
318 if errors.Is(err, db.ErrRecordNotFound) {
319 return nil, transfer.ErrNotFound
320 }
321 l.logger.Errorf("error getting lock: %v", err)
322 return nil, err
323 }
324
325 lock.backend = l
326
327 return &lock, nil
328}
329
330// Range implements transfer.LockBackend.
331func (l *lfsLockBackend) Range(cursor string, limit int, fn func(transfer.Lock) error) (string, error) {
332 var nextCursor string
333 var locks []*LFSLock
334
335 page, _ := strconv.Atoi(cursor)
336 if page <= 0 {
337 page = 1
338 }
339
340 if limit <= 0 {
341 limit = lfs.DefaultLocksLimit
342 } else if limit > 100 {
343 limit = 100
344 }
345
346 if err := l.dbx.TransactionContext(l.ctx, func(tx *db.Tx) error {
347 l.logger.Debug("getting locks", "limit", limit, "page", page)
348 mlocks, err := l.store.GetLFSLocks(l.ctx, tx, l.repo.ID(), page, limit)
349 if err != nil {
350 return db.WrapError(err)
351 }
352
353 if len(mlocks) == limit {
354 nextCursor = strconv.Itoa(page + 1)
355 }
356
357 users := make(map[int64]models.User, 0)
358 for _, mlock := range mlocks {
359 owner, ok := users[mlock.UserID]
360 if !ok {
361 owner, err = l.store.GetUserByID(l.ctx, tx, mlock.UserID)
362 if err != nil {
363 return db.WrapError(err)
364 }
365
366 users[mlock.UserID] = owner
367 }
368
369 locks = append(locks, &LFSLock{lock: mlock, owner: owner, backend: l})
370 }
371
372 return nil
373 }); err != nil {
374 return "", err
375 }
376
377 for _, lock := range locks {
378 if err := fn(lock); err != nil {
379 return "", err
380 }
381 }
382
383 return nextCursor, nil
384}
385
386// Unlock implements transfer.LockBackend.
387func (l *lfsLockBackend) Unlock(lock transfer.Lock) error {
388 id, err := strconv.ParseInt(lock.ID(), 10, 64)
389 if err != nil {
390 return err
391 }
392
393 err = l.dbx.TransactionContext(l.ctx, func(tx *db.Tx) error {
394 return db.WrapError(
395 l.store.DeleteLFSLockForUserByID(l.ctx, tx, l.repo.ID(), l.user.ID(), id),
396 )
397 })
398 if err != nil {
399 if errors.Is(err, db.ErrRecordNotFound) {
400 return transfer.ErrNotFound
401 }
402 l.logger.Error("error unlocking lock", "err", err)
403 return err
404 }
405
406 return nil
407}
408
409// LFSLock is a Git LFS lock object.
410// It implements transfer.Lock.
411type LFSLock struct {
412 lock models.LFSLock
413 owner models.User
414 backend *lfsLockBackend
415}
416
417var _ transfer.Lock = (*LFSLock)(nil)
418
419// AsArguments implements transfer.Lock.
420func (l *LFSLock) AsArguments() []string {
421 return []string{
422 fmt.Sprintf("id=%s", l.ID()),
423 fmt.Sprintf("path=%s", l.Path()),
424 fmt.Sprintf("locked-at=%s", l.FormattedTimestamp()),
425 fmt.Sprintf("ownername=%s", l.OwnerName()),
426 }
427}
428
429// AsLockSpec implements transfer.Lock.
430func (l *LFSLock) AsLockSpec(ownerID bool) ([]string, error) {
431 id := l.ID()
432 spec := []string{
433 fmt.Sprintf("lock %s", id),
434 fmt.Sprintf("path %s %s", id, l.Path()),
435 fmt.Sprintf("locked-at %s %s", id, l.FormattedTimestamp()),
436 fmt.Sprintf("ownername %s %s", id, l.OwnerName()),
437 }
438
439 if ownerID {
440 who := "theirs"
441 if l.lock.UserID == l.owner.ID {
442 who = "ours"
443 }
444
445 spec = append(spec, fmt.Sprintf("owner %s %s", id, who))
446 }
447
448 return spec, nil
449}
450
451// FormattedTimestamp implements transfer.Lock.
452func (l *LFSLock) FormattedTimestamp() string {
453 return l.lock.CreatedAt.Format(time.RFC3339)
454}
455
456// ID implements transfer.Lock.
457func (l *LFSLock) ID() string {
458 return strconv.FormatInt(l.lock.ID, 10)
459}
460
461// OwnerName implements transfer.Lock.
462func (l *LFSLock) OwnerName() string {
463 return l.owner.Username
464}
465
466// Path implements transfer.Lock.
467func (l *LFSLock) Path() string {
468 return l.lock.Path
469}
470
471// Unlock implements transfer.Lock.
472func (l *LFSLock) Unlock() error {
473 return l.backend.Unlock(l)
474}