git_lfs.go
15015 bytes
1package web
2
3import (
4 "encoding/json"
5 "errors"
6 "fmt"
7 "io"
8 "io/fs"
9 "net/http"
10 "path"
11 "path/filepath"
12 "strconv"
13 "strings"
14
15 "charm.land/log/v2"
16 "github.com/charmbracelet/soft-serve/pkg/access"
17 "github.com/charmbracelet/soft-serve/pkg/config"
18 "github.com/charmbracelet/soft-serve/pkg/db"
19 "github.com/charmbracelet/soft-serve/pkg/lfs"
20 "github.com/charmbracelet/soft-serve/pkg/proto"
21 "github.com/charmbracelet/soft-serve/pkg/storage"
22 "github.com/charmbracelet/soft-serve/pkg/store"
23 "github.com/gorilla/mux"
24)
25
26// serviceLfsBatch handles a Git LFS batch requests.
27// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md
28// TODO: support refname
29// POST: /<repo>.git/info/lfs/objects/batch
30func serviceLfsBatch(w http.ResponseWriter, r *http.Request) {
31 ctx := r.Context()
32 logger := log.FromContext(ctx).WithPrefix("http.lfs")
33
34 if !isLfs(r) {
35 logger.Errorf("invalid content type: %s", r.Header.Get("Content-Type"))
36 renderNotAcceptable(w)
37 return
38 }
39
40 var batchRequest lfs.BatchRequest
41 defer r.Body.Close() //nolint: errcheck
42 if err := json.NewDecoder(r.Body).Decode(&batchRequest); err != nil {
43 logger.Errorf("error decoding json: %s", err)
44 renderJSON(w, http.StatusUnprocessableEntity, lfs.ErrorResponse{
45 Message: "validation error in request: " + err.Error(),
46 })
47 return
48 }
49
50 // We only accept basic transfers for now
51 // Default to basic if no transfer is specified
52 if len(batchRequest.Transfers) > 0 {
53 var isBasic bool
54 for _, t := range batchRequest.Transfers {
55 if t == lfs.TransferBasic {
56 isBasic = true
57 break
58 }
59 }
60
61 if !isBasic {
62 renderJSON(w, http.StatusUnprocessableEntity, lfs.ErrorResponse{
63 Message: "unsupported transfer",
64 })
65 return
66 }
67 }
68
69 if len(batchRequest.Objects) == 0 {
70 renderJSON(w, http.StatusUnprocessableEntity, lfs.ErrorResponse{
71 Message: "no objects found",
72 })
73 return
74 }
75
76 name := mux.Vars(r)["repo"]
77 repo, ok := lfsRepository(w, r)
78 if !ok {
79 return
80 }
81
82 cfg := config.FromContext(ctx)
83 dbx := db.FromContext(ctx)
84 datastore := store.FromContext(ctx)
85 // TODO: support S3 storage
86 repoID := strconv.FormatInt(repo.ID(), 10)
87 strg := storage.NewLocalStorage(filepath.Join(cfg.DataPath, "lfs", repoID))
88
89 baseHref := fmt.Sprintf("%s/%s/info/lfs/objects/basic", cfg.HTTP.PublicURL, name+".git")
90
91 var batchResponse lfs.BatchResponse
92 batchResponse.Transfer = lfs.TransferBasic
93 batchResponse.HashAlgo = lfs.HashAlgorithmSHA256
94
95 objects := make([]*lfs.ObjectResponse, 0, len(batchRequest.Objects))
96 // XXX: We don't support objects TTL for now, probably implement that with
97 // S3 using object "expires_at" & "expires_in"
98 switch batchRequest.Operation {
99 case lfs.OperationDownload:
100 for _, o := range batchRequest.Objects {
101 // Object IDs become storage paths, so validate before touching the
102 // filesystem.
103 if !o.IsValid() {
104 logger.Error("invalid object", "oid", o.Oid, "repo", name)
105 objects = append(objects, &lfs.ObjectResponse{
106 Pointer: o,
107 Error: &lfs.ObjectError{
108 Code: http.StatusUnprocessableEntity,
109 Message: "invalid object",
110 },
111 })
112 continue
113 }
114
115 exist, err := strg.Exists(path.Join("objects", o.RelativePath()))
116 if err != nil && !errors.Is(err, fs.ErrNotExist) {
117 logger.Error("error getting object stat", "oid", o.Oid, "repo", name, "err", err)
118 renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
119 Message: "internal server error",
120 })
121 return
122 }
123
124 obj, err := datastore.GetLFSObjectByOid(ctx, dbx, repo.ID(), o.Oid)
125 if err != nil && !errors.Is(err, db.ErrRecordNotFound) {
126 logger.Error("error getting object from database", "oid", o.Oid, "repo", name, "err", err)
127 renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
128 Message: "internal server error",
129 })
130 return
131 }
132
133 if !exist {
134 objects = append(objects, &lfs.ObjectResponse{
135 Pointer: o,
136 Error: &lfs.ObjectError{
137 Code: http.StatusNotFound,
138 Message: "object not found",
139 },
140 })
141 } else if obj.Size != o.Size {
142 objects = append(objects, &lfs.ObjectResponse{
143 Pointer: o,
144 Error: &lfs.ObjectError{
145 Code: http.StatusUnprocessableEntity,
146 Message: "size mismatch",
147 },
148 })
149 } else {
150 download := &lfs.Link{
151 Href: fmt.Sprintf("%s/%s", baseHref, o.Oid),
152 }
153 if auth := r.Header.Get("Authorization"); auth != "" {
154 download.Header = map[string]string{
155 "Authorization": auth,
156 }
157 }
158
159 objects = append(objects, &lfs.ObjectResponse{
160 Pointer: o,
161 Actions: map[string]*lfs.Link{
162 lfs.ActionDownload: download,
163 },
164 })
165
166 // If the object doesn't exist in the database, create it
167 if exist && obj.ID == 0 {
168 if err := datastore.CreateLFSObject(ctx, dbx, repo.ID(), o.Oid, o.Size); err != nil {
169 logger.Error("error creating object in datastore", "oid", o.Oid, "repo", name, "err", err)
170 renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
171 Message: "internal server error",
172 })
173 return
174 }
175 }
176 }
177 }
178 case lfs.OperationUpload:
179 // Check authorization
180 accessLevel := access.FromContext(ctx)
181 if accessLevel < access.ReadWriteAccess {
182 askCredentials(w, r)
183 renderJSON(w, http.StatusForbidden, lfs.ErrorResponse{
184 Message: "write access required",
185 })
186 return
187 }
188
189 // Object upload logic happens in the "basic" API route
190 for _, o := range batchRequest.Objects {
191 if !o.IsValid() {
192 objects = append(objects, &lfs.ObjectResponse{
193 Pointer: o,
194 Error: &lfs.ObjectError{
195 Code: http.StatusUnprocessableEntity,
196 Message: "invalid object",
197 },
198 })
199 } else {
200 upload := &lfs.Link{
201 Href: fmt.Sprintf("%s/%s", baseHref, o.Oid),
202 Header: map[string]string{
203 // NOTE: git-lfs v2.5.0 sets the Content-Type based on the uploaded file.
204 // This ensures that the client always uses the designated value for the header.
205 "Content-Type": "application/octet-stream",
206 },
207 }
208 verify := &lfs.Link{
209 Href: fmt.Sprintf("%s/verify", baseHref),
210 }
211 if auth := r.Header.Get("Authorization"); auth != "" {
212 upload.Header["Authorization"] = auth
213 verify.Header = map[string]string{
214 "Authorization": auth,
215 }
216 }
217
218 objects = append(objects, &lfs.ObjectResponse{
219 Pointer: o,
220 Actions: map[string]*lfs.Link{
221 lfs.ActionUpload: upload,
222 // Verify uploaded objects
223 // https://github.com/git-lfs/git-lfs/blob/main/docs/api/basic-transfers.md#verification
224 lfs.ActionVerify: verify,
225 },
226 })
227 }
228 }
229 default:
230 renderJSON(w, http.StatusUnprocessableEntity, lfs.ErrorResponse{
231 Message: "unsupported operation",
232 })
233 return
234 }
235
236 batchResponse.Objects = objects
237 renderJSON(w, http.StatusOK, batchResponse)
238}
239
240// serviceLfsBasic implements Git LFS basic transfer API
241// https://github.com/git-lfs/git-lfs/blob/main/docs/api/basic-transfers.md
242func serviceLfsBasic(w http.ResponseWriter, r *http.Request) {
243 switch r.Method {
244 case http.MethodGet:
245 serviceLfsBasicDownload(w, r)
246 case http.MethodPut:
247 serviceLfsBasicUpload(w, r)
248 }
249}
250
251// GET: /<repo>.git/info/lfs/objects/basic/<oid>
252func serviceLfsBasicDownload(w http.ResponseWriter, r *http.Request) {
253 ctx := r.Context()
254 oid := mux.Vars(r)["oid"]
255 repo, ok := lfsRepository(w, r)
256 if !ok {
257 return
258 }
259
260 cfg := config.FromContext(ctx)
261 logger := log.FromContext(ctx).WithPrefix("http.lfs-basic")
262 datastore := store.FromContext(ctx)
263 dbx := db.FromContext(ctx)
264 repoID := strconv.FormatInt(repo.ID(), 10)
265 strg := storage.NewLocalStorage(filepath.Join(cfg.DataPath, "lfs", repoID))
266
267 obj, err := datastore.GetLFSObjectByOid(ctx, dbx, repo.ID(), oid)
268 if err != nil && !errors.Is(err, db.ErrRecordNotFound) {
269 logger.Error("error getting object from database", "oid", oid, "repo", repo.Name(), "err", err)
270 renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
271 Message: "internal server error",
272 })
273 return
274 }
275
276 pointer := lfs.Pointer{Oid: oid}
277 f, err := strg.Open(path.Join("objects", pointer.RelativePath()))
278 if err != nil {
279 logger.Error("error opening object", "oid", oid, "err", err)
280 renderJSON(w, http.StatusNotFound, lfs.ErrorResponse{
281 Message: "object not found",
282 })
283 return
284 }
285
286 w.Header().Set("Content-Type", "application/octet-stream")
287 w.Header().Set("Content-Length", strconv.FormatInt(obj.Size, 10))
288 defer f.Close() //nolint: errcheck
289 if _, err := io.Copy(w, f); err != nil {
290 logger.Error("error copying object to response", "oid", oid, "err", err)
291 renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
292 Message: "internal server error",
293 })
294 return
295 }
296}
297
298// PUT: /<repo>.git/info/lfs/objects/basic/<oid>
299func serviceLfsBasicUpload(w http.ResponseWriter, r *http.Request) {
300 if !isBinary(r) {
301 renderJSON(w, http.StatusUnsupportedMediaType, lfs.ErrorResponse{
302 Message: "invalid content type",
303 })
304 return
305 }
306
307 ctx := r.Context()
308
309 // Checked in withAccess too. Repeated here so a routing or middleware
310 // mistake cannot turn into an unauthenticated write.
311 if access.FromContext(ctx) < access.ReadWriteAccess {
312 askCredentials(w, r)
313 renderJSON(w, http.StatusForbidden, lfs.ErrorResponse{
314 Message: "write access required",
315 })
316 return
317 }
318
319 oid := mux.Vars(r)["oid"]
320 cfg := config.FromContext(ctx)
321 dbx := db.FromContext(ctx)
322 datastore := store.FromContext(ctx)
323 logger := log.FromContext(ctx).WithPrefix("http.lfs-basic")
324 repo, ok := lfsRepository(w, r)
325 if !ok {
326 return
327 }
328
329 repoID := strconv.FormatInt(repo.ID(), 10)
330 strg := storage.NewLocalStorage(filepath.Join(cfg.DataPath, "lfs", repoID))
331
332 defer r.Body.Close() //nolint: errcheck
333
334 // NOTE: Git LFS client will retry uploading the same object if there was a
335 // partial error, so we need to skip existing objects.
336 if _, err := datastore.GetLFSObjectByOid(ctx, dbx, repo.ID(), oid); err == nil {
337 // Object exists, skip request
338 io.Copy(io.Discard, r.Body) //nolint: errcheck
339 renderStatus(http.StatusOK)(w, nil)
340 return
341 } else if !errors.Is(err, db.ErrRecordNotFound) {
342 logger.Error("error getting object", "oid", oid, "err", err)
343 renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
344 Message: "internal server error",
345 })
346 return
347 }
348
349 pointer := lfs.Pointer{Oid: oid}
350 if _, err := strg.Put(path.Join("objects", pointer.RelativePath()), r.Body); err != nil {
351 logger.Error("error writing object", "oid", oid, "err", err)
352 renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
353 Message: "internal server error",
354 })
355 return
356 }
357
358 size, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64)
359 if err != nil {
360 logger.Error("error parsing content length", "err", err)
361 renderJSON(w, http.StatusBadRequest, lfs.ErrorResponse{
362 Message: "invalid content length",
363 })
364 return
365 }
366
367 if err := datastore.CreateLFSObject(ctx, dbx, repo.ID(), oid, size); err != nil {
368 logger.Error("error creating object", "oid", oid, "err", err)
369 renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
370 Message: "internal server error",
371 })
372 return
373 }
374
375 renderStatus(http.StatusOK)(w, nil)
376}
377
378// POST: /<repo>.git/info/lfs/objects/basic/verify
379func serviceLfsBasicVerify(w http.ResponseWriter, r *http.Request) {
380 if !isLfs(r) {
381 renderNotAcceptable(w)
382 return
383 }
384
385 var pointer lfs.Pointer
386 ctx := r.Context()
387 logger := log.FromContext(ctx).WithPrefix("http.lfs-basic")
388 repo, ok := lfsRepository(w, r)
389 if !ok {
390 return
391 }
392
393 defer r.Body.Close() //nolint: errcheck
394 if err := json.NewDecoder(r.Body).Decode(&pointer); err != nil {
395 logger.Error("error decoding json", "err", err)
396 renderJSON(w, http.StatusBadRequest, lfs.ErrorResponse{
397 Message: "invalid request: " + err.Error(),
398 })
399 return
400 }
401
402 // Object IDs become storage paths, so validate before touching the
403 // filesystem.
404 if !pointer.IsValid() {
405 logger.Error("invalid object", "oid", pointer.Oid)
406 renderJSON(w, http.StatusUnprocessableEntity, lfs.ErrorResponse{
407 Message: "invalid object",
408 })
409 return
410 }
411
412 cfg := config.FromContext(ctx)
413 dbx := db.FromContext(ctx)
414 datastore := store.FromContext(ctx)
415 repoID := strconv.FormatInt(repo.ID(), 10)
416 strg := storage.NewLocalStorage(filepath.Join(cfg.DataPath, "lfs", repoID))
417 if stat, err := strg.Stat(path.Join("objects", pointer.RelativePath())); err == nil {
418 // Verify object is in the database.
419 obj, err := datastore.GetLFSObjectByOid(ctx, dbx, repo.ID(), pointer.Oid)
420 if err != nil {
421 if errors.Is(err, db.ErrRecordNotFound) {
422 logger.Error("object not found", "oid", pointer.Oid)
423 renderJSON(w, http.StatusNotFound, lfs.ErrorResponse{
424 Message: "object not found",
425 })
426 return
427 }
428 logger.Error("error getting object", "oid", pointer.Oid, "err", err)
429 renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
430 Message: "internal server error",
431 })
432 return
433 }
434
435 if obj.Size != pointer.Size {
436 renderJSON(w, http.StatusBadRequest, lfs.ErrorResponse{
437 Message: "object size mismatch",
438 })
439 return
440 }
441
442 if stat.Size() == pointer.Size {
443 renderStatus(http.StatusOK)(w, nil)
444 return
445 }
446 } else if errors.Is(err, fs.ErrNotExist) {
447 logger.Error("file not found", "oid", pointer.Oid)
448 renderJSON(w, http.StatusNotFound, lfs.ErrorResponse{
449 Message: "object not found",
450 })
451 return
452 } else {
453 logger.Error("error getting object", "oid", pointer.Oid, "err", err)
454 renderJSON(w, http.StatusInternalServerError, lfs.ErrorResponse{
455 Message: "internal server error",
456 })
457 return
458 }
459}
460
461// renderJSON renders a JSON response with the given status code and value. It
462// also sets the Content-Type header to the JSON LFS media type (application/vnd.git-lfs+json).
463func renderJSON(w http.ResponseWriter, statusCode int, v interface{}) {
464 hdrLfs(w)
465 w.WriteHeader(statusCode)
466 if err := json.NewEncoder(w).Encode(v); err != nil {
467 log.Error("error encoding json", "err", err)
468 }
469}
470
471func renderNotAcceptable(w http.ResponseWriter) {
472 renderStatus(http.StatusNotAcceptable)(w, nil)
473}
474
475func isLfs(r *http.Request) bool {
476 contentType := r.Header.Get("Content-Type")
477 accept := r.Header.Get("Accept")
478 return strings.HasPrefix(contentType, lfs.MediaType) && strings.HasPrefix(accept, lfs.MediaType)
479}
480
481func isBinary(r *http.Request) bool {
482 contentType := r.Header.Get("Content-Type")
483 return strings.HasPrefix(contentType, "application/octet-stream")
484}
485
486func hdrLfs(w http.ResponseWriter) {
487 w.Header().Set("Content-Type", lfs.MediaType)
488 w.Header().Set("Accept", lfs.MediaType)
489}
490
491// lfsRepository returns the repository that withAccess stored in the request
492// context, answering 404 if there is none.
493func lfsRepository(w http.ResponseWriter, r *http.Request) (proto.Repository, bool) {
494 repo := proto.RepositoryFromContext(r.Context())
495 if repo == nil {
496 log.FromContext(r.Context()).Error("error getting repository from context")
497 renderJSON(w, http.StatusNotFound, lfs.ErrorResponse{
498 Message: "repository not found",
499 })
500 return nil, false
501 }
502 return repo, true
503}