6cd8ca6a694ecaf2d8a663c60641f247e41d348f

Author
Ayman Bagabas <ayman.bagabas@gmail.com>
Committer
Ayman Bagabas <ayman.bagabas@gmail.com>
Date

Message

feat(server): add pull mirror repos

Diff

  1diff --git a/server/backend/file/file.go b/server/backend/file/file.go
  2index cf0b368feb147df29de5ae0b52ea03c50c553985..230a83592762c9950e083750ac8bb2e0dffa3596 100644
  3--- a/server/backend/file/file.go
  4+++ b/server/backend/file/file.go
  5@@ -51,6 +51,7 @@ const (
  6 	private      = "private"
  7 	projectName  = "project-name"
  8 	settings     = "settings"
  9+	mirror       = "mirror"
 10 )
 11 
 12 var (
 13@@ -591,12 +592,19 @@ func (fb *FileBackend) SetProjectName(repo string, name string) error {
 14 	return os.WriteFile(filepath.Join(fb.reposPath(), repo, projectName), []byte(name), 0600)
 15 }
 16 
 17+// IsMirror returns true if the given repo is a mirror.
 18+func (fb *FileBackend) IsMirror(repo string) bool {
 19+	repo = utils.SanitizeRepo(repo) + ".git"
 20+	r := &Repo{path: filepath.Join(fb.reposPath(), repo), root: fb.reposPath()}
 21+	return r.IsMirror()
 22+}
 23+
 24 // CreateRepository creates a new repository.
 25 //
 26 // Created repositories are always bare.
 27 //
 28 // It implements backend.Backend.
 29-func (fb *FileBackend) CreateRepository(repo string, private bool) (backend.Repository, error) {
 30+func (fb *FileBackend) CreateRepository(repo string, opts backend.RepositoryOptions) (backend.Repository, error) {
 31 	name := utils.SanitizeRepo(repo)
 32 	repo = name + ".git"
 33 	rp := filepath.Join(fb.reposPath(), repo)
 34@@ -604,6 +612,20 @@ func (fb *FileBackend) CreateRepository(repo string, private bool) (backend.Repo
 35 		return nil, os.ErrExist
 36 	}
 37 
 38+	if opts.Mirror != "" {
 39+		if err := git.Clone(opts.Mirror, rp, git.CloneOptions{
 40+			Mirror: true,
 41+		}); err != nil {
 42+			logger.Debug("failed to clone mirror repository", "err", err)
 43+			return nil, err
 44+		}
 45+
 46+		if err := os.WriteFile(filepath.Join(rp, mirror), nil, 0600); err != nil {
 47+			logger.Debug("failed to create mirror file", "err", err)
 48+			return nil, err
 49+		}
 50+	}
 51+
 52 	rr, err := git.Init(rp, true)
 53 	if err != nil {
 54 		logger.Debug("failed to create repository", "err", err)
 55@@ -615,17 +637,17 @@ func (fb *FileBackend) CreateRepository(repo string, private bool) (backend.Repo
 56 		return nil, err
 57 	}
 58 
 59-	if err := fb.SetPrivate(repo, private); err != nil {
 60+	if err := fb.SetPrivate(repo, opts.Private); err != nil {
 61 		logger.Debug("failed to set private status", "err", err)
 62 		return nil, err
 63 	}
 64 
 65-	if err := fb.SetDescription(repo, ""); err != nil {
 66+	if err := fb.SetDescription(repo, opts.Description); err != nil {
 67 		logger.Debug("failed to set description", "err", err)
 68 		return nil, err
 69 	}
 70 
 71-	if err := fb.SetProjectName(repo, name); err != nil {
 72+	if err := fb.SetProjectName(repo, opts.ProjectName); err != nil {
 73 		logger.Debug("failed to set project name", "err", err)
 74 		return nil, err
 75 	}
 76diff --git a/server/backend/file/repo.go b/server/backend/file/repo.go
 77index 3888201045f5a16b39f85a373b2adfec9a46a452..93d7dab9bba91f5318006929a9225fac2d97b8e4 100644
 78--- a/server/backend/file/repo.go
 79+++ b/server/backend/file/repo.go
 80@@ -57,6 +57,14 @@ func (r *Repo) IsPrivate() bool {
 81 	return err == nil
 82 }
 83 
 84+// IsMirror returns whether the repository is a mirror.
 85+//
 86+// It implements backend.Repository.
 87+func (r *Repo) IsMirror() bool {
 88+	_, err := os.Stat(filepath.Join(r.path, mirror))
 89+	return err == nil
 90+}
 91+
 92 // Open returns the underlying git.Repository.
 93 //
 94 // It implements backend.Repository.
 95diff --git a/server/backend/repo.go b/server/backend/repo.go
 96index 9fcad1c45516d21201e65021877635f82c6320b8..b429a7a0132480f701e3fa74b0e90c3f11713271 100644
 97--- a/server/backend/repo.go
 98+++ b/server/backend/repo.go
 99@@ -5,6 +5,14 @@ import (
100 	"golang.org/x/crypto/ssh"
101 )
102 
103+// RepositoryOptions are options for creating a new repository.
104+type RepositoryOptions struct {
105+	Private     bool
106+	Mirror      string
107+	Description string
108+	ProjectName string
109+}
110+
111 // RepositoryStore is an interface for managing repositories.
112 type RepositoryStore interface {
113 	// Repository finds the given repository.
114@@ -12,7 +20,7 @@ type RepositoryStore interface {
115 	// Repositories returns a list of all repositories.
116 	Repositories() ([]Repository, error)
117 	// CreateRepository creates a new repository.
118-	CreateRepository(name string, private bool) (Repository, error)
119+	CreateRepository(name string, opts RepositoryOptions) (Repository, error)
120 	// DeleteRepository deletes a repository.
121 	DeleteRepository(name string) error
122 	// RenameRepository renames a repository.
123@@ -33,6 +41,8 @@ type RepositoryMetadata interface {
124 	IsPrivate(repo string) bool
125 	// SetPrivate sets whether the repository is private.
126 	SetPrivate(repo string, private bool) error
127+	// IsMirror returns whether the repository is a mirror.
128+	IsMirror(repo string) bool
129 }
130 
131 // RepositoryAccess is an interface for managing repository access.
132@@ -67,6 +77,8 @@ type Repository interface {
133 	Description() string
134 	// IsPrivate returns whether the repository is private.
135 	IsPrivate() bool
136+	// IsMirror returns whether the repository is a mirror.
137+	IsMirror() bool
138 	// Open returns the underlying git.Repository.
139 	Open() (*git.Repository, error)
140 }
141diff --git a/server/cmd/create.go b/server/cmd/create.go
142index d7cfdb04da9c1e5371bea6207c14308b40cacea2..891ad94f9a51301b9a502e7101c7a745f34e6da2 100644
143--- a/server/cmd/create.go
144+++ b/server/cmd/create.go
145@@ -1,11 +1,17 @@
146 package cmd
147 
148-import "github.com/spf13/cobra"
149+import (
150+	"github.com/charmbracelet/soft-serve/server/backend"
151+	"github.com/spf13/cobra"
152+)
153 
154 // createCommand is the command for creating a new repository.
155 func createCommand() *cobra.Command {
156 	var private bool
157 	var description string
158+	var mirror string
159+	var projectName string
160+
161 	cmd := &cobra.Command{
162 		Use:               "create REPOSITORY",
163 		Short:             "Create a new repository",
164@@ -14,13 +20,22 @@ func createCommand() *cobra.Command {
165 		RunE: func(cmd *cobra.Command, args []string) error {
166 			cfg, _ := fromContext(cmd)
167 			name := args[0]
168-			if _, err := cfg.Backend.CreateRepository(name, private); err != nil {
169+			if _, err := cfg.Backend.CreateRepository(name, backend.RepositoryOptions{
170+				Private:     private,
171+				Mirror:      mirror,
172+				Description: description,
173+				ProjectName: projectName,
174+			}); err != nil {
175 				return err
176 			}
177 			return nil
178 		},
179 	}
180+
181 	cmd.Flags().BoolVarP(&private, "private", "p", false, "make the repository private")
182 	cmd.Flags().StringVarP(&description, "description", "d", "", "set the repository description")
183+	cmd.Flags().StringVarP(&mirror, "mirror", "m", "", "set the mirror repository")
184+	cmd.Flags().StringVarP(&projectName, "name", "n", "", "set the project name")
185+
186 	return cmd
187 }
188diff --git a/server/cron/cron.go b/server/cron/cron.go
189new file mode 100644
190index 0000000000000000000000000000000000000000..9f98d5a69c709c624f93738bc407f510b20a45c1
191--- /dev/null
192+++ b/server/cron/cron.go
193@@ -0,0 +1,58 @@
194+package cron
195+
196+import (
197+	"context"
198+	"time"
199+
200+	"github.com/charmbracelet/log"
201+	"github.com/robfig/cron/v3"
202+)
203+
204+// CronScheduler is a cron-like job scheduler.
205+type CronScheduler struct {
206+	*cron.Cron
207+	logger cron.Logger
208+}
209+
210+// Entry is a cron job.
211+type Entry struct {
212+	ID   cron.EntryID
213+	Desc string
214+	Spec string
215+}
216+
217+// cronLogger is a wrapper around the logger to make it compatible with the
218+// cron logger.
219+type cronLogger struct {
220+	logger *log.Logger
221+}
222+
223+// Info logs routine messages about cron's operation.
224+func (l cronLogger) Info(msg string, keysAndValues ...interface{}) {
225+	l.logger.Info(msg, keysAndValues...)
226+}
227+
228+// Error logs an error condition.
229+func (l cronLogger) Error(err error, msg string, keysAndValues ...interface{}) {
230+	l.logger.Error(msg, append(keysAndValues, "err", err)...)
231+}
232+
233+// NewCronScheduler returns a new Cron.
234+func NewCronScheduler() *CronScheduler {
235+	logger := cronLogger{log.WithPrefix("server.cron")}
236+	return &CronScheduler{
237+		Cron: cron.New(cron.WithLogger(logger)),
238+	}
239+}
240+
241+// Shutdonw gracefully shuts down the CronServer.
242+func (s *CronScheduler) Shutdown() {
243+	ctx, cancel := context.WithTimeout(s.Cron.Stop(), 30*time.Second)
244+	defer func() { cancel() }()
245+	<-ctx.Done()
246+}
247+
248+// Start starts the CronServer.
249+func (s *CronScheduler) Start() {
250+	s.Cron.Start()
251+}
252diff --git a/server/jobs.go b/server/jobs.go
253new file mode 100644
254index 0000000000000000000000000000000000000000..d9621b8b064ffc4c0d3aee0202217f4d9f7b4085
255--- /dev/null
256+++ b/server/jobs.go
257@@ -0,0 +1,40 @@
258+package server
259+
260+import (
261+	"github.com/charmbracelet/soft-serve/git"
262+	"github.com/charmbracelet/soft-serve/server/backend"
263+)
264+
265+var (
266+	jobSpecs = map[string]string{
267+		"mirror": "@every 10m",
268+	}
269+)
270+
271+// mirrorJob runs the (pull) mirror job task.
272+func mirrorJob(b backend.Backend) func() {
273+	logger := logger.WithPrefix("server.mirrorJob")
274+	return func() {
275+		repos, err := b.Repositories()
276+		if err != nil {
277+			logger.Error("error getting repositories", "err", err)
278+			return
279+		}
280+
281+		for _, repo := range repos {
282+			if repo.IsMirror() {
283+				logger.Debug("updating mirror", "repo", repo.Name())
284+				r, err := repo.Open()
285+				if err != nil {
286+					logger.Error("error opening repository", "repo", repo.Name(), "err", err)
287+					continue
288+				}
289+
290+				cmd := git.NewCommand("remote", "update", "--prune")
291+				if _, err := cmd.RunInDir(r.Path); err != nil {
292+					logger.Error("error running git remote update", "repo", repo.Name(), "err", err)
293+				}
294+			}
295+		}
296+	}
297+}
298diff --git a/server/server.go b/server/server.go
299index 976b1b9fa7423976936ccf5fc03cd4c48ee6ab79..bcb5d904d75367ba4c35a2d81d72751eeb120c92 100644
300--- a/server/server.go
301+++ b/server/server.go
302@@ -11,6 +11,7 @@ import (
303 	"github.com/charmbracelet/soft-serve/server/backend"
304 	"github.com/charmbracelet/soft-serve/server/backend/file"
305 	"github.com/charmbracelet/soft-serve/server/config"
306+	"github.com/charmbracelet/soft-serve/server/cron"
307 	"github.com/charmbracelet/ssh"
308 	"golang.org/x/sync/errgroup"
309 )
310@@ -25,6 +26,7 @@ type Server struct {
311 	GitDaemon   *GitDaemon
312 	HTTPServer  *HTTPServer
313 	StatsServer *StatsServer
314+	Cron        *cron.CronScheduler
315 	Config      *config.Config
316 	Backend     backend.Backend
317 }
318@@ -57,9 +59,14 @@ func NewServer(cfg *config.Config) (*Server, error) {
319 	}
320 
321 	srv := &Server{
322+		Cron:    cron.NewCronScheduler(),
323 		Config:  cfg,
324 		Backend: cfg.Backend,
325 	}
326+
327+	// Add cron jobs.
328+	srv.Cron.AddFunc(jobSpecs["mirror"], mirrorJob(cfg.Backend))
329+
330 	srv.SSHServer, err = NewSSHServer(cfg, srv)
331 	if err != nil {
332 		return nil, err
333@@ -114,6 +121,11 @@ func (s *Server) Start() error {
334 		}
335 		return nil
336 	})
337+	errg.Go(func() error {
338+		log.Print("Starting cron scheduler")
339+		s.Cron.Start()
340+		return nil
341+	})
342 	return errg.Wait()
343 }
344 
345diff --git a/server/ssh.go b/server/ssh.go
346index 64f48a00f12144fb49704b2f38ca79e33bfdcd03..06916813d2cf7dc922b36202663f48ceb227a79f 100644
347--- a/server/ssh.go
348+++ b/server/ssh.go
349@@ -185,7 +185,7 @@ func (s *SSHServer) Middleware(cfg *config.Config) wish.Middleware {
350 							return
351 						}
352 						if _, err := cfg.Backend.Repository(name); err != nil {
353-							if _, err := cfg.Backend.CreateRepository(name, false); err != nil {
354+							if _, err := cfg.Backend.CreateRepository(name, backend.RepositoryOptions{Private: false}); err != nil {
355 								log.Errorf("failed to create repo: %s", err)
356 								sshFatal(s, err)
357 								return