ac1bc316770df0c565b5e21b201c189b2bca5167
- Author
- TheEdgeOfRage <git@theedgeofrage.com>
- Committer
- TheEdgeOfRage <git@theedgeofrage.com>
- Date
Message
Improve feed parsing concurrency and clean up auth page
Diff
This diff is truncated to protect this page.
1diff --git a/feedparser/feedparser.go b/feedparser/feedparser.go
2index cd2a1672cbd36669c1ff71042b6ad338c963e5bd..eb55097e297ced2badfc928f568b4c3cd82bb086 100644
3--- a/feedparser/feedparser.go
4+++ b/feedparser/feedparser.go
5@@ -8,7 +8,6 @@ import (
6 "log/slog"
7 "net/http"
8 "strings"
9- "sync"
10
11 "github.com/paulrosania/go-charset/charset"
12 )
13@@ -22,7 +21,6 @@ var urlFormat = "https://www.youtube.com/feeds/videos.xml?channel_id=%s"
14
15 type Parser interface {
16 Parse(channelID string) (*Channel, error)
17- ParseThreadSafe(channelID string, channelChan chan *Channel, errChan chan error, mu *sync.Mutex, wg *sync.WaitGroup)
18 }
19
20 type parser struct {
21@@ -81,17 +79,3 @@ func (p *parser) Parse(channelID string) (*Channel, error) {
22
23 return &channel, nil
24 }
25-
26-// ParseThreadSafe calls Parse, but additionally accepts an out parameter to store the result,
27-// as well as a mutex and wait group to run multiple fetches in parallel
28-func (p *parser) ParseThreadSafe(
29- channelID string, channelChan chan *Channel, errChan chan error, mu *sync.Mutex, wg *sync.WaitGroup,
30-) {
31- channel, err := p.Parse(channelID)
32-
33- mu.Lock()
34- channelChan <- channel
35- errChan <- err
36- mu.Unlock()
37- wg.Done()
38-}
39diff --git a/handler/videos.go b/handler/videos.go
40index 2c4755252e0d599879cbb1759f0523f289cbbe95..108e1bd29fd81675d33815da724238baef4c8485 100644
41--- a/handler/videos.go
42+++ b/handler/videos.go
43@@ -71,6 +71,11 @@ func (h *handler) addVideosForChannel(ctx context.Context, parsedChannel *feedpa
44 }
45 }
46
47+type parseResult struct {
48+ channel *feedparser.Channel
49+ err error
50+}
51+
52 func (h *handler) FetchVideos(ctx context.Context) error {
53 h.log.Info("Fetching new videos for all channels")
54
55@@ -78,24 +83,27 @@ func (h *handler) FetchVideos(ctx context.Context) error {
56 if err != nil {
57 return err
58 }
59- parsedChannels := make(chan *feedparser.Channel, len(channels))
60- errors := make(chan error, len(channels))
61+
62 var wg sync.WaitGroup
63- var mu sync.Mutex
64+ results := make(chan parseResult, 1)
65 for _, channel := range channels {
66- wg.Add(1)
67- go h.parser.ParseThreadSafe(channel.ID, parsedChannels, errors, &mu, &wg)
68+ wg.Go(func() {
69+ parsedChannel, err := h.parser.Parse(channel.ID)
70+ results <- parseResult{channel: parsedChannel, err: err}
71+ })
72 }
73- wg.Wait()
74
75- for _, c := range channels {
76- parsedChannel := <-parsedChannels
77- err = <-errors
78- if err != nil {
79- h.log.Error("failed to parse channel feed", "channelID", c.ID, "error", err)
80+ go func() {
81+ wg.Wait()
82+ close(results)
83+ }()
84+
85+ for result := range results {
86+ if result.err != nil {
87+ h.log.Error("failed to parse channel feed", "error", result.err)
88 continue
89 }
90- h.addVideosForChannel(ctx, parsedChannel)
91+ h.addVideosForChannel(ctx, result.channel)
92 }
93
94 return nil
95diff --git a/mocks/feedparser/feedparser.go b/mocks/feedparser/feedparser.go
96index d0506dae58080e1d8915488f73cd4c8e2641e6a5..a53efb559be161897dad9bbfed07a3e532de3da3 100644
97--- a/mocks/feedparser/feedparser.go
98+++ b/mocks/feedparser/feedparser.go
99@@ -21,9 +21,6 @@ var _ feedparser.Parser = &ParserMock{}
100 // ParseFunc: func(channelID string) (*feedparser.Channel, error) {
101 // panic("mock out the Parse method")
102 // },
103-// ParseThreadSafeFunc: func(channelID string, channelChan chan *feedparser.Channel, errChan chan error, mu *sync.Mutex, wg *sync.WaitGroup) {
104-// panic("mock out the ParseThreadSafe method")
105-// },
106 // }
107 //
108 // // use mockedParser in code that requires feedparser.Parser
109@@ -34,9 +31,6 @@ type ParserMock struct {
110 // ParseFunc mocks the Parse method.
111 ParseFunc func(channelID string) (*feedparser.Channel, error)
112
113- // ParseThreadSafeFunc mocks the ParseThreadSafe method.
114- ParseThreadSafeFunc func(channelID string, channelChan chan *feedparser.Channel, errChan chan error, mu *sync.Mutex, wg *sync.WaitGroup)
115-
116 // calls tracks calls to the methods.
117 calls struct {
118 // Parse holds details about calls to the Parse method.
119@@ -44,22 +38,8 @@ type ParserMock struct {
120 // ChannelID is the channelID argument value.
121 ChannelID string
122 }
123- // ParseThreadSafe holds details about calls to the ParseThreadSafe method.
124- ParseThreadSafe []struct {
125- // ChannelID is the channelID argument value.
126- ChannelID string
127- // ChannelChan is the channelChan argument value.
128- ChannelChan chan *feedparser.Channel
129- // ErrChan is the errChan argument value.
130- ErrChan chan error
131- // Mu is the mu argument value.
132- Mu *sync.Mutex
133- // Wg is the wg argument value.
134- Wg *sync.WaitGroup
135- }
136 }
137- lockParse sync.RWMutex
138- lockParseThreadSafe sync.RWMutex
139+ lockParse sync.RWMutex
140 }
141
142 // Parse calls ParseFunc.
143@@ -93,51 +73,3 @@ func (mock *ParserMock) ParseCalls() []struct {
144 mock.lockParse.RUnlock()
145 return calls
146 }
147-
148-// ParseThreadSafe calls ParseThreadSafeFunc.
149-func (mock *ParserMock) ParseThreadSafe(channelID string, channelChan chan *feedparser.Channel, errChan chan error, mu *sync.Mutex, wg *sync.WaitGroup) {
150- if mock.ParseThreadSafeFunc == nil {
151- panic("ParserMock.ParseThreadSafeFunc: method is nil but Parser.ParseThreadSafe was just called")
152- }
153- callInfo := struct {
154- ChannelID string
155- ChannelChan chan *feedparser.Channel
156- ErrChan chan error
157- Mu *sync.Mutex
158- Wg *sync.WaitGroup
159- }{
160- ChannelID: channelID,
161- ChannelChan: channelChan,
162- ErrChan: errChan,
163- Mu: mu,
164- Wg: wg,
165- }
166- mock.lockParseThreadSafe.Lock()
167- mock.calls.ParseThreadSafe = append(mock.calls.ParseThreadSafe, callInfo)
168- mock.lockParseThreadSafe.Unlock()
169- mock.ParseThreadSafeFunc(channelID, channelChan, errChan, mu, wg)
170-}
171-
172-// ParseThreadSafeCalls gets all the calls that were made to ParseThreadSafe.
173-// Check the length with:
174-//
175-// len(mockedParser.ParseThreadSafeCalls())
176-func (mock *ParserMock) ParseThreadSafeCalls() []struct {
177- ChannelID string
178- ChannelChan chan *feedparser.Channel
179- ErrChan chan error
180- Mu *sync.Mutex
181- Wg *sync.WaitGroup
182-} {
183- var calls []struct {
184- ChannelID string
185- ChannelChan chan *feedparser.Channel
186- ErrChan chan error
187- Mu *sync.Mutex
188- Wg *sync.WaitGroup
189- }
190- mock.lockParseThreadSafe.RLock()
191- calls = mock.calls.ParseThreadSafe
192- mock.lockParseThreadSafe.RUnlock()
193- return calls
194-}
195diff --git a/pages/navbar.templ b/pages/navbar.templ
196index d75e4ae66d51bdcb030498ac3047c8b5ecec414f..523376f900b9937fd17d2a7850e2f43eeed4e6e1 100644
197--- a/pages/navbar.templ
198+++ b/pages/navbar.templ
199@@ -33,54 +33,56 @@ templ navbar(route string) {
200 <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent">
201 <span class="navbar-toggler-icon"></span>
202 </button>
203- <div class="collapse navbar-collapse" id="navbarSupportedContent">
204- <ul class="navbar-nav me-auto mb-2 mb-lg-0">
205- <li class="nav-item me-md-2 mt-2 mt-md-0">
206- <button
207- type="button"
208- class="btn btn-outline-secondary"
209- style="--bs-btn-color: var(--bs-body-color);"
210- data-bs-toggle="modal"
211- data-bs-target="#subscription-modal"
212- >
213- Subscribe <i class="bi bi-bookmark-plus"></i>
214- </button>
215- </li>
216- <li class="nav-item me-md-2 mt-2 mt-md-0">
217- <button
218- type="button"
219- class="btn btn-outline-secondary"
220- style="--bs-btn-color: var(--bs-body-color);"
221- hx-post="/fetch"
222- hx-swap="none"
223- hx-on::after-settle="console.log(`AAAAA`); window.location.reload()"
224- >
225- Fetch <i class="bi bi-cloud-download"></i>
226- </button>
227- </li>
228- <li class="nav-item me-md-2 mt-2 mt-md-0">
229- <button
230- type="button"
231- class="btn btn-outline-secondary"
232- style="--bs-btn-color: var(--bs-body-color);"
233- data-bs-toggle="modal"
234- data-bs-target="#add-video-modal"
235- >
236- Add Video <i class="bi bi-camera-video"></i>
237- </button>
238- </li>
239- for _, button := range navButtons(route) {
240+ if route != "auth" {
241+ <div class="collapse navbar-collapse" id="navbarSupportedContent">
242+ <ul class="navbar-nav me-auto mb-2 mb-lg-0">
243 <li class="nav-item me-md-2 mt-2 mt-md-0">
244- <a href={ button.link } class="btn btn-outline-secondary" style="--bs-btn-color: var(--bs-body-color);">
245- { button.text } <i class={ fmt.Sprintf("bi bi-%s", button.icon) }></i>
246- </a>
247+ <button
248+ type="button"
249+ class="btn btn-outline-secondary"
250+ style="--bs-btn-color: var(--bs-body-color);"
251+ data-bs-toggle="modal"
252+ data-bs-target="#subscription-modal"
253+ >
254+ Subscribe <i class="bi bi-bookmark-plus"></i>
255+ </button>
256 </li>
257- }
258- </ul>
259- <div class="d-flex" role="search">
260- <input id="video-search" class="form-control" type="search" placeholder="Search videos..."/>
261+ <li class="nav-item me-md-2 mt-2 mt-md-0">
262+ <button
263+ type="button"
264+ class="btn btn-outline-secondary"
265+ style="--bs-btn-color: var(--bs-body-color);"
266+ hx-post="/fetch"
267+ hx-swap="none"
268+ hx-on::after-settle="console.log(`AAAAA`); window.location.reload()"
269+ >
270+ Fetch <i class="bi bi-cloud-download"></i>
271+ </button>
272+ </li>
273+ <li class="nav-item me-md-2 mt-2 mt-md-0">
274+ <button
275+ type="button"
276+ class="btn btn-outline-secondary"
277+ style="--bs-btn-color: var(--bs-body-color);"
278+ data-bs-toggle="modal"
279+ data-bs-target="#add-video-modal"
280+ >
281+ Add Video <i class="bi bi-camera-video"></i>
282+ </button>
283+ </li>
284+ for _, button := range navButtons(route) {
285+ <li class="nav-item me-md-2 mt-2 mt-md-0">
286+ <a href={ button.link } class="btn btn-outline-secondary" style="--bs-btn-color: var(--bs-body-color);">
287+ { button.text } <i class={ fmt.Sprintf("bi bi-%s", button.icon) }></i>
288+ </a>
289+ </li>
290+ }
291+ </ul>
292+ <div class="d-flex" role="search">
293+ <input id="video-search" class="form-control" type="search" placeholder="Search videos..."/>
294+ </div>
295 </div>
296- </div>
297+ }
298 </div>
299diff --git a/pages/navbar_templ.go b/pages/navbar_templ.go
300index 15ee96aface66cdfdefdd4ed0bec4d99141ca692..3c786db0a51a311f3defe18eea8faa304ec9e2ee 100644
301--- a/pages/navbar_templ.go
302+++ b/pages/navbar_templ.go
303@@ -53,65 +53,75 @@ func navbar(route string) templ.Component {
304 templ_7745c5c3_Var1 = templ.NopComponent
305 }
306 ctx = templ.ClearChildren(ctx)