Skip to content

feat: Use conditional requests for RSS feeds #462

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: dev
Choose a base branch
from
+102 −32
Open

feat: Use conditional requests for RSS feeds #462

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 101 additions & 32 deletions internal/glance/widget-rss.go
View file
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@ package glance

import (
"context"
"errors"
"fmt"
"html"
"html/template"
@@ -25,18 +26,25 @@ var (
rssWidgetHorizontalCards2Template = mustParseTemplate("rss-horizontal-cards-2.html", "widget-base.html")
)

type cachedFeed struct {
LastModified time.Time
Etag string
Items rssFeedItemList
}

type rssWidget struct {
widgetBase `yaml:",inline"`
FeedRequests []rssFeedRequest `yaml:"feeds"`
Style string `yaml:"style"`
ThumbnailHeight float64 `yaml:"thumbnail-height"`
CardHeight float64 `yaml:"card-height"`
Items rssFeedItemList `yaml:"-"`
Limit int `yaml:"limit"`
CollapseAfter int `yaml:"collapse-after"`
SingleLineTitles bool `yaml:"single-line-titles"`
PreserveOrder bool `yaml:"preserve-order"`
NoItemsMessage string `yaml:"-"`
FeedRequests []rssFeedRequest `yaml:"feeds"`
Style string `yaml:"style"`
ThumbnailHeight float64 `yaml:"thumbnail-height"`
CardHeight float64 `yaml:"card-height"`
Items rssFeedItemList `yaml:"-"`
Limit int `yaml:"limit"`
CollapseAfter int `yaml:"collapse-after"`
SingleLineTitles bool `yaml:"single-line-titles"`
PreserveOrder bool `yaml:"preserve-order"`
NoItemsMessage string `yaml:"-"`
CachedFeeds map[string]cachedFeed `yaml:"-"`
}

func (widget *rssWidget) initialize() error {
@@ -70,21 +78,41 @@ func (widget *rssWidget) initialize() error {
}

func (widget *rssWidget) update(ctx context.Context) {
items, err := fetchItemsFromRSSFeeds(widget.FeedRequests)
// Populate If-Modified-Since header and Etag
for i, req := range widget.FeedRequests {
if cachedFeed, ok := widget.CachedFeeds[req.URL]; ok {
widget.FeedRequests[i].IfModifiedSince = cachedFeed.LastModified
widget.FeedRequests[i].Etag = cachedFeed.Etag
}
}

allItems, feeds, err := fetchItemsFromRSSFeeds(widget.FeedRequests, widget.CachedFeeds)

if !widget.canContinueUpdateAfterHandlingErr(err) {
return
}

if !widget.PreserveOrder {
items.sortByNewest()
allItems.sortByNewest()
}

if len(items) > widget.Limit {
items = items[:widget.Limit]
if len(allItems) > widget.Limit {
allItems = allItems[:widget.Limit]
}

widget.Items = items
widget.Items = allItems

cachedFeeds := make(map[string]cachedFeed)
for _, feed := range feeds {
if !feed.LastModified.IsZero() || feed.Etag != "" {
cachedFeeds[feed.URL] = cachedFeed{
LastModified: feed.LastModified,
Etag: feed.Etag,
Items: feed.Items,
}
}
}
widget.CachedFeeds = cachedFeeds
}

func (widget *rssWidget) Render() template.HTML {
@@ -152,10 +180,19 @@ type rssFeedRequest struct {
ItemLinkPrefix string `yaml:"item-link-prefix"`
Headers map[string]string `yaml:"headers"`
IsDetailed bool `yaml:"-"`
IfModifiedSince time.Time `yaml:"-"`
Etag string `yaml:"-"`
}

type rssFeedItemList []rssFeedItem

type rssFeedResponse struct {
URL string
Items rssFeedItemList
LastModified time.Time
Etag string
}

func (f rssFeedItemList) sortByNewest() rssFeedItemList {
sort.Slice(f, func(i, j int) bool {
return f[i].PublishedAt.After(f[j].PublishedAt)
@@ -166,41 +203,67 @@ func (f rssFeedItemList) sortByNewest() rssFeedItemList {

var feedParser = gofeed.NewParser()

func fetchItemsFromRSSFeedTask(request rssFeedRequest) ([]rssFeedItem, error) {
func fetchItemsFromRSSFeedTask(request rssFeedRequest) (rssFeedResponse, error) {
feedResponse := rssFeedResponse{URL: request.URL}

req, err := http.NewRequest("GET", request.URL, nil)
if err != nil {
return nil, err
return feedResponse, err
}

req.Header.Add("User-Agent", fmt.Sprintf("Glance v%s", buildVersion))

for key, value := range request.Headers {
req.Header.Add(key, value)
}

if !request.IfModifiedSince.IsZero() {
req.Header.Add("If-Modified-Since", request.IfModifiedSince.Format(http.TimeFormat))
}

if request.Etag != "" {
req.Header.Add("If-None-Match", request.Etag)
}

resp, err := defaultHTTPClient.Do(req)
if err != nil {
return nil, err
return feedResponse, err
}
defer resp.Body.Close()

if resp.StatusCode == http.StatusNotModified {
return feedResponse, errNotModified
}

if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, request.URL)
return feedResponse, fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, request.URL)
}

body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
return feedResponse, err
}

feed, err := feedParser.ParseString(string(body))
if err != nil {
return nil, err
return feedResponse, err
}

if request.Limit > 0 && len(feed.Items) > request.Limit {
feed.Items = feed.Items[:request.Limit]
}

items := make(rssFeedItemList, 0, len(feed.Items))
items := make([]rssFeedItem, 0, len(feed.Items))

if lastModified := resp.Header.Get("Last-Modified"); lastModified != "" {
if t, err := time.Parse(http.TimeFormat, lastModified); err == nil {
feedResponse.LastModified = t
}
}

if etag := resp.Header.Get("Etag"); etag != "" {
feedResponse.Etag = etag
}

for i := range feed.Items {
item := feed.Items[i]
@@ -289,7 +352,8 @@ func fetchItemsFromRSSFeedTask(request rssFeedRequest) ([]rssFeedItem, error) {
items = append(items, rssItem)
}

return items, nil
feedResponse.Items = items
return feedResponse, nil
}

func recursiveFindThumbnailInExtensions(extensions map[string][]gofeedext.Extension) string {
@@ -322,33 +386,38 @@ func findThumbnailInItemExtensions(item *gofeed.Item) string {
return recursiveFindThumbnailInExtensions(media)
}

func fetchItemsFromRSSFeeds(requests []rssFeedRequest) (rssFeedItemList, error) {
func fetchItemsFromRSSFeeds(requests []rssFeedRequest, cachedFeeds map[string]cachedFeed) (rssFeedItemList, []rssFeedResponse, error) {
job := newJob(fetchItemsFromRSSFeedTask, requests).withWorkers(30)
feeds, errs, err := workerPoolDo(job)
if err != nil {
return nil, fmt.Errorf("%w: %v", errNoContent, err)
return nil, nil, fmt.Errorf("%w: %v", errNoContent, err)
}

failed := 0
notModified := 0

entries := make(rssFeedItemList, 0, len(feeds)*10)

for i := range feeds {
if errs[i] != nil {
if errs[i] == nil {
entries = append(entries, feeds[i].Items...)
} else if errors.Is(errs[i], errNotModified) {
notModified++
entries = append(entries, cachedFeeds[feeds[i].URL].Items...)
slog.Debug("Feed not modified", "url", requests[i].URL, "debug", errs[i])
} else {
failed++
slog.Error("Failed to get RSS feed", "url", requests[i].URL, "error", errs[i])
continue
}

entries = append(entries, feeds[i]...)
}

if failed == len(requests) {
return nil, errNoContent
return nil, nil, errNoContent
}

if failed > 0 {
return entries, fmt.Errorf("%w: missing %d RSS feeds", errPartialContent, failed)
return entries, feeds, fmt.Errorf("%w: missing %d RSS feeds", errPartialContent, failed)
}

return entries, nil
return entries, feeds, nil
}
1 change: 1 addition & 0 deletions internal/glance/widget-utils.go
View file
Original file line number Diff line number Diff line change
@@ -19,6 +19,7 @@ import (
var (
errNoContent = errors.New("failed to retrieve any content")
errPartialContent = errors.New("failed to retrieve some of the content")
errNotModified = errors.New("content not modified")
)

const defaultClientTimeout = 5 * time.Second