75 lines
1.3 KiB
Go
75 lines
1.3 KiB
Go
package feeds
|
|
|
|
import (
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"sync"
|
|
|
|
browser "github.com/EDDYCJY/fake-useragent"
|
|
"github.com/mmcdole/gofeed"
|
|
)
|
|
|
|
type RSS struct {
|
|
Feeds []*gofeed.Feed
|
|
}
|
|
|
|
// Update fetches all articles for all feeds
|
|
func (r *RSS) Update(feeds []string) {
|
|
fp := gofeed.NewParser()
|
|
var wg sync.WaitGroup
|
|
var mux sync.Mutex
|
|
r.Feeds = []*gofeed.Feed{}
|
|
for _, f := range feeds {
|
|
f := f
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
feed, err := r.FetchURL(fp, f)
|
|
if err != nil {
|
|
log.Printf("error fetching url: %s, err: %v", f, err)
|
|
}
|
|
mux.Lock()
|
|
if feed != nil {
|
|
r.Feeds = append(r.Feeds, feed)
|
|
}
|
|
mux.Unlock()
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
}
|
|
|
|
// FetchURL fetches the feed URL and parses it
|
|
func (r *RSS) FetchURL(fp *gofeed.Parser, url string) (feed *gofeed.Feed, err error) {
|
|
client := &http.Client{}
|
|
|
|
req, err := http.NewRequest("GET", url, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
userAgent := browser.Firefox()
|
|
|
|
req.Header.Set("User-Agent", userAgent)
|
|
resp, err := client.Do(req)
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if resp != nil {
|
|
defer func() {
|
|
ce := resp.Body.Close()
|
|
if ce != nil {
|
|
err = ce
|
|
}
|
|
}()
|
|
}
|
|
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
return nil, fmt.Errorf("Failed to get url %v, %v", resp.StatusCode, resp.Status)
|
|
}
|
|
|
|
return fp.Parse(resp.Body)
|
|
}
|