From c085b76f14a4734d0f08a4596b35de05d70ed6ac Mon Sep 17 00:00:00 2001 From: M4tthewDE Date: Fri, 8 Sep 2023 15:10:04 +0200 Subject: [PATCH 1/5] Adjust User-Agent to fetch twitter opengraph tags --- pkg/resolver/request.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/resolver/request.go b/pkg/resolver/request.go index 67f84f7f..34e6daaf 100644 --- a/pkg/resolver/request.go +++ b/pkg/resolver/request.go @@ -31,7 +31,7 @@ func RequestGET(ctx context.Context, url string) (response *http.Response, err e // ensures websites return pages in english (e.g. twitter would return french preview // when the request came from a french IP.) req.Header.Add("Accept-Language", "en-US, en;q=0.9, *;q=0.5") - req.Header.Set("User-Agent", fmt.Sprintf("chatterino-api-cache/%s link-resolver", version.Version)) + req.Header.Set("User-Agent", fmt.Sprintf("chatterino-api-cache/%s link-resolver (bot)", version.Version)) return httpClient.Do(req) } From 789e55274fad2bbef69bc55a06605738fd7e6572 Mon Sep 17 00:00:00 2001 From: M4tthewDE Date: Fri, 8 Sep 2023 15:18:11 +0200 Subject: [PATCH 2/5] Add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fdc7e22..36d3b294 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Minor: Use OG resolver for Twitter by default (#522) - Breaking: Remove the `/twitchemotes/` endpoints. See [issue 332](https://github.com/Chatterino/api/issues/332) for more information. (#465) - Fix: We do some more YouTube video ID parsing to ensure broken links (such as `youtube.com/watch?v=foobar?feature=share`) still return the actual video ID, since this is how the browser operates. (#488) - Dev: Document the `log-development` setting. (#491) From 142bf63eb1006474f1d3af0ddf3925c5234090b9 Mon Sep 17 00:00:00 2001 From: M4tthewDE Date: Fri, 8 Sep 2023 15:59:32 +0200 Subject: [PATCH 3/5] Only send adjusted User-Agent for twitter requests I really don't like how this changes twitter/resolver.go, but I haven't managed to figure out a better way to check if a request matches. --- internal/resolvers/default/link_loader.go | 10 +++++++++- internal/resolvers/twitter/resolver.go | 6 +++++- pkg/resolver/request.go | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/internal/resolvers/default/link_loader.go b/internal/resolvers/default/link_loader.go index 54ea5f3e..54148438 100644 --- a/internal/resolvers/default/link_loader.go +++ b/internal/resolvers/default/link_loader.go @@ -13,7 +13,9 @@ import ( "time" "github.com/Chatterino/api/internal/logger" + "github.com/Chatterino/api/internal/resolvers/twitter" "github.com/Chatterino/api/internal/staticresponse" + "github.com/Chatterino/api/internal/version" "github.com/Chatterino/api/pkg/cache" "github.com/Chatterino/api/pkg/resolver" "github.com/Chatterino/api/pkg/thumbnail" @@ -48,7 +50,13 @@ func (l *LinkLoader) Load(ctx context.Context, urlString string, r *http.Request return resolver.ReturnInvalidURL() } - resp, err := resolver.RequestGET(ctx, requestUrl.String()) + extraHeaders := make(map[string]string) + ctx, isTwitterRequest := twitter.Check(ctx, requestUrl) + if isTwitterRequest { + extraHeaders["User-Agent"] = fmt.Sprintf("chatterino-api-cache/%s link-resolver (bot)", version.Version) + } + + resp, err := resolver.RequestGETWithHeaders(requestUrl.String(), extraHeaders) if err != nil { if strings.HasSuffix(err.Error(), "no such host") { return staticresponse.SNoLinkInfoFound. diff --git a/internal/resolvers/twitter/resolver.go b/internal/resolvers/twitter/resolver.go index 56465a39..050bf97c 100644 --- a/internal/resolvers/twitter/resolver.go +++ b/internal/resolvers/twitter/resolver.go @@ -18,7 +18,7 @@ type TwitterResolver struct { userCache cache.Cache } -func (r *TwitterResolver) Check(ctx context.Context, url *url.URL) (context.Context, bool) { +func Check(ctx context.Context, url *url.URL) (context.Context, bool) { if !utils.IsSubdomainOf(url, "twitter.com") { return ctx, false } @@ -45,6 +45,10 @@ func (r *TwitterResolver) Check(ctx context.Context, url *url.URL) (context.Cont return ctx, isTwitterUser } +func (r *TwitterResolver) Check(ctx context.Context, url *url.URL) (context.Context, bool) { + return Check(ctx, url) +} + func (r *TwitterResolver) Run(ctx context.Context, url *url.URL, req *http.Request) (*cache.Response, error) { tweetMatch := tweetRegexp.FindStringSubmatch(url.Path) if len(tweetMatch) == 2 && len(tweetMatch[1]) > 0 { diff --git a/pkg/resolver/request.go b/pkg/resolver/request.go index 34e6daaf..67f84f7f 100644 --- a/pkg/resolver/request.go +++ b/pkg/resolver/request.go @@ -31,7 +31,7 @@ func RequestGET(ctx context.Context, url string) (response *http.Response, err e // ensures websites return pages in english (e.g. twitter would return french preview // when the request came from a french IP.) req.Header.Add("Accept-Language", "en-US, en;q=0.9, *;q=0.5") - req.Header.Set("User-Agent", fmt.Sprintf("chatterino-api-cache/%s link-resolver (bot)", version.Version)) + req.Header.Set("User-Agent", fmt.Sprintf("chatterino-api-cache/%s link-resolver", version.Version)) return httpClient.Do(req) } From e5207a3019e8ab8934d6b5d45c5922c23e025bca Mon Sep 17 00:00:00 2001 From: Rasmus Karlsson Date: Sat, 9 Sep 2023 11:50:26 +0200 Subject: [PATCH 4/5] Reword changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36d3b294..a922b4b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Minor: Use OG resolver for Twitter by default (#522) +- Minor: Use Twitter OG tags if no Twitter credentials are configured. (#522) - Breaking: Remove the `/twitchemotes/` endpoints. See [issue 332](https://github.com/Chatterino/api/issues/332) for more information. (#465) - Fix: We do some more YouTube video ID parsing to ensure broken links (such as `youtube.com/watch?v=foobar?feature=share`) still return the actual video ID, since this is how the browser operates. (#488) - Dev: Document the `log-development` setting. (#491) From c4151e08ed984e5b60d67a9a6ad5d967731d05e5 Mon Sep 17 00:00:00 2001 From: Rasmus Karlsson Date: Sat, 9 Sep 2023 11:50:34 +0200 Subject: [PATCH 5/5] Move changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a922b4b1..6d1e4e2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,8 @@ ## Unreleased -- Minor: Use Twitter OG tags if no Twitter credentials are configured. (#522) - Breaking: Remove the `/twitchemotes/` endpoints. See [issue 332](https://github.com/Chatterino/api/issues/332) for more information. (#465) +- Minor: Use Twitter OG tags if no Twitter credentials are configured. (#522) - Fix: We do some more YouTube video ID parsing to ensure broken links (such as `youtube.com/watch?v=foobar?feature=share`) still return the actual video ID, since this is how the browser operates. (#488) - Dev: Document the `log-development` setting. (#491) - Dev: Document the `log-level` setting. (#490)