From 82099de55bd0da029c6a489e00de8b34e06ab573 Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 2 Jun 2026 17:21:20 +0200 Subject: [PATCH 01/47] Include session.kind in all debug output Fixes #1330 --- src/apiutils.nim | 34 +++++++++++++++++++++------------- src/auth.nim | 19 +++++++++++++++++-- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/src/apiutils.nim b/src/apiutils.nim index d6952e8..6f4f727 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -130,7 +130,7 @@ template fetchImpl(result, fetchBody) {.dirty.} = raise newException(BadClientError, "Bad client") if resp.status == $Http404 and result.len == 0: - echo "[sessions] transient 404 (empty body), retrying: ", url.path + echo "[sessions] transient 404 (empty body), retrying: ", url.path, ", session: ", session.pretty raise rateLimitError() if resp.headers.hasKey(rlRemaining): @@ -147,7 +147,7 @@ template fetchImpl(result, fetchBody) {.dirty.} = if result.startsWith("{\"errors"): let errors = result.fromJson(Errors) if errors notin errorsToSkip: - echo "Fetch error, API: ", url.path, ", errors: ", errors + echo "Fetch error, API: ", url.path, ", errors: ", errors, ", session: ", session.pretty if errors in {expiredToken, badToken, locked}: invalidate(session) raise rateLimitError() @@ -162,7 +162,7 @@ template fetchImpl(result, fetchBody) {.dirty.} = fetchBody if resp.status == $Http400: - echo "ERROR 400, ", url.path, ": ", result + echo "ERROR 400, ", url.path, ": ", result, ", session: ", session.pretty raise newException(InternalError, $url) except InternalError as e: raise e @@ -177,22 +177,30 @@ template fetchImpl(result, fetchBody) {.dirty.} = finally: release(session) -template retry(bod) = +template retry(bod) {.dirty.} = + var session: Session for i in 0 ..< maxRetries: try: + session = nil bod break except RateLimitError: - echo "[sessions] Rate limited, retrying ", req.cookie.endpoint, - " request (", i, "/", maxRetries, ")..." + let api = if session.isNil: req.cookie.endpoint + else: req.endpoint(session) + if session.isNil: + echo "[sessions] Rate limited, retrying ", api, + " request (", i, "/", maxRetries, ")..." + else: + echo "[sessions] Rate limited, retrying ", api, + " request (", i, "/", maxRetries, ")..., session: ", session.pretty + session = nil if retryDelayMs > 0: await sleepAsync(retryDelayMs) proc fetch*(req: ApiReq): Future[JsonNode] {.async.} = retry: - var - body: string - session = await getAndValidateSession(req) + var body: string + session = await getAndValidateSession(req) let url = req.toUrl(session.kind) @@ -200,22 +208,22 @@ proc fetch*(req: ApiReq): Future[JsonNode] {.async.} = if body.startsWith('{') or body.startsWith('['): result = parseJson(body) else: - echo resp.status, ": ", body, " --- url: ", url + echo resp.status, ": ", body, " --- url: ", url, ", session: ", session.pretty result = newJNull() let error = result.getError if error != null and error notin errorsToSkip: - echo "Fetch error, API: ", url.path, ", error: ", error + echo "Fetch error, API: ", url.path, ", error: ", error, ", session: ", session.pretty if error in {expiredToken, badToken, locked}: invalidate(session) raise rateLimitError() proc fetchRaw*(req: ApiReq): Future[string] {.async.} = retry: - var session = await getAndValidateSession(req) + session = await getAndValidateSession(req) let url = req.toUrl(session.kind) fetchImpl result: if not (result.startsWith('{') or result.startsWith('[')): - echo resp.status, ": ", result, " --- url: ", url + echo resp.status, ": ", result, " --- url: ", url, ", session: ", session.pretty result.setLen(0) diff --git a/src/auth.nim b/src/auth.nim index d801489..259c360 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -18,7 +18,7 @@ proc setMaxConcurrentReqs*(reqs: int) = template log(str: varargs[string, `$`]) = echo "[sessions] ", str.join("") -proc endpoint(req: ApiReq; session: Session): string = +proc endpoint*(req: ApiReq; session: Session): string = case session.kind of oauth: req.oauth.endpoint of cookie: req.cookie.endpoint @@ -50,6 +50,8 @@ proc getSessionPoolHealth*(): JsonNode = oldest = now.int64 newest = 0'i64 average = 0'i64 + oauthTotal, cookieTotal = 0 + oauthLimited, cookieLimited = 0 for session in sessionPool: let created = snowflakeToEpoch(session.id) @@ -59,8 +61,15 @@ proc getSessionPoolHealth*(): JsonNode = oldest = created average += created + case session.kind + of oauth: inc oauthTotal + of cookie: inc cookieTotal + if session.limited: limited.incl session.id + case session.kind + of oauth: inc oauthLimited + of cookie: inc cookieLimited for api in session.apis.keys: let @@ -84,6 +93,8 @@ proc getSessionPoolHealth*(): JsonNode = "sessions": %*{ "total": sessionPool.len, "limited": limited.card, + "oauth": %*{"total": oauthTotal, "limited": oauthLimited}, + "cookie": %*{"total": cookieTotal, "limited": cookieLimited}, "oldest": $fromUnix(oldest), "newest": $fromUnix(newest), "average": $fromUnix(average) @@ -100,6 +111,7 @@ proc getSessionPoolDebug*(): JsonNode = for session in sessionPool: let sessionJson = %*{ + "kind": $session.kind, "apis": newJObject(), "pending": session.pending, } @@ -173,7 +185,10 @@ proc getSession*(req: ApiReq): Future[Session] {.async.} = if not result.isNil and result.isReady(req): inc result.pending else: - log "no sessions available for API: ", req.cookie.endpoint + if result.isNil: + log "no sessions available for API: ", req.cookie.endpoint + else: + log "no sessions available for API: ", req.endpoint(result), ", last tried: ", result.pretty raise noSessionsError() proc setLimited*(session: Session; req: ApiReq) = From 5a4faa03678c6d049d66fdd8c99d225fef64f18e Mon Sep 17 00:00:00 2001 From: Ian Brown Date: Tue, 2 Jun 2026 13:31:40 -0700 Subject: [PATCH 02/47] Fix OpenSearch response crash (#1400) --- src/routes/search.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/routes/search.nim b/src/routes/search.nim index 7d72f34..ba023fd 100644 --- a/src/routes/search.nim +++ b/src/routes/search.nim @@ -46,6 +46,7 @@ proc createSearchRouter*(cfg: Config) = redirect("/search?f=tweets&q=" & encodeUrl("#" & @"hash")) get "/opensearch": - let url = getUrlPrefix(cfg) & "/search?f=tweets&q=" - resp Http200, {"Content-Type": "application/opensearchdescription+xml"}, - generateOpenSearchXML(cfg.title, cfg.hostname, url) + let + url = getUrlPrefix(cfg) & "/search?f=tweets&q=" + headers = {"Content-Type": "application/opensearchdescription+xml"} + resp Http200, headers, generateOpenSearchXML(cfg.title, cfg.hostname, url) From d5ff410c5d30c27fb6450230de80d82e80fed816 Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 2 Jun 2026 23:57:06 +0200 Subject: [PATCH 03/47] Add same-origin referrer policy Fixes #1346 --- src/views/general.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/src/views/general.nim b/src/views/general.nim index 4110bdc..f753610 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -84,6 +84,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; text cfg.title meta(name="viewport", content="width=device-width, initial-scale=1.0") + meta(name="referrer", content="same-origin") meta(name="theme-color", content="#1F1F1F") meta(property="og:type", content=ogType) meta(property="og:title", content=(if ogTitle.len > 0: ogTitle else: titleText)) From 1d57f1f4323ecd4fc9c4a99040b56a4f7725b9d4 Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 5 Jun 2026 22:26:38 +0200 Subject: [PATCH 04/47] Add video attribution link support --- src/types.nim | 1 + src/views/tweet.nim | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/types.nim b/src/types.nim index 848f8ae..c755589 100644 --- a/src/types.nim +++ b/src/types.nim @@ -264,6 +264,7 @@ type stats*: TweetStats retweet*: Option[Tweet] attribution*: Option[User] + attributionLink*: string mediaTags*: seq[User] quote*: Option[Tweet] card*: Option[Card] diff --git a/src/views/tweet.nim b/src/views/tweet.nim index e15cf1d..5546eb4 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -239,8 +239,9 @@ proc renderReply(tweet: Tweet): VNode = if i > 0: text " " a(href=("/" & u)): text "@" & u -proc renderAttribution(user: User; prefs: Prefs): VNode = - buildHtml(a(class="attribution", href=("/" & user.username))): +proc renderAttribution(user: User; prefs: Prefs; link = ""): VNode = + let href = if link.len > 0: link else: "/" & user.username + buildHtml(a(class="attribution", href=href)): renderMiniAvatar(user, prefs) strong: text user.fullname verifiedIcon(user) @@ -386,7 +387,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; verbatim replaceUrls(tweet.text, prefs) & renderLocation(tweet) if tweet.attribution.isSome: - renderAttribution(tweet.attribution.get(), prefs) + renderAttribution(tweet.attribution.get(), prefs, tweet.attributionLink) if tweet.card.isSome and tweet.card.get().kind != hidden: renderCard(tweet.card.get(), prefs, path) From 083d65a8cf1d6bfc6638e8e63a7cfba3f18e8bd0 Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 5 Jun 2026 22:26:41 +0200 Subject: [PATCH 05/47] Update tests --- tests/test_card.py | 29 ++++++++++++----------------- tests/test_thread.py | 2 +- tests/test_tweet.py | 6 +++--- tests/test_tweet_media.py | 14 +++++++------- 4 files changed, 23 insertions(+), 28 deletions(-) diff --git a/tests/test_card.py b/tests/test_card.py index 129e65a..f0a916c 100644 --- a/tests/test_card.py +++ b/tests/test_card.py @@ -11,18 +11,18 @@ card = [ ['voidtarget/status/1094632512926605312', 'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too)', 'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim', - 'gist.github.com', True] + 'gist.github.com', True], + + ['NASA/status/2061872347477418301', + 'Nancy Grace Roman Space Telescope Mission - NASA Science', + 'The Nancy Grace Roman Space Telescope will settle essential questions in the areas of dark energy, exoplanets, and astrophysics.', + 'science.nasa.gov', True] ] no_thumb = [ - ['FluentAI/status/1116417904831029248', - 'LinkedIn', - 'This link will take you to a page that’s not on LinkedIn', - 'lnkd.in'], - ['Thom_Wolf/status/1122466524860702729', - 'GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in', - '', + 'GitHub - facebookresearch/XLM: PyTorch original implementation of Cross-lingual Language Model', + 'PyTorch original implementation of Cross-lingual Language Model Pretraining.', 'github.com'], ['brent_p/status/1088857328680488961', @@ -37,14 +37,9 @@ no_thumb = [ ] playable = [ - ['nim_lang/status/1118234460904919042', - 'Nim development blog 2019-03', - 'Arne (aka Krux02)* debugging: * improved nim-gdb, $ works, framefilter * alias for --debugger:native: -g* bugs: * forwarding of .pure. * sizeof union* fe...', - 'youtube.com'], - - ['nim_lang/status/1121090879823986688', - 'Nim - First natively compiled language w/ hot code-reloading at...', - '#nim #c++ #ACCUConfNim is a statically typed systems and applications programming language which offers perhaps some of the most powerful metaprogramming cap...', + ['NASA/status/2047048645845897398', + 'NASA\'s Artemis II News Conference with Moon Astronauts', + 'Live from NASA\'s Johnson Space Center in Houston', 'youtube.com'] ] @@ -72,7 +67,7 @@ class CardTest(BaseTestCase): if len(description) > 0: self.assert_text(description, c.description) - @parameterized.expand(playable) + @parameterized.expand(playable, skip_on_empty=True) def test_card_playable(self, tweet, title, description, destination): self.open_nitter(tweet) c = Card(Conversation.main + " ") diff --git a/tests/test_thread.py b/tests/test_thread.py index aa8ce32..48319bb 100644 --- a/tests/test_thread.py +++ b/tests/test_thread.py @@ -8,7 +8,7 @@ thread = [ [], "Based", ["Crystal", "Julia"], - [["yeah,"]], + [["For", "Then"], ["yeah,"]], ], ["octonion/status/975254452625002496", ["Based"], "Crystal", ["Julia"], []], ["octonion/status/975256058384887808", ["Based", "Crystal"], "Julia", [], []], diff --git a/tests/test_tweet.py b/tests/test_tweet.py index bf9e267..0fa7672 100644 --- a/tests/test_tweet.py +++ b/tests/test_tweet.py @@ -71,8 +71,8 @@ emoji = [ ] retweet = [ - [7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test', '1234'], - [3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr'] + [7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test', + 'Testing. 1234.'] ] @@ -120,7 +120,7 @@ class TweetTest(BaseTestCase): link = self.find_link_text(f'@{un}') self.assertIn(f'/{un}', link.get_property('href')) - @parameterized.expand(retweet) + @parameterized.expand(retweet, skip_on_empty=True) def test_retweet(self, index, url, retweet_by, fullname, username, text): self.open_nitter(url) tweet = get_timeline_tweet(index) diff --git a/tests/test_tweet_media.py b/tests/test_tweet_media.py index f54cea7..96f630a 100644 --- a/tests/test_tweet_media.py +++ b/tests/test_tweet_media.py @@ -28,14 +28,14 @@ video_m3u8 = [ ] gallery = [ - # ['mobile_test/status/451108446603980803', [ - # ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO'] - # ]], + ['mobile_test/status/451108446603980803', [ + ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO'] + ]], - # ['mobile_test/status/471539824713691137', [ - # ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'], - # ['Bos--IqIQAAav23'] - # ]], + ['mobile_test/status/471539824713691137', [ + ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'], + ['Bos--IqIQAAav23'] + ]], ['mobile_test/status/469530783384743936', [ ['BoQbwJAIUAA0QCY', 'BoQbwN1IMAAuTiP'], From e4e6dd13e6667bb5337194df14a031218d224219 Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 5 Jun 2026 22:26:46 +0200 Subject: [PATCH 06/47] Update API endpoints --- src/api.nim | 49 ++++++++-------- src/apiutils.nim | 7 ++- src/consts.nim | 145 +++++++++++++++++------------------------------ src/parser.nim | 48 +++++++++++++--- 4 files changed, 122 insertions(+), 127 deletions(-) diff --git a/src/api.nim b/src/api.nim index f3c6c12..c66ca3a 100644 --- a/src/api.nim +++ b/src/api.nim @@ -25,27 +25,27 @@ proc mediaUrl(id, cursor: string; count=20): ApiReq = ) proc userTweetsUrl(id: string; cursor: string): ApiReq = - result = ApiReq( - # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles), - oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"]) - ) - # might change this in the future pending testing - result.cookie = result.oauth + return apiReq(graphUserTweetsV2, restIdVars % [id, cursor, "20"]) + # result = ApiReq( + # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles), + # oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"]) + # ) proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq = - let cookieVars = userTweetsAndRepliesVars % [id, cursor] - result = ApiReq( - cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles), - oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"]) - ) + return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"]) + #let cookieVars = userTweetsAndRepliesVars % [id, cursor] + # result = ApiReq( + # cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles), + # oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"]) + # ) proc tweetDetailUrl(id: string; cursor: string): ApiReq = - let cookieVars = tweetDetailVars % [id, cursor] - result = ApiReq( - # cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles), - cookie: apiUrl(graphTweet, tweetVars % [id, cursor]), - oauth: apiUrl(graphTweet, tweetVars % [id, cursor]) - ) + return apiReq(graphTweet, tweetVars % [id, cursor]) + # let cookieVars = tweetDetailVars % [id, cursor] + # result = ApiReq( + # cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles), + # oauth: apiUrl(graphTweet, tweetVars % [id, cursor]) + # ) proc userUrl(username: string): ApiReq = let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username @@ -184,13 +184,13 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} = var variables = %*{ "rawQuery": q, - "query_source": "typedQuery", "count": 20, + "querySource": "typed_query", "product": "Latest", - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false + "withGrokTranslatedBio":true, + "withQuickPromoteEligibilityTweetFields":false } + if after.len > 0 and maxId.len == 0: variables["cursor"] = % after let @@ -212,12 +212,11 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} var variables = %*{ "rawQuery": query.text, - "query_source": "typedQuery", "count": 20, + "querySource": "typed_query", "product": "People", - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false + "withGrokTranslatedBio":true, + "withQuickPromoteEligibilityTweetFields":false } if after.len > 0: variables["cursor"] = % after diff --git a/src/apiutils.nim b/src/apiutils.nim index 6f4f727..bf6c4b9 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -84,12 +84,13 @@ proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} = result["x-twitter-auth-type"] = "OAuth2Session" result["x-csrf-token"] = session.ct0 result["cookie"] = getCookieHeader(session.authToken, session.ct0) + result["referer"] = "https://x.com/" result["sec-ch-ua"] = """"Google Chrome";v="142", "Chromium";v="142", "Not A(Brand";v="24"""" result["sec-ch-ua-mobile"] = "?0" result["sec-ch-ua-platform"] = "Windows" result["sec-fetch-dest"] = "empty" result["sec-fetch-mode"] = "cors" - result["sec-fetch-site"] = "same-site" + result["sec-fetch-site"] = "same-origin" if disableTid or "/1.1/" in url.path: result["authorization"] = bearerToken2 else: @@ -114,7 +115,9 @@ template fetchImpl(result, fetchBody) {.dirty.} = try: var resp: AsyncResponse - pool.use(await genHeaders(session, url)): + let headers = await genHeaders(session, url) + + pool.use(headers): template getContent = # TODO: this is a temporary simple implementation if apiProxy.len > 0 and "/1.1/" notin url.path: diff --git a/src/consts.nim b/src/consts.nim index 29a582b..f903528 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -7,109 +7,70 @@ const bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" bearerToken2* = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F" - graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName" - graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery" - graphUserById* = "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery" - graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2" - graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2" - graphUserTweets* = "oRJs8SLCRNRbQzuZG93_oA/UserTweets" - graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies" - graphUserMedia* = "36oKqyQ7E_9CmtONGjJRsA/UserMedia" - graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2" - graphTweet* = "b4pV7sWOe97RncwHcGESUA/ConversationTimeline" - graphTweetDetail* = "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail" - graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery" - graphTweetEditHistory* = "upS9teTSG45aljmP9oTuXA/TweetEditHistory" - graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline" - graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId" - graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug" - graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers" - graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline" - graphAboutAccount* = "zs_jFPFT78rBpXv9Z3U2YQ/AboutAccountQuery" + graphUser* = "IGgvgiOx4QZndDHuD3x9TQ/UserByScreenName" + graphUserV2* = "-ZzAG_Bckx16LMbEvHC3lg/UserResultByScreenNameQuery" + graphUserById* = "-DAaa9jPxPswYeI2fZ9rug/UserResultByIdQuery" + graphUserTweetsV2* = "PHTSTXqZYuHIeK4B1HQprQ/UserWithProfileTweetsQueryV2" + graphUserTweetsAndRepliesV2* = "AcYHjc_YAx-9_rKWdMsKvA/UserWithProfileTweetsAndRepliesQueryV2" + graphUserTweets* = "PNd0vlufvrcIwrAnBYKE9g/UserTweets" + graphUserTweetsAndReplies* = "EqtpEwt0CoQXmDfq5DKH0A/UserTweetsAndReplies" + graphUserMedia* = "g_rGPF0fLON-M9cyVjXuzA/UserMedia" + graphUserMediaV2* = "WK111rbR0vM0ZX4lyZCYjw/MediaTimelineV2" + graphTweet* = "OZMbEnEa96AN8Pq6HyTWdw/ConversationTimeline" + graphTweetDetail* = "6uCvnic3m5reVuehkvHa3w/TweetDetail" + graphTweetResult* = "xYOrBQoTlfKJJPsX76MZEw/TweetResultByIdQuery" + graphTweetEditHistory* = "MGElmrYILE8wUfI8GorUYA/TweetEditHistory" + graphSearchTimeline* = "-TFXKoMnMTKdEXcCn-eahw/SearchTimeline" - graphBroadcast* = "0nMmbMh-_JwwRRFNXkyH3Q/BroadcastQuery" + graphListById* = "t9AbdyHaJVfjL9jsODwgpQ/ListByRestId" + graphListBySlug* = "LDQpQ89B5ipR8izCKrWU0g/ListBySlug" + graphListMembers* = "EM7YRaM3gCnzDESmchA7RA/ListMembers" + graphListTweets* = "0QJtcuMzVywHGAWD6Dtjlw/ListTimeline" + graphAboutAccount* = "zUnx-DLN9dkwOkNhTLySjg/AboutAccountQuery" + + graphBroadcast* = "FJLCzpXCLPM1jUZqmM7oEA/BroadcastQuery" restLiveStream* = "1.1/live_video_stream/status/" gqlFeatures* = """{ - "android_ad_formats_media_component_render_overlay_enabled": false, - "android_graphql_skip_api_media_color_palette": false, - "android_professional_link_spotlight_display_enabled": false, - "articles_api_enabled": false, - "articles_preview_enabled": true, - "blue_business_profile_image_shape_enabled": false, - "c9s_tweet_anatomy_moderator_badge_enabled": true, - "commerce_android_shop_module_enabled": false, - "communities_web_enable_tweet_community_results_fetch": true, - "creator_subscriptions_quote_tweet_preview_enabled": false, - "creator_subscriptions_subscription_count_enabled": false, - "creator_subscriptions_tweet_preview_api_enabled": true, - "freedom_of_speech_not_reach_fetch_enabled": true, - "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, - "grok_android_analyze_trend_fetch_enabled": false, - "grok_translations_community_note_auto_translation_is_enabled": false, - "grok_translations_community_note_translation_is_enabled": false, - "grok_translations_post_auto_translation_is_enabled": false, - "grok_translations_timeline_user_bio_auto_translation_is_enabled": false, - "hidden_profile_likes_enabled": false, - "highlights_tweets_tab_ui_enabled": false, - "immersive_video_status_linkable_timestamps": false, - "interactive_text_enabled": false, - "longform_notetweets_consumption_enabled": true, - "longform_notetweets_inline_media_enabled": true, - "longform_notetweets_richtext_consumption_enabled": true, - "longform_notetweets_rich_text_read_enabled": true, - "mobile_app_spotlight_module_enabled": false, - "payments_enabled": false, - "post_ctas_fetch_enabled": true, - "premium_content_api_read_enabled": false, + "rweb_video_screen_enabled": false, + "rweb_cashtags_enabled": true, "profile_label_improvements_pcf_label_in_post_enabled": true, - "profile_label_improvements_pcf_label_in_profile_enabled": false, - "responsive_web_edit_tweet_api_enabled": true, - "responsive_web_enhance_cards_enabled": false, - "responsive_web_graphql_exclude_directive_enabled": true, - "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, + "responsive_web_profile_redirect_enabled": false, + "rweb_tipjar_consumption_enabled": false, + "verified_phone_label_enabled": false, + "creator_subscriptions_tweet_preview_api_enabled": true, "responsive_web_graphql_timeline_navigation_enabled": true, - "responsive_web_grok_analysis_button_from_backend": true, + "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, + "premium_content_api_read_enabled": false, + "communities_web_enable_tweet_community_results_fetch": true, + "c9s_tweet_anatomy_moderator_badge_enabled": true, "responsive_web_grok_analyze_button_fetch_trends_enabled": false, "responsive_web_grok_analyze_post_followups_enabled": true, + "rweb_cashtags_composer_attachment_enabled": true, + "responsive_web_jetfuel_frame": true, + "responsive_web_grok_share_attachment_enabled": true, "responsive_web_grok_annotations_enabled": true, - "responsive_web_grok_community_note_auto_translation_is_enabled": false, + "articles_preview_enabled": true, + "responsive_web_edit_tweet_api_enabled": true, + "rweb_conversational_replies_downvote_enabled": false, + "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, + "view_counts_everywhere_api_enabled": true, + "longform_notetweets_consumption_enabled": true, + "responsive_web_twitter_article_tweet_consumption_enabled": true, + "content_disclosure_indicator_enabled": true, + "content_disclosure_ai_generated_indicator_enabled": true, + "responsive_web_grok_show_grok_translated_post": true, + "responsive_web_grok_analysis_button_from_backend": true, + "post_ctas_fetch_enabled": true, + "freedom_of_speech_not_reach_fetch_enabled": true, + "standardized_nudges_misinfo": true, + "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, + "longform_notetweets_rich_text_read_enabled": true, + "longform_notetweets_inline_media_enabled": false, "responsive_web_grok_image_annotation_enabled": true, "responsive_web_grok_imagine_annotation_enabled": true, - "responsive_web_grok_share_attachment_enabled": true, - "responsive_web_grok_show_grok_translated_post": false, - "responsive_web_jetfuel_frame": true, - "responsive_web_media_download_video_enabled": false, - "responsive_web_profile_redirect_enabled": false, - "responsive_web_text_conversations_enabled": false, - "responsive_web_twitter_article_notes_tab_enabled": false, - "responsive_web_twitter_article_tweet_consumption_enabled": true, - "responsive_web_twitter_blue_verified_badge_is_enabled": true, - "rweb_lists_timeline_redesign_enabled": true, - "rweb_tipjar_consumption_enabled": true, - "rweb_video_screen_enabled": false, - "rweb_video_timestamps_enabled": false, - "spaces_2022_h2_clipping": true, - "spaces_2022_h2_spaces_communities": true, - "standardized_nudges_misinfo": true, - "subscriptions_feature_can_gift_premium": false, - "subscriptions_verification_info_enabled": true, - "subscriptions_verification_info_is_identity_verified_enabled": false, - "subscriptions_verification_info_reason_enabled": true, - "subscriptions_verification_info_verified_since_enabled": true, - "super_follow_badge_privacy_enabled": false, - "super_follow_exclusive_tweet_notifications_enabled": false, - "super_follow_tweet_api_enabled": false, - "super_follow_user_api_enabled": false, - "tweet_awards_web_tipping_enabled": false, - "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, - "tweetypie_unmention_optimization_enabled": false, - "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false, - "unified_cards_destination_url_params_enabled": false, - "verified_phone_label_enabled": false, - "vibe_api_enabled": false, - "view_counts_everywhere_api_enabled": true, - "hidden_profile_subscriptions_enabled": false + "responsive_web_grok_community_note_auto_translation_is_enabled": true, + "responsive_web_enhance_cards_enabled": false }""".replace(" ", "").replace("\n", "") tweetVars* = """{ diff --git a/src/parser.nim b/src/parser.nim index d55c546..902aebd 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -4,7 +4,7 @@ import packedjson, packedjson/deserialiser import types, parserutils, utils import experimental/parser/unifiedcard -proc parseGraphTweet(js: JsonNode): Tweet +proc parseGraphTweet*(js: JsonNode): Tweet proc parseVerifiedType(s: string; current: VerifiedType): VerifiedType = try: parseEnum[VerifiedType](s) @@ -46,10 +46,10 @@ proc parseUser(js: JsonNode; id=""): User = proc parseGraphUser(js: JsonNode): User = var user = js{"user_result", "result"} if user.isNull: - user = ? js{"user_results", "result"} + user = js{"user_results", "result"} if user.isNull: - if js{"core"}.notNull and js{"legacy"}.notNull: + if js{"core"}.notNull: user = js else: return @@ -61,6 +61,7 @@ proc parseGraphUser(js: JsonNode): User = # fallback to support UserMedia/recent GraphQL updates if result.username.len == 0: + result.id = user{"rest_id"}.getStr result.username = user{"core", "screen_name"}.getStr result.fullname = user{"core", "name"}.getStr result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "") @@ -261,6 +262,20 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) = durationMs: mediaInfo{"duration_millis"}.getInt, variants: parseVideoVariants(mediaInfo{"variants"}) )) + + # Parse source user for video attribution + with sourceUser, mediaEntity{"source_user_results", "result"}: + if result.attribution.isNone: + let + expanded = mediaEntity{"expanded_url"}.getStr + pathStart = expanded.find('/', expanded.find("://") + 3) + if pathStart >= 0: + result.attributionLink = expanded[pathStart .. ^1].replace("/video/1", "") + result.attribution = some(User( + id: sourceUser{"rest_id"}.getStr, + fullname: sourceUser{"core", "name"}.getStr, + userPic: sourceUser{"avatar", "image_url"}.getImageStr.replace("_normal", "") + )) of "ApiGif": parsedMedia.addMedia(Gif( url: mediaInfo{"variants"}[0]{"url"}.getImageStr, @@ -428,13 +443,13 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull(); # graphql with rt, js{"retweeted_status_result", "result"}: # needed due to weird edgecase where the actual tweet data isn't included - if "legacy" in rt: + if "legacy" in rt or "rest_id" in rt: result.retweet = some parseGraphTweet(rt) return with reposts, js{"repostedStatusResults"}: with rt, reposts{"result"}: - if "legacy" in rt: + if "legacy" in rt or "rest_id" in rt: result.retweet = some parseGraphTweet(rt) return @@ -449,7 +464,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull(); result.poll = some parsePoll(jsCard) elif name == "amplify": result.media.addMedia(parsePromoVideo(jsCard{"binding_values"})) - else: + elif name.len > 0 and jsCard{"binding_values"}.notNull: result.card = some parseCard(jsCard, js{"entities", "urls"}) result.expandTweetEntities(js) @@ -469,7 +484,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull(); result.text.removeSuffix(" Learn more.") result.available = false -proc parseGraphTweet(js: JsonNode): Tweet = +proc parseGraphTweet*(js: JsonNode): Tweet = if js.kind == JNull: return Tweet() @@ -537,10 +552,21 @@ proc parseGraphTweet(js: JsonNode): Tweet = result.poll = some parsePoll(jsCard) elif name == "amplify": result.media.addMedia(parsePromoVideo(jsCard{"binding_values"})) - else: + elif name.len > 0 and jsCard{"binding_values"}.notNull: result.card = some parseCard(jsCard, js{"url_entities"}) result.expandTweetEntitiesV2(js) + + # Strip video source URL from text (for videos from other tweets) + with mediaEntities, js{"media_entities"}: + for m in mediaEntities: + if "source_status_id_str" in m: + let mediaUrl = m{"url"}.getStr + if mediaUrl.len > 0: + let idx = result.text.rfind(mediaUrl) + if idx >= 0: + result.text = result.text[0 ..< idx].strip() + break else: result = parseTweet(js{"legacy"}, jsCard, replyId) result.id = js{"rest_id"}.getId @@ -559,6 +585,12 @@ proc parseGraphTweet(js: JsonNode): Tweet = parseMediaEntities(js, result) + # Handle retweets - check both legacy and top-level paths + with reposts, js{"legacy", "repostedStatusResults"}: + with rt, reposts{"result"}: + if "legacy" in rt or "rest_id" in rt: + result.retweet = some parseGraphTweet(rt) + with quoted, js{"quoted_status_result", "result"}: result.quote = some(parseGraphTweet(quoted)) From c956f7c37374721aafe28a62b22798f513b39c6f Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 5 Jun 2026 23:54:39 +0200 Subject: [PATCH 07/47] Add //about tests --- tests/test_about_account.py | 76 +++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 tests/test_about_account.py diff --git a/tests/test_about_account.py b/tests/test_about_account.py new file mode 100644 index 0000000..2239d9d --- /dev/null +++ b/tests/test_about_account.py @@ -0,0 +1,76 @@ +from base import BaseTestCase, Profile +from parameterized import parameterized + + +class AboutAccount(object): + header = '.about-account-header' + name = '.about-account-name' + body = '.about-account-body' + row = '.about-account-row' + label = '.about-account-label' + value = '.about-account-value' + + +# (username, expected_labels) +# Each label is checked for presence in the page text +about_data = [ + ['jack', ['Date joined', 'Account based in', 'Connected via']], + ['NASA', ['Date joined']], + ['elonmusk', ['Date joined']], +] + +about_verified = [ + ['jack', 'Verified', 'Since '], +] + +about_affiliate = [ + ['jack', 'An affiliate of', 'Square'], + ['elonmusk', 'An affiliate of', 'X'], +] + + +class AboutAccountTest(BaseTestCase): + @parameterized.expand(about_data) + def test_about_page_has_labels(self, username, expected_labels): + """About page shows expected info labels""" + self.open_nitter(f'{username}/about') + self.assert_element_visible(AboutAccount.header) + self.assert_element_visible(AboutAccount.body) + for label in expected_labels: + self.assert_text(label, AboutAccount.body) + + @parameterized.expand(about_verified) + def test_about_verified(self, username, label, value_prefix): + """About page shows verification info for verified accounts""" + self.open_nitter(f'{username}/about') + self.assert_text(label, AboutAccount.body) + self.assert_text(value_prefix, AboutAccount.body) + + @parameterized.expand(about_affiliate) + def test_about_affiliate(self, username, label, affiliate): + """About page shows affiliate info""" + self.open_nitter(f'{username}/about') + self.assert_text(label, AboutAccount.body) + self.assert_text(f'@{affiliate}', AboutAccount.body) + + def test_about_page_title(self): + """Title contains account name""" + self.open_nitter('jack/about') + self.assert_text('jack', AboutAccount.name) + + def test_about_join_date(self): + """About page always shows join date""" + self.open_nitter('jack/about') + self.assert_text('Date joined', AboutAccount.body) + self.assert_text('March 2006', AboutAccount.body) + + def test_about_invalid_user(self): + """About page for non-existent user shows error""" + self.open_nitter('thisprofiledoesntexist/about') + self.assert_text('User "thisprofiledoesntexist" not found') + + def test_joindate_links_to_about(self): + """Join date on profile page links to about page""" + self.open_nitter('jack') + link = self.find_element(Profile.joinDate + ' a') + self.assertIn('/jack/about', link.get_attribute('href')) From a86be15f852eadfd95b7441db97f1605a709e768 Mon Sep 17 00:00:00 2001 From: Zed Date: Sat, 6 Jun 2026 00:14:28 +0200 Subject: [PATCH 08/47] Fix broken 2.2.x build --- .github/workflows/run-tests.yml | 8 ++++---- nitter.nimble | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index c86087a..f70e8e5 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -33,9 +33,9 @@ jobs: uses: actions/cache@v5 with: path: ~/.nimble - key: ${{ matrix.nim }}-nimble-v2-${{ hashFiles('*.nimble') }} + key: ${{ matrix.nim }}-nimble-v3-${{ hashFiles('*.nimble') }} restore-keys: | - ${{ matrix.nim }}-nimble-v2- + ${{ matrix.nim }}-nimble-v3- - name: Setup Nim uses: jiro4989/setup-nim-action@v2 @@ -104,9 +104,9 @@ jobs: uses: actions/cache@v5 with: path: ~/.nimble - key: 2.2.x-nimble-v2-${{ hashFiles('*.nimble') }} + key: 2.2.x-nimble-v3-${{ hashFiles('*.nimble') }} restore-keys: | - 2.2.x-nimble-v2- + 2.2.x-nimble-v3- - name: Setup Nim uses: jiro4989/setup-nim-action@v2 diff --git a/nitter.nimble b/nitter.nimble index c206105..8e17353 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -18,7 +18,7 @@ requires "nimcrypto#a079df9" requires "markdown#158efe3" requires "packedjson#9e6fbb6" requires "supersnappy#6c94198" -requires "redpool#8b7c1db" +requires "https://github.com/zedeus/redpool#8b7c1db" requires "https://github.com/zedeus/redis#d0a0e6f" requires "zippy#ca5989a" requires "flatty#e668085" From 6ab2143df0fd2576b6e8ba7f96104a8b130325eb Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 7 Jun 2026 00:14:43 +0200 Subject: [PATCH 09/47] Add note about creating nitter.conf for Docker Fixes #1392 --- README.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 05c2be4..c14cd23 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,17 @@ To run Nitter with Docker, you'll need to install and run Redis separately before you can run the container. See below for how to also run Redis using Docker. +First create your config file. The Docker commands mount it into the container, +so it has to exist on the host beforehand. If you've cloned the repo: + +```bash +cp nitter.example.conf nitter.conf +``` + +If you're using the prebuilt image without a local clone, download +[`nitter.example.conf`](https://raw.githubusercontent.com/zedeus/nitter/master/nitter.example.conf) +and save it as `nitter.conf` instead. + To build and run Nitter in Docker: ```bash @@ -151,8 +162,11 @@ Change `redisHost` from `localhost` to `nitter-redis` in `nitter.conf`, then run docker-compose up -d ``` -Note the Docker commands expect a `nitter.conf` file in the directory you run -them. +Note the Docker commands mount `nitter.conf` (and `sessions.jsonl` for +docker-compose) from the directory you run them in. If a mounted file doesn't +exist, Docker silently creates a directory in its place and the container fails +with `not a directory: Are you trying to mount a directory onto a file`. Remove +that directory and create the file as shown above. ### systemd From 40d17bf042900e385a4461a177f1983add150036 Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 7 Jun 2026 00:43:29 +0200 Subject: [PATCH 10/47] Use Nim 2.2.6 for Docker builds, unify arm64 nimlang/nim's alpine-regular images cap at 2.2.6 and are now multi-arch, while Alpine's apk nim is stuck at the segfaulting 2.2.0. Base both arches on 2.2.6-alpine-regular, drop the separate Dockerfile.arm64, and build ./Dockerfile in the arm64 CI job. Fixes #1404 --- .github/workflows/build-docker.yml | 2 +- Dockerfile | 4 ++-- Dockerfile.arm64 | 25 ------------------------- 3 files changed, 3 insertions(+), 28 deletions(-) delete mode 100644 Dockerfile.arm64 diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 7a46257..9a6c490 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -55,7 +55,7 @@ jobs: uses: docker/build-push-action@v3 with: context: . - file: ./Dockerfile.arm64 + file: ./Dockerfile platforms: linux/arm64 push: true tags: zedeus/nitter:latest-arm64,zedeus/nitter:${{ github.sha }}-arm64 diff --git a/Dockerfile b/Dockerfile index ab442ba..251b63a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM nimlang/nim:2.2.0-alpine-regular as nim +FROM nimlang/nim:2.2.6-alpine-regular as nim LABEL maintainer="setenforce@protonmail.com" RUN apk --no-cache add libsass-dev pcre @@ -15,7 +15,7 @@ RUN nimble build -d:danger -d:lto -d:strip --mm:refc \ FROM alpine:latest WORKDIR /src/ -RUN apk --no-cache add pcre ca-certificates +RUN apk --no-cache add pcre ca-certificates openssl COPY --from=nim /src/nitter/nitter ./ COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf COPY --from=nim /src/nitter/public ./public diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 deleted file mode 100644 index 46352c7..0000000 --- a/Dockerfile.arm64 +++ /dev/null @@ -1,25 +0,0 @@ -FROM alpine:3.20.6 as nim -LABEL maintainer="setenforce@protonmail.com" - -RUN apk --no-cache add libsass-dev pcre gcc git libc-dev nim nimble - -WORKDIR /src/nitter - -COPY nitter.nimble . -RUN nimble install -y --depsOnly - -COPY . . -RUN nimble build -d:danger -d:lto -d:strip --mm:refc \ - && nimble scss \ - && nimble md - -FROM alpine:3.20.6 -WORKDIR /src/ -RUN apk --no-cache add pcre ca-certificates openssl -COPY --from=nim /src/nitter/nitter ./ -COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf -COPY --from=nim /src/nitter/public ./public -EXPOSE 8080 -RUN adduser -h /src/ -D -s /bin/sh nitter -USER nitter -CMD ./nitter From f629507537707478c7a948760964b74b3ac54200 Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 7 Jun 2026 01:02:47 +0200 Subject: [PATCH 11/47] Publish a single multi-arch Docker image Build amd64 and arm64 natively (no emulation), push each by digest, then merge them into one multi-arch manifest so `zedeus/nitter:latest` and `:` resolve to the right image on any CPU. Replaces the separate `latest-arm64` tag, which is no longer needed. Update the README notes accordingly. --- .github/workflows/build-docker.yml | 120 +++++++++++++++++++++-------- README.md | 4 +- 2 files changed, 88 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 9a6c490..f50cad7 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -7,55 +7,109 @@ on: branches: - master +env: + IMAGE: zedeus/nitter + jobs: tests: uses: ./.github/workflows/run-tests.yml secrets: inherit - build-docker-amd64: + # Build each architecture natively (no emulation) and push by digest only. + # The digests are stitched into a single multi-arch tag by the merge job. + build: needs: [tests] + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + platform: linux/amd64 + - runner: ubuntu-24.04-arm + platform: linux/arm64 + runs-on: ${{ matrix.runner }} + steps: + - name: Prepare platform name + run: echo "PLATFORM_PAIR=${platform//\//-}" >> "$GITHUB_ENV" + env: + platform: ${{ matrix.platform }} + + - uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + version: latest + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + platforms: ${{ matrix.platform }} + outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true + # Attestations turn a single-platform push into a manifest index, which + # breaks push-by-digest + the imagetools merge below. Disable them. + provenance: false + sbom: false + + - name: Export digest + run: | + mkdir -p "${{ runner.temp }}/digests" + digest="${{ steps.build.outputs.digest }}" + touch "${{ runner.temp }}/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ env.PLATFORM_PAIR }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 + + # Combine the per-arch digests into one multi-arch manifest so that + # `docker pull zedeus/nitter:latest` serves the right image on any CPU. + merge: + needs: [build] runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 + - name: Download digests + uses: actions/download-artifact@v4 with: - version: latest - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and push AMD64 Docker image - uses: docker/build-push-action@v3 - with: - context: . - file: ./Dockerfile - platforms: linux/amd64 - push: true - tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }} + path: ${{ runner.temp }}/digests + pattern: digests-* + merge-multiple: true - build-docker-arm64: - needs: [tests] - runs-on: ubuntu-24.04-arm - steps: - - uses: actions/checkout@v6 - name: Set up Docker Buildx - id: buildx uses: docker/setup-buildx-action@v3 with: version: latest + - name: Login to DockerHub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and push ARM64 Docker image - uses: docker/build-push-action@v3 - with: - context: . - file: ./Dockerfile - platforms: linux/arm64 - push: true - tags: zedeus/nitter:latest-arm64,zedeus/nitter:${{ github.sha }}-arm64 + + - name: Create manifest list and push + working-directory: ${{ runner.temp }}/digests + run: | + # latest-arm64 is a backward-compat alias of the (now multi-arch) + # latest tag, for users still pinned to the old ARM64-only image. + # word splitting is intentional: one image ref arg per digest file + # shellcheck disable=SC2046 + docker buildx imagetools create \ + -t ${{ env.IMAGE }}:latest \ + -t ${{ env.IMAGE }}:latest-arm64 \ + -t ${{ env.IMAGE }}:${{ github.sha }} \ + $(printf '${{ env.IMAGE }}@sha256:%s ' *) + + - name: Inspect image + run: docker buildx imagetools inspect ${{ env.IMAGE }}:${{ github.sha }} diff --git a/README.md b/README.md index c14cd23..eabd2ac 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ performance reasons. Page for the Docker image: https://hub.docker.com/r/zedeus/nitter -#### NOTE: For ARM64 support, please use the separate ARM64 docker image: [`zedeus/nitter:latest-arm64`](https://hub.docker.com/r/zedeus/nitter/tags). +#### NOTE: The published image is multi-arch — `zedeus/nitter:latest` runs natively on both `amd64` and `arm64`. To run Nitter with Docker, you'll need to install and run Redis separately before you can run the container. See below for how to also run Redis using @@ -147,8 +147,6 @@ docker build -t nitter:latest . docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest ``` -Note: For ARM64, use this Dockerfile: [`Dockerfile.arm64`](https://github.com/zedeus/nitter/blob/master/Dockerfile.arm64). - A prebuilt Docker image is provided as well: ```bash From a8bc1bbb2d0828b932f5a6198e94f07b98bc210d Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 7 Jun 2026 03:41:40 +0200 Subject: [PATCH 12/47] Fix nimble dependency caching in CI --- .github/workflows/build-docker.yml | 12 ++++-------- .github/workflows/run-tests.yml | 25 +++++++++++++++---------- nitter.nimble | 3 +-- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index f50cad7..20a15a0 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -7,6 +7,10 @@ on: branches: - master +concurrency: + group: docker-publish-${{ github.ref }} + cancel-in-progress: true + env: IMAGE: zedeus/nitter @@ -15,8 +19,6 @@ jobs: uses: ./.github/workflows/run-tests.yml secrets: inherit - # Build each architecture natively (no emulation) and push by digest only. - # The digests are stitched into a single multi-arch tag by the merge job. build: needs: [tests] strategy: @@ -55,8 +57,6 @@ jobs: file: ./Dockerfile platforms: ${{ matrix.platform }} outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true - # Attestations turn a single-platform push into a manifest index, which - # breaks push-by-digest + the imagetools merge below. Disable them. provenance: false sbom: false @@ -101,10 +101,6 @@ jobs: - name: Create manifest list and push working-directory: ${{ runner.temp }}/digests run: | - # latest-arm64 is a backward-compat alias of the (now multi-arch) - # latest tag, for users still pinned to the old ARM64-only image. - # word splitting is intentional: one image ref arg per digest file - # shellcheck disable=SC2046 docker buildx imagetools create \ -t ${{ env.IMAGE }}:latest \ -t ${{ env.IMAGE }}:latest-arm64 \ diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index f70e8e5..dcfff52 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -32,10 +32,12 @@ jobs: id: cache-nimble uses: actions/cache@v5 with: - path: ~/.nimble - key: ${{ matrix.nim }}-nimble-v3-${{ hashFiles('*.nimble') }} + path: | + ~/.nimble/pkgcache + ~/.nimble/packages_official.json + key: ${{ matrix.nim }}-nimble-v6-${{ hashFiles('*.nimble') }} restore-keys: | - ${{ matrix.nim }}-nimble-v3- + ${{ matrix.nim }}-nimble-v6- - name: Setup Nim uses: jiro4989/setup-nim-action@v2 @@ -103,10 +105,12 @@ jobs: - name: Cache Nimble Dependencies uses: actions/cache@v5 with: - path: ~/.nimble - key: 2.2.x-nimble-v3-${{ hashFiles('*.nimble') }} + path: | + ~/.nimble/pkgcache + ~/.nimble/packages_official.json + key: 2.2.x-nimble-v6-${{ hashFiles('*.nimble') }} restore-keys: | - 2.2.x-nimble-v3- + 2.2.x-nimble-v6- - name: Setup Nim uses: jiro4989/setup-nim-action@v2 @@ -115,6 +119,9 @@ jobs: use-nightlies: true repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install Nimble dependencies + run: nimble install -y --depsOnly + - name: Download 2.2.x build artifact uses: actions/download-artifact@v4 with: @@ -130,10 +137,8 @@ jobs: sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf sed -i 's/maxRetries = 1/maxRetries = 10/g' nitter.conf - # Run both Nimble tasks concurrently - nim r tools/rendermd.nim & - nim r tools/gencss.nim & - wait + nim r tools/rendermd.nim + nim r tools/gencss.nim echo '${{ secrets.SESSIONS }}' | head -n1 echo '${{ secrets.SESSIONS }}' > ./sessions.jsonl diff --git a/nitter.nimble b/nitter.nimble index 8e17353..ba0a8c2 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -18,8 +18,7 @@ requires "nimcrypto#a079df9" requires "markdown#158efe3" requires "packedjson#9e6fbb6" requires "supersnappy#6c94198" -requires "https://github.com/zedeus/redpool#8b7c1db" -requires "https://github.com/zedeus/redis#d0a0e6f" +requires "redpool >= 0.2.0" requires "zippy#ca5989a" requires "flatty#e668085" requires "jsony#1de1f08" From 55d067957c7fbb5bcf88388bd165e6c6eab08be9 Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 7 Jun 2026 23:17:43 +0200 Subject: [PATCH 13/47] Fix arm32 compilation Fixes #1389 --- src/parser.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parser.nim b/src/parser.nim index 902aebd..c34ffa9 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -521,7 +521,7 @@ proc parseGraphTweet*(js: JsonNode): Tweet = "binding_values": %bindingObj } - var replyId = 0 + var replyId: int64 = 0 with restId, js{"reply_to_results", "rest_id"}: replyId = restId.getId From 9b9c86a15c24874262b0b70441d5b7a05cca0158 Mon Sep 17 00:00:00 2001 From: Zed Date: Mon, 8 Jun 2026 00:47:42 +0200 Subject: [PATCH 14/47] Use Nimble local deps to fix deployment issue Fixes #1405 --- .gitignore | 3 +++ README.md | 6 +++--- config.nims | 4 ++++ nitter.nimble | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 09bdaa4..2e52163 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ sessions.json* dump.rdb *.bak /tools/*.json* +nimbledeps/ +nimble.paths +nimble.develop diff --git a/README.md b/README.md index eabd2ac..0b0a14f 100644 --- a/README.md +++ b/README.md @@ -104,9 +104,9 @@ along with the scss and md files. # su nitter $ git clone https://github.com/zedeus/nitter $ cd nitter -$ nimble build -d:danger --mm:refc -$ nimble scss -$ nimble md +$ nimble -l build -d:danger --mm:refc +$ nimble -l scss +$ nimble -l md $ cp nitter.example.conf nitter.conf ``` diff --git a/config.nims b/config.nims index 4a7af27..3ee4842 100644 --- a/config.nims +++ b/config.nims @@ -11,3 +11,7 @@ warning("HoleEnumConv", off) hint("XDeclaredButNotUsed", off) hint("XCannotRaiseY", off) hint("User", off) +# begin Nimble config (version 2) +when withDir(thisDir(), system.fileExists("nimble.paths")): + include "nimble.paths" +# end Nimble config diff --git a/nitter.nimble b/nitter.nimble index ba0a8c2..90ec7b0 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -18,7 +18,7 @@ requires "nimcrypto#a079df9" requires "markdown#158efe3" requires "packedjson#9e6fbb6" requires "supersnappy#6c94198" -requires "redpool >= 0.2.0" +requires "redpool == 0.2.2" requires "zippy#ca5989a" requires "flatty#e668085" requires "jsony#1de1f08" From ef1de42593d6a2f0ef2ee0bbf5800605a0a35fe7 Mon Sep 17 00:00:00 2001 From: Zed Date: Mon, 8 Jun 2026 06:12:52 +0200 Subject: [PATCH 15/47] Update dependencies to version tags --- nitter.nimble | 20 ++++++++++---------- src/apiutils.nim | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/nitter.nimble b/nitter.nimble index 90ec7b0..b36f498 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -11,18 +11,18 @@ bin = @["nitter"] # Dependencies requires "nim >= 2.0.0" -requires "jester#baca3f" -requires "karax#5cf360c" -requires "sass#7dfdd03" -requires "nimcrypto#a079df9" -requires "markdown#158efe3" +requires "jester == 0.6.0" +requires "karax == 1.5.0" +requires "sass == 0.2.0" +requires "nimcrypto == 0.7.3" +requires "markdown == 0.8.8" requires "packedjson#9e6fbb6" -requires "supersnappy#6c94198" +requires "supersnappy == 2.1.4" requires "redpool == 0.2.2" -requires "zippy#ca5989a" -requires "flatty#e668085" -requires "jsony#1de1f08" -requires "oauth#b8c163b" +requires "zippy == 0.10.19" +requires "flatty == 0.4.0" +requires "jsony == 1.1.6" +requires "oauth == 0.11" # Tasks diff --git a/src/apiutils.nim b/src/apiutils.nim index bf6c4b9..6ceab6f 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -1,6 +1,6 @@ # SPDX-License-Identifier: AGPL-3.0-only import httpclient, asyncdispatch, options, strutils, uri, times, math, tables -import jsony, packedjson, zippy, oauth1 +import jsony, packedjson, zippy, oauth/oauth1 import types, auth, consts, parserutils, http_pool, tid import experimental/types/common From 60cb10229fe18c5f6b59e6d4e554a86b884ea7bb Mon Sep 17 00:00:00 2001 From: Zed Date: Mon, 8 Jun 2026 06:12:58 +0200 Subject: [PATCH 16/47] Fix SameSite cookie handling for HTTP --- src/routes/router_utils.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/routes/router_utils.nim b/src/routes/router_utils.nim index 379280c..8a26193 100644 --- a/src/routes/router_utils.nim +++ b/src/routes/router_utils.nim @@ -8,8 +8,9 @@ export utils, prefs, types, uri template savePref*(pref, value: string; req: Request; expire=false) = if not expire or pref in cookies(req): + let sameSite = if cfg.useHttps: None else: Lax setCookie(pref, value, daysForward(when expire: -10 else: 360), - httpOnly=true, secure=cfg.useHttps, sameSite=None, path="/") + httpOnly=true, secure=cfg.useHttps, sameSite=sameSite, path="/") template requestPrefs*(): untyped {.dirty.} = getPrefs(cookies(request), params(request)) From fb9107cff6d00be2cac1acd5945b5088329b46e9 Mon Sep 17 00:00:00 2001 From: Zed Date: Mon, 8 Jun 2026 06:14:18 +0200 Subject: [PATCH 17/47] Fix crash on malformed request paths --- src/nitter.nim | 9 +++++-- tests/test_security.py | 60 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 tests/test_security.py diff --git a/src/nitter.nim b/src/nitter.nim index d1c0ef1..685b608 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -2,7 +2,7 @@ import asyncdispatch, strformat, logging from net import Port from htmlgen import a -from os import getEnv +from os import getEnv, normalizedPath import jester @@ -63,12 +63,17 @@ createDebugRouter(cfg) settings: port = Port(cfg.port) - staticDir = cfg.staticDir + staticDir = normalizedPath(cfg.staticDir) bindAddr = cfg.address reusePort = true + maxBody = 64 * 1024 routes: before: + # Reject malformed paths + if request.path.len == 0 or request.path[0] != '/': + halt Http400 + # skip all file URLs cond "." notin request.path applyUrlPrefs() diff --git a/tests/test_security.py b/tests/test_security.py new file mode 100644 index 0000000..65d5e9a --- /dev/null +++ b/tests/test_security.py @@ -0,0 +1,60 @@ +import subprocess +from parameterized import parameterized + +BASE_URL = 'http://localhost:8080' + + +def curl_status(url): + """Get HTTP status code using curl to avoid URL normalization by Python libs.""" + result = subprocess.run( + ['curl', '-s', '-o', '/dev/null', '-w', '%{http_code}', url], + capture_output=True, text=True, timeout=10 + ) + return int(result.stdout) + + +class TestMalformedPaths: + """Test that malformed paths don't crash the server. + + URLs like //foo are parsed as having 'foo' as the authority (host), + resulting in an empty path. Empty paths previously crashed jester's + static file handler. Now they return 400. + + URLs like //foo/bar are parsed as authority='foo', path='/bar', + so they route normally (not empty path). + """ + + @parameterized.expand([ + # These parse to empty paths -> 400 + ('//lefty_rae', 400), + ('//test', 400), + ('//anyuser', 400), + ]) + def test_empty_path_returns_400(self, path, expected_status): + """URLs that parse to empty paths should return 400, not crash.""" + status = curl_status(f'{BASE_URL}{path}') + assert status == expected_status, \ + f'Expected {expected_status} for {path}, got {status}' + + @parameterized.expand([ + ('/jack', 200), + ('/about', 200), + ('/', 200), + ]) + def test_normal_paths_work(self, path, expected_status): + """Normal paths should still work.""" + status = curl_status(f'{BASE_URL}{path}') + assert status == expected_status, \ + f'Expected {expected_status} for {path}, got {status}' + + def test_server_survives_malformed_requests(self): + """Server should handle malformed requests without crashing.""" + # These all parse to empty paths + malformed_paths = ['//a', '//b', '//c', '//user', '//test'] + for path in malformed_paths: + status = curl_status(f'{BASE_URL}{path}') + assert status == 400, f'Expected 400 for {path}, got {status}' + + # Verify server is still responding after malformed requests + status = curl_status(f'{BASE_URL}/') + assert status == 200, 'Server should still be alive' From 5a4c8dd12c616958b59dfd9259fcb7befc9b3ff9 Mon Sep 17 00:00:00 2001 From: Zed Date: Mon, 8 Jun 2026 23:59:50 +0200 Subject: [PATCH 18/47] Improve retry failure handling Fixes #1388 --- src/apiutils.nim | 4 ++++ src/redis_cache.nim | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/apiutils.nim b/src/apiutils.nim index 6ceab6f..5fbd17c 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -182,10 +182,12 @@ template fetchImpl(result, fetchBody) {.dirty.} = template retry(bod) {.dirty.} = var session: Session + var retrySuccess = false for i in 0 ..< maxRetries: try: session = nil bod + retrySuccess = true break except RateLimitError: let api = if session.isNil: req.cookie.endpoint @@ -199,6 +201,8 @@ template retry(bod) {.dirty.} = session = nil if retryDelayMs > 0: await sleepAsync(retryDelayMs) + if not retrySuccess: + raise rateLimitError() proc fetch*(req: ApiReq): Future[JsonNode] {.async.} = retry: diff --git a/src/redis_cache.nim b/src/redis_cache.nim index bfd271f..e503e46 100644 --- a/src/redis_cache.nim +++ b/src/redis_cache.nim @@ -144,9 +144,10 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} = else: let user = await getGraphUserById(userId) result = user.username - await setEx(key, baseCacheTime, result) - if result.len > 0 and user.id.len > 0: - await all(cacheUserId(result, user.id), cache(user)) + if result.len > 0: + await setEx(key, baseCacheTime, result) + if user.id.len > 0: + await all(cacheUserId(result, user.id), cache(user)) # proc getCachedTweet*(id: int64): Future[Tweet] {.async.} = # if id == 0: return From 1e33ca045d199e84aaebc965ed54a8dfd719079b Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 10 Jun 2026 06:20:01 +0200 Subject: [PATCH 19/47] Fix restIdVars whitespace handling --- src/consts.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/consts.nim b/src/consts.nim index f903528..beefa57 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -104,7 +104,7 @@ const restIdVars* = """{ "rest_id": "$1", $2 "count": $3 -}""" +}""".replace(" ", "").replace("\n", "") userMediaVars* = """{ "userId": "$1", $2 From ac2f93b3619ed7604539f0bf1c5dc132cfd38b7f Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 10 Jun 2026 06:20:05 +0200 Subject: [PATCH 20/47] Add skipTid field for cookie session TID handling --- src/api.nim | 15 +++++---------- src/apiutils.nim | 9 ++++++--- src/types.nim | 1 + 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/api.nim b/src/api.nim index c66ca3a..404a01e 100644 --- a/src/api.nim +++ b/src/api.nim @@ -11,11 +11,11 @@ proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] = if fieldToggles.len > 0: result.add ("fieldToggles", fieldToggles) -proc apiUrl(endpoint, variables: string; fieldToggles = ""): ApiUrl = - return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles)) +proc apiUrl(endpoint, variables: string; fieldToggles = ""; skipTid = false): ApiUrl = + return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles), skipTid: skipTid) -proc apiReq(endpoint, variables: string; fieldToggles = ""): ApiReq = - let url = apiUrl(endpoint, variables, fieldToggles) +proc apiReq(endpoint, variables: string; fieldToggles = ""; skipTid = false): ApiReq = + let url = apiUrl(endpoint, variables, fieldToggles, skipTid) return ApiReq(cookie: url, oauth: url) proc mediaUrl(id, cursor: string; count=20): ApiReq = @@ -32,12 +32,7 @@ proc userTweetsUrl(id: string; cursor: string): ApiReq = # ) proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq = - return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"]) - #let cookieVars = userTweetsAndRepliesVars % [id, cursor] - # result = ApiReq( - # cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles), - # oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"]) - # ) + return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"], skipTid=true) proc tweetDetailUrl(id: string; cursor: string): ApiReq = return apiReq(graphTweet, tweetVars % [id, cursor]) diff --git a/src/apiutils.nim b/src/apiutils.nim index 5fbd17c..8e1a852 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -63,7 +63,7 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string = proc getCookieHeader(authToken, ct0: string): string = "auth_token=" & authToken & "; ct0=" & ct0 -proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} = +proc genHeaders*(session: Session, url: Uri, skipTid: bool): Future[HttpHeaders] {.async.} = result = newHttpHeaders({ "accept": "*/*", "accept-encoding": "gzip", @@ -91,7 +91,7 @@ proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} = result["sec-fetch-dest"] = "empty" result["sec-fetch-mode"] = "cors" result["sec-fetch-site"] = "same-origin" - if disableTid or "/1.1/" in url.path: + if disableTid or skipTid or "/1.1/" in url.path: result["authorization"] = bearerToken2 else: result["authorization"] = bearerToken @@ -115,7 +115,10 @@ template fetchImpl(result, fetchBody) {.dirty.} = try: var resp: AsyncResponse - let headers = await genHeaders(session, url) + let skipTid = case session.kind + of oauth: req.oauth.skipTid + of cookie: req.cookie.skipTid + let headers = await genHeaders(session, url, skipTid) pool.use(headers): template getContent = diff --git a/src/types.nim b/src/types.nim index c755589..3604efb 100644 --- a/src/types.nim +++ b/src/types.nim @@ -16,6 +16,7 @@ type ApiUrl* = object endpoint*: string params*: seq[(string, string)] + skipTid*: bool ApiReq* = object oauth*: ApiUrl From 63891a04ff5154dd471869325b2acb5d54c4d376 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 10 Jun 2026 06:20:08 +0200 Subject: [PATCH 21/47] Update tests --- tests/test_card.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_card.py b/tests/test_card.py index f0a916c..16863ea 100644 --- a/tests/test_card.py +++ b/tests/test_card.py @@ -14,7 +14,7 @@ card = [ 'gist.github.com', True], ['NASA/status/2061872347477418301', - 'Nancy Grace Roman Space Telescope Mission - NASA Science', + 'Nancy Grace Roman Space Telescope - NASA Science', 'The Nancy Grace Roman Space Telescope will settle essential questions in the areas of dark energy, exoplanets, and astrophysics.', 'science.nasa.gov', True] ] From ac5ba9469eb52cf310a2cca9828351a3ab598a37 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 10 Jun 2026 14:28:01 +0200 Subject: [PATCH 22/47] Fix incorrectly shown Twitter video cards Fixes #1407 --- src/parser.nim | 28 +++++++++++++++++++++------- src/parserutils.nim | 11 ++++++++--- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/parser.nim b/src/parser.nim index c34ffa9..097d316 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -1,5 +1,5 @@ # SPDX-License-Identifier: AGPL-3.0-only -import strutils, options, times, math, tables +import strutils, options, times, math, tables, uri import packedjson, packedjson/deserialiser import types, parserutils, utils import experimental/parser/unifiedcard @@ -229,6 +229,10 @@ proc parseLegacyMediaEntities(js: JsonNode; result: var Tweet) = result.attribution = some(parseUser(user)) else: result.attribution = some(parseGraphUser(user)) + # Set attribution link from expanded_url (strip /video/N suffix) + let expanded = m{"expanded_url"}.getStr + if expanded.len > 0: + result.attributionLink = expanded.parseUri.path.replace("/video/1", "") of "animated_gif": result.media.addMedia(Gif( url: m{"video_info", "variants"}[0]{"url"}.getImageStr, @@ -266,11 +270,9 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) = # Parse source user for video attribution with sourceUser, mediaEntity{"source_user_results", "result"}: if result.attribution.isNone: - let - expanded = mediaEntity{"expanded_url"}.getStr - pathStart = expanded.find('/', expanded.find("://") + 3) - if pathStart >= 0: - result.attributionLink = expanded[pathStart .. ^1].replace("/video/1", "") + let expanded = mediaEntity{"expanded_url"}.getStr + if expanded.len > 0: + result.attributionLink = expanded.parseUri.path.replace("/video/1", "") result.attribution = some(User( id: sourceUser{"rest_id"}.getStr, fullname: sourceUser{"core", "name"}.getStr, @@ -467,8 +469,8 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull(); elif name.len > 0 and jsCard{"binding_values"}.notNull: result.card = some parseCard(jsCard, js{"entities", "urls"}) - result.expandTweetEntities(js) parseLegacyMediaEntities(js, result) + result.expandTweetEntities(js) with jsWithheld, js{"withheld_in_countries"}: let withheldInCountries: seq[string] = @@ -555,6 +557,10 @@ proc parseGraphTweet*(js: JsonNode): Tweet = elif name.len > 0 and jsCard{"binding_values"}.notNull: result.card = some parseCard(jsCard, js{"url_entities"}) + parseMediaEntities(js, result) + if result.attribution.isNone: + parseLegacyMediaEntities(js{"legacy"}, result) + result.expandTweetEntitiesV2(js) # Strip video source URL from text (for videos from other tweets) @@ -585,6 +591,14 @@ proc parseGraphTweet*(js: JsonNode): Tweet = parseMediaEntities(js, result) + # Hide card if it's redundant with attribution (same video shown via embed) + if result.attribution.isSome and result.card.isSome: + let cardUri = get(result.card).url.parseUri + if cardUri.isTwitterUrl: + let cardPath = cardUri.path.replace("/video/1", "") + if cardPath.len > 0 and cardPath == result.attributionLink: + get(result.card).kind = hidden + # Handle retweets - check both legacy and top-level paths with reposts, js{"legacy", "repostedStatusResults"}: with rt, reposts{"result"}: diff --git a/src/parserutils.nim b/src/parserutils.nim index 8d6ea2e..bb20425 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -319,6 +319,7 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = textSlice = textRange{0}.getInt .. textRange{1}.getInt hasQuote = js{"is_quote_status"}.getBool hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails + hasAttribution = tweet.attribution.isSome var replyTo = "" if tweet.replyId != 0: @@ -326,7 +327,8 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = replyTo = reply.getStr tweet.reply.add replyTo - tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote or hasJobCard) + tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, + hasQuote or hasJobCard or hasAttribution) proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: Slice[int]; hasRedundantLink=false) = @@ -377,16 +379,19 @@ proc expandTweetEntitiesV2*(tweet: Tweet; js: JsonNode) = textSlice = textRange{0}.getInt .. textRange{1}.getInt hasQuote = "quoted_tweet_results" in js hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails + hasAttribution = tweet.attribution.isSome - tweet.expandTextEntitiesV2(js, tweet.text, textSlice, hasQuote or hasJobCard) + tweet.expandTextEntitiesV2(js, tweet.text, textSlice, + hasQuote or hasJobCard or hasAttribution) proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) = let entities = ? js{"entity_set"} text = js{"text"}.getStr.multiReplace(("<", unicodeOpen), (">", unicodeClose)) textSlice = 0..text.runeLen + hasAttribution = tweet.attribution.isSome - tweet.expandTextEntities(entities, text, textSlice) + tweet.expandTextEntities(entities, text, textSlice, hasRedundantLink=hasAttribution) tweet.text = tweet.text.multiReplace((unicodeOpen, xmlOpen), (unicodeClose, xmlClose)) From 7eed720894abf3496b9db3c7071be821fb40a7c7 Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 10 Jun 2026 14:54:36 +0200 Subject: [PATCH 23/47] Fix entity expansion crash, simplify URL handling Fixes #1409 --- src/parser.nim | 21 +-------------------- src/parserutils.nim | 3 +-- temp | 0 3 files changed, 2 insertions(+), 22 deletions(-) create mode 100644 temp diff --git a/src/parser.nim b/src/parser.nim index 097d316..78d40f7 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -241,11 +241,6 @@ proc parseLegacyMediaEntities(js: JsonNode; result: var Tweet) = )) else: discard - with url, m{"url"}: - if result.text.endsWith(url.getStr): - result.text.removeSuffix(url.getStr) - result.text = result.text.strip() - proc parseMediaEntities(js: JsonNode; result: var Tweet) = with mediaEntities, js{"media_entities"}: var parsedMedia: MediaEntities @@ -286,23 +281,9 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) = )) else: discard - if "expanded_url" in mediaEntity: - let expandedUrl = js.getExpandedUrl - if result.text.endsWith(expandedUrl): - result.text.removeSuffix(expandedUrl) - result.text = result.text.strip() - if mediaEntities.len > 0 and parsedMedia.len == mediaEntities.len: result.media = parsedMedia - # Remove media URLs from text - with mediaList, js{"legacy", "entities", "media"}: - for url in mediaList: - let expandedUrl = url.getExpandedUrl - if result.text.endsWith(expandedUrl): - result.text.removeSuffix(expandedUrl) - result.text = result.text.strip() - proc parsePromoVideo(js: JsonNode): Video = result = Video( thumb: js{"player_image_large"}.getImageVal, @@ -469,8 +450,8 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull(); elif name.len > 0 and jsCard{"binding_values"}.notNull: result.card = some parseCard(jsCard, js{"entities", "urls"}) - parseLegacyMediaEntities(js, result) result.expandTweetEntities(js) + parseLegacyMediaEntities(js, result) with jsWithheld, js{"withheld_in_countries"}: let withheldInCountries: seq[string] = diff --git a/src/parserutils.nim b/src/parserutils.nim index bb20425..e7479d5 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -319,7 +319,6 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = textSlice = textRange{0}.getInt .. textRange{1}.getInt hasQuote = js{"is_quote_status"}.getBool hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails - hasAttribution = tweet.attribution.isSome var replyTo = "" if tweet.replyId != 0: @@ -328,7 +327,7 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = tweet.reply.add replyTo tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, - hasQuote or hasJobCard or hasAttribution) + hasQuote or hasJobCard) proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: Slice[int]; hasRedundantLink=false) = diff --git a/temp b/temp new file mode 100644 index 0000000..e69de29 From bd9d492d3619916c86036f7b9a1b8ed02d8e8905 Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 11 Jun 2026 23:26:21 +0200 Subject: [PATCH 24/47] Fix bounds checking Fixes #1410 --- src/formatters.nim | 18 +++++++++++++----- src/parserutils.nim | 31 ++++++++++++++++++++++--------- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/formatters.nim b/src/formatters.nim index aef1c12..958e518 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -140,25 +140,30 @@ proc pageDesc*(user: User): string = "The latest tweets from " & user.fullname proc getJoinDate*(user: User): string = + if user.joinDate.year == 0: return "" user.joinDate.format("'Joined' MMMM YYYY") proc getJoinDateFull*(user: User): string = + if user.joinDate.year == 0: return "" user.joinDate.format("h:mm tt - d MMM YYYY") proc getTime*(tweet: Tweet): string = + if tweet.time.year == 0: return "" tweet.time.format("MMM d', 'YYYY' · 'h:mm tt' UTC'") proc getRfc822Time*(tweet: Tweet): string = + if tweet.time.year == 0: return "" tweet.time.format("ddd', 'dd MMM yyyy HH:mm:ss 'GMT'") -proc getShortTime*(tweet: Tweet): string = +proc getShortTime*(time: DateTime): string = + if time.year == 0: return "" let now = now() - let since = now - tweet.time + let since = now - time - if now.year != tweet.time.year: - result = tweet.time.format("d MMM yyyy") + if now.year != time.year: + result = time.format("d MMM yyyy") elif since.inDays >= 1: - result = tweet.time.format("MMM d") + result = time.format("MMM d") elif since.inHours >= 1: result = $since.inHours & "h" elif since.inMinutes >= 1: @@ -168,6 +173,9 @@ proc getShortTime*(tweet: Tweet): string = else: result = "now" +proc getShortTime*(tweet: Tweet): string = + getShortTime(tweet.time) + proc getDuration*(ms: int): string = let sec = int(round(ms / 1000)) diff --git a/src/parserutils.nim b/src/parserutils.nim index e7479d5..860c9d2 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -202,19 +202,32 @@ proc extractHashtags(result: var seq[ReplaceSlice]; js: JsonNode) = proc replacedWith(runes: seq[Rune]; repls: openArray[ReplaceSlice]; textSlice: Slice[int]): string = + let + runeLen = runes.len + safeStart = max(0, textSlice.a) + safeEnd = min(runeLen, textSlice.b) + + var validRepls: seq[ReplaceSlice] + for rep in repls: + if rep.slice.a >= 0 and rep.slice.b >= 0 and rep.slice.b < runeLen and rep.slice.a <= rep.slice.b: + validRepls.add rep + template extractLowerBound(i: int; idx): int = - if i > 0: repls[idx].slice.b.succ else: textSlice.a + if i > 0: min(validRepls[idx].slice.b.succ, runeLen) else: safeStart result = newStringOfCap(runes.len) - for i, rep in repls: - result.add $runes[extractLowerBound(i, i - 1) ..< rep.slice.a] + for i, rep in validRepls: + let lower = extractLowerBound(i, i - 1) + if lower < rep.slice.a: + result.add $runes[lower ..< rep.slice.a] case rep.kind of rkHashtag: - let - name = $runes[rep.slice.a.succ .. rep.slice.b] - symbol = $runes[rep.slice.a] - result.add a(symbol & name, href = "/search?f=tweets&q=%23" & name) + if rep.slice.a.succ <= rep.slice.b: + let + name = $runes[rep.slice.a.succ .. rep.slice.b] + symbol = $runes[rep.slice.a] + result.add a(symbol & name, href = "/search?f=tweets&q=%23" & name) of rkMention: result.add a($runes[rep.slice], href = rep.url, title = rep.display) of rkUrl: @@ -222,8 +235,8 @@ proc replacedWith(runes: seq[Rune]; repls: openArray[ReplaceSlice]; of rkRemove: discard - let rest = extractLowerBound(repls.len, ^1) ..< textSlice.b - if rest.a <= rest.b: + let rest = extractLowerBound(validRepls.len, ^1) ..< safeEnd + if rest.a >= 0 and rest.a <= rest.b and rest.b <= runeLen: result.add $runes[rest] proc deduplicate(s: var seq[ReplaceSlice]) = From 35882ed88d422b1355b66a1ff8c1144bffdc7bdf Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 16 Jun 2026 14:37:23 +0200 Subject: [PATCH 25/47] Add Twitter Article rendering Fixes #646 --- src/api.nim | 25 ++- src/consts.nim | 25 ++- src/experimental/parser.nim | 4 +- src/experimental/parser/article.nim | 87 ++++++++ src/experimental/types/article.nim | 78 +++++++ src/nitter.nim | 4 +- src/parser.nim | 23 ++- src/parserutils.nim | 18 +- src/routes/article.nim | 48 +++++ src/sass/_article.scss | 272 ++++++++++++++++++++++++ src/sass/index.scss | 1 + src/types.nim | 44 ++++ src/views/article.nim | 246 ++++++++++++++++++++++ src/views/general.nim | 9 +- src/views/tweet.nim | 25 ++- tests/test_article.py | 307 ++++++++++++++++++++++++++++ 16 files changed, 1192 insertions(+), 24 deletions(-) create mode 100644 src/experimental/parser/article.nim create mode 100644 src/experimental/types/article.nim create mode 100644 src/routes/article.nim create mode 100644 src/sass/_article.scss create mode 100644 src/views/article.nim create mode 100644 tests/test_article.py diff --git a/src/api.nim b/src/api.nim index 404a01e..2702f7c 100644 --- a/src/api.nim +++ b/src/api.nim @@ -2,7 +2,7 @@ import asyncdispatch, httpclient, strutils, sequtils, sugar import packedjson import types, query, formatters, consts, apiutils, parser, utils -import experimental/parser as newParser +import experimental/parser # Helper to generate params object for GraphQL requests proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] = @@ -25,14 +25,10 @@ proc mediaUrl(id, cursor: string; count=20): ApiReq = ) proc userTweetsUrl(id: string; cursor: string): ApiReq = - return apiReq(graphUserTweetsV2, restIdVars % [id, cursor, "20"]) - # result = ApiReq( - # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles), - # oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"]) - # ) + return apiReq(graphUserTweetsV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles) proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq = - return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"], skipTid=true) + return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles, skipTid=true) proc tweetDetailUrl(id: string; cursor: string): ApiReq = return apiReq(graphTweet, tweetVars % [id, cursor]) @@ -228,6 +224,21 @@ proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} = let js = await fetch(mediaUrl(id, "", 30)) result = parseGraphPhotoRail(js) +proc getGraphArticle*(id: string): Future[Article] {.async.} = + if id.len == 0: return + let + url = apiReq(graphTweetResultByRestId, articleVars % id, articleFieldToggles) + json = await fetchRaw(url) + result = parseGraphArticle(json) + +proc getGraphTweetResults*(ids: seq[string]): Future[seq[Tweet]] {.async.} = + if ids.len == 0: return + let + idsJson = "[" & ids.mapIt("\"" & it & "\"").join(",") & "]" + url = apiReq(graphTweetResultsByRestIds, articleBatchVars % idsJson, articleFieldToggles) + js = await fetch(url) + result = parseGraphTweetResults(js) + proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} = let client = newAsyncHttpClient(maxRedirects=0) try: diff --git a/src/consts.nim b/src/consts.nim index beefa57..5ac3c32 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -10,7 +10,7 @@ const graphUser* = "IGgvgiOx4QZndDHuD3x9TQ/UserByScreenName" graphUserV2* = "-ZzAG_Bckx16LMbEvHC3lg/UserResultByScreenNameQuery" graphUserById* = "-DAaa9jPxPswYeI2fZ9rug/UserResultByIdQuery" - graphUserTweetsV2* = "PHTSTXqZYuHIeK4B1HQprQ/UserWithProfileTweetsQueryV2" + graphUserTweetsV2* = "LE3eTyeqhBh2g-fX85O2eQ/UserWithProfileTweetsQueryV2" graphUserTweetsAndRepliesV2* = "AcYHjc_YAx-9_rKWdMsKvA/UserWithProfileTweetsAndRepliesQueryV2" graphUserTweets* = "PNd0vlufvrcIwrAnBYKE9g/UserTweets" graphUserTweetsAndReplies* = "EqtpEwt0CoQXmDfq5DKH0A/UserTweetsAndReplies" @@ -28,6 +28,9 @@ const graphListTweets* = "0QJtcuMzVywHGAWD6Dtjlw/ListTimeline" graphAboutAccount* = "zUnx-DLN9dkwOkNhTLySjg/AboutAccountQuery" + graphTweetResultByRestId* = "qtXMy1p5Y62uCskc_NUPJw/TweetResultByRestId" + graphTweetResultsByRestIds* = "Sc9EUQTZNEH-wzegn-nHvQ/TweetResultsByRestIds" + graphBroadcast* = "FJLCzpXCLPM1jUZqmM7oEA/BroadcastQuery" restLiveStream* = "1.1/live_video_stream/status/" @@ -131,6 +134,24 @@ const "withVoice": true }""".replace(" ", "").replace("\n", "") + articleVars* = """{ + "tweetId": "$1", + "includePromotedContent": false, + "withBirdwatchNotes": true, + "withVoice": true, + "withCommunity": true +}""".replace(" ", "").replace("\n", "") + + articleBatchVars* = """{ + "tweetIds": $1, + "includePromotedContent": false, + "withBirdwatchNotes": true, + "withVoice": true, + "withCommunity": true +}""".replace(" ", "").replace("\n", "") + + articleFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withArticleSummaryText":true,"withArticleVoiceOver":true}""" + userFieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}""" - userTweetsFieldToggles* = """{"withArticlePlainText":false}""" + userTweetsFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false}""" tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}""" diff --git a/src/experimental/parser.nim b/src/experimental/parser.nim index 40986f5..e22a51f 100644 --- a/src/experimental/parser.nim +++ b/src/experimental/parser.nim @@ -1,2 +1,2 @@ -import parser/[user, graphql] -export user, graphql +import parser/[user, graphql, article] +export user, graphql, article diff --git a/src/experimental/parser/article.nim b/src/experimental/parser/article.nim new file mode 100644 index 0000000..2ae7f4a --- /dev/null +++ b/src/experimental/parser/article.nim @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import std/[strutils, tables, times, options] +import jsony +import utils, graphql, ../types/article +from ../../types import Article, ArticleParagraph, ArticleEntity, ArticleMedia, + User, TweetStats + +proc parseGraphArticle*(json: string): Article = + if json.len == 0 or json[0] != '{': + return + + var raw: GraphArticle + try: + raw = json.fromJson(GraphArticle) + except CatchableError: + return + + let + tweet = raw.data.tweetResult.result + article = tweet.article.articleResults.result + + if article.title.len == 0: + return + + let publishedAt = article.metadata.firstPublishedAtSecs + var articleTime: DateTime + if publishedAt > 0: + articleTime = publishedAt.int64.fromUnix.utc + elif tweet.legacy.createdAt.len > 0: + articleTime = parseTwitterDate(tweet.legacy.createdAt) + + result = Article( + title: article.title, + coverImage: getImageUrl(article.coverMedia.mediaInfo.originalImgUrl), + time: articleTime, + user: parseUserResult(tweet.core.userResults.result), + ) + + result.stats = TweetStats( + replies: tweet.legacy.replyCount, + retweets: tweet.legacy.retweetCount, + likes: tweet.legacy.favoriteCount, + ) + if tweet.views.count.len > 0: + try: result.stats.views = parseInt(tweet.views.count) + except ValueError: discard + + for blk in article.contentState.blocks: + result.paragraphs.add ArticleParagraph( + text: blk.text, + kind: blk.blockKind, + inlineStyles: blk.inlineStyleRanges, + entityRanges: blk.entityRanges, + ) + + for entry in article.contentState.entityMap: + let key = try: parseInt(entry.key) except ValueError: continue + var entity = ArticleEntity(kind: entry.value.entityKind) + case entity.kind + of "LINK": entity.url = entry.value.data.url + of "MEDIA": + for mi in entry.value.data.mediaItems: + entity.mediaIds.add mi.mediaId + of "TWEET": entity.tweetId = entry.value.data.tweetId + of "MARKDOWN": entity.markdown = entry.value.data.markdown + else: discard + result.entities[key] = entity + + for me in article.mediaEntities: + let typeName = me.mediaInfo.typeName + var media = ArticleMedia(kind: typeName) + if me.mediaInfo.videoInfo.isSome: + let variants = me.mediaInfo.videoInfo.get.variants + case typeName + of "ApiGif": + if variants.len > 0: + media.url = variants[0].url + of "ApiVideo": + var bestBitrate = -1 + for v in variants: + if v.bitrate > bestBitrate: + bestBitrate = v.bitrate + media.url = v.url + else: discard + elif typeName == "ApiImage": + media.url = getImageUrl(me.mediaInfo.originalImgUrl) + result.media[me.mediaId] = media diff --git a/src/experimental/types/article.nim b/src/experimental/types/article.nim new file mode 100644 index 0000000..024c69f --- /dev/null +++ b/src/experimental/types/article.nim @@ -0,0 +1,78 @@ +import std/options +import graphuser +from ../../types import ArticleStyle, ArticleEntityRange + +type + GraphArticle* = object + data*: tuple[tweetResult: tuple[result: TweetResultNode]] + + TweetResultNode* = object + article*: tuple[articleResults: tuple[result: ArticleResultNode]] + legacy*: TweetLegacy + core*: tuple[userResults: UserData] + views*: tuple[count: string] + + TweetLegacy* = object + createdAt*: string + replyCount*: int + retweetCount*: int + favoriteCount*: int + + ArticleResultNode* = object + title*: string + coverMedia*: tuple[mediaInfo: MediaInfoNode] + contentState*: ContentState + metadata*: tuple[firstPublishedAtSecs: int] + mediaEntities*: seq[RawMediaEntity] + + ContentState* = object + blocks*: seq[ContentBlock] + entityMap*: seq[EntityMapEntry] + + ContentBlock* = object + text*: string + blockKind*: string + inlineStyleRanges*: seq[ArticleStyle] + entityRanges*: seq[ArticleEntityRange] + + EntityMapEntry* = object + key*: string + value*: EntityMapValue + + EntityMapValue* = object + entityKind*: string + data*: EntityDataNode + + EntityDataNode* = object + url*: string + mediaItems*: seq[tuple[mediaId: string]] + tweetId*: string + markdown*: string + + RawMediaEntity* = object + mediaId*: string + mediaInfo*: MediaInfoNode + + MediaInfoNode* = object + typeName*: string + originalImgUrl*: string + videoInfo*: Option[VideoInfoNode] + + VideoInfoNode* = object + variants*: seq[VideoVariant] + + VideoVariant* = object + url*: string + bitrate*: int + +proc renameHook*(v: var ContentBlock; fieldName: var string) = + if fieldName == "type": + fieldName = "blockKind" + +proc renameHook*(v: var EntityMapValue; fieldName: var string) = + if fieldName == "type": + fieldName = "entityKind" + +proc renameHook*(v: var MediaInfoNode; fieldName: var string) = + if fieldName == "__typename": + fieldName = "typeName" diff --git a/src/nitter.nim b/src/nitter.nim index 685b608..78cace4 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -10,7 +10,7 @@ import types, config, prefs, formatters, redis_cache, http_pool, auth, apiutils import views/[general, about] import routes/[ preferences, timeline, status, media, search, rss, list, debug, - unsupported, embed, resolver, broadcast, router_utils] + unsupported, embed, resolver, broadcast, article, router_utils] const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances" const issuesUrl = "https://github.com/zedeus/nitter/issues" @@ -48,6 +48,7 @@ waitFor initRedisPool(cfg) stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n" stdout.flushFile +createArticleRouter(cfg) createUnsupportedRouter(cfg) createResolverRouter(cfg) createPrefRouter(cfg) @@ -118,6 +119,7 @@ routes: resp Http429, showError( &"Instance has no auth tokens, or is fully rate limited.
Use {link} or try again later.", cfg) + extend articleRoute, "" extend rss, "" extend status, "" extend search, "" diff --git a/src/parser.nim b/src/parser.nim index 78d40f7..d7b3f8d 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -542,7 +542,8 @@ proc parseGraphTweet*(js: JsonNode): Tweet = if result.attribution.isNone: parseLegacyMediaEntities(js{"legacy"}, result) - result.expandTweetEntitiesV2(js) + let hasArticle = js{"article", "article_results", "result", "title"}.getStr.len > 0 + result.expandTweetEntitiesV2(js, hasArticle) # Strip video source URL from text (for videos from other tweets) with mediaEntities, js{"media_entities"}: @@ -558,6 +559,16 @@ proc parseGraphTweet*(js: JsonNode): Tweet = result = parseTweet(js{"legacy"}, jsCard, replyId) result.id = js{"rest_id"}.getId + with artNode, js{"article", "article_results", "result"}: + let artTitle = artNode{"title"}.getStr + if artTitle.len > 0: + result.articlePreview = some ArticlePreview( + title: artTitle, + previewText: artNode{"preview_text"}.getStr, + coverImage: artNode{"cover_media_results", "result", "media_info", "original_img_url"}.getImageStr, + tweetId: result.id + ) + result.user = parseGraphUser(js{"core"}) if result.reply.len == 0: @@ -627,6 +638,16 @@ proc parseGraphTweetResult*(js: JsonNode): Tweet = with tweet, js{"data", "tweet_result", "result"}: result = parseGraphTweet(tweet) +proc parseGraphTweetResults*(js: JsonNode): seq[Tweet] = + let results = js{"data", "tweetResult"} + if results.kind != JArray: return + for item in results: + let tweet = item{"result"} + if tweet.isNull: continue + let t = parseGraphTweet(tweet) + if t != nil: + result.add t + proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = result = Conversation(replies: Result[Chain](beginning: true)) diff --git a/src/parserutils.nim b/src/parserutils.nim index 860c9d2..07aa088 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -185,12 +185,16 @@ proc extractSlice(js: JsonNode): Slice[int] = result = js["indices"][0].getInt ..< js["indices"][1].getInt proc extractUrls(result: var seq[ReplaceSlice]; js: JsonNode; - textLen: int; hideTwitter = false) = + textLen: int; hideTwitter = false; + hideArticle = false) = let url = js.getExpandedUrl slice = js.extractSlice - if hideTwitter and slice.b.succ >= textLen and url.isTwitterUrl: + if hideArticle and url.isTwitterUrl and "/article/" in url: + if slice.a < textLen: + result.add ReplaceSlice(kind: rkRemove, slice: slice) + elif hideTwitter and slice.b.succ >= textLen and url.isTwitterUrl: if slice.a < textLen: result.add ReplaceSlice(kind: rkRemove, slice: slice) else: @@ -343,7 +347,7 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = hasQuote or hasJobCard) proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: Slice[int]; - hasRedundantLink=false) = + hasRedundantLink=false; hasArticle=false) = let hasCard = tweet.card.isSome var replacements = newSeq[ReplaceSlice]() @@ -354,7 +358,8 @@ proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: S if urlStr.len == 0 or urlStr notin text: continue - replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink) + replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink, + hideArticle = hasArticle) if hasCard and u{"url"}.getStr == get(tweet.card).url: get(tweet.card).url = u.getExpandedUrl @@ -385,7 +390,7 @@ proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: S tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false) -proc expandTweetEntitiesV2*(tweet: Tweet; js: JsonNode) = +proc expandTweetEntitiesV2*(tweet: Tweet; js: JsonNode; hasArticle=false) = let textRange = js{"details", "display_text_range"} textSlice = textRange{0}.getInt .. textRange{1}.getInt @@ -394,7 +399,8 @@ proc expandTweetEntitiesV2*(tweet: Tweet; js: JsonNode) = hasAttribution = tweet.attribution.isSome tweet.expandTextEntitiesV2(js, tweet.text, textSlice, - hasQuote or hasJobCard or hasAttribution) + hasQuote or hasJobCard or hasAttribution, + hasArticle) proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) = let diff --git a/src/routes/article.nim b/src/routes/article.nim new file mode 100644 index 0000000..0a7d1dc --- /dev/null +++ b/src/routes/article.nim @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import asyncdispatch, tables, strutils +import jester, karax/vdom +import ".."/[types, api] +import ../views/[article, general] +import router_utils + +export api, article, vdom, general, router_utils + +proc createArticleRouter*(cfg: Config) = + router articleRoute: + get "/i/article/@id": + cond @"id".allCharsInSet(Digits) + + let article = await getGraphArticle(@"id") + if article == nil: + resp Http404, showError("Article not found", cfg) + + var tweetIds: seq[string] + for e in article.entities.values: + if e.kind == "TWEET": + tweetIds.add e.tweetId + + var tweets = initTable[int64, Tweet]() + if tweetIds.len > 0: + try: + for t in await getGraphTweetResults(tweetIds): + tweets[t.id] = t + except CatchableError: + discard + + let + prefs = requestPrefs() + path = getPath() + html = renderArticle(article, tweets, path, prefs, @"id") + twitterUrl = "https://x.com/" & article.user.username & "/article/" & @"id" + resp renderMain(html, request, cfg, prefs, titleText=article.title, + twitterLink=twitterUrl) + + get "/@name/article/@id/?": + cond '.' notin @"name" + cond @"id".allCharsInSet(Digits) + redirect("/i/article/" & @"id") + + get "/@name/status/@id/article": + cond '.' notin @"name" + cond @"id".allCharsInSet(Digits) + redirect("/i/article/" & @"id") diff --git a/src/sass/_article.scss b/src/sass/_article.scss new file mode 100644 index 0000000..9cc728a --- /dev/null +++ b/src/sass/_article.scss @@ -0,0 +1,272 @@ +.article-page { + max-width: 700px; + margin: 0 auto 20px; + background-color: var(--bg_panel); + + > .top-ref { + padding-top: 20px; + } + + .article-cover { + width: 100%; + display: block; + } + + .article-body { + padding: 20px; + + > :last-child { + margin-bottom: 0; + } + + .article-title { + display: block; + font-size: 2rem; + line-height: 1.3; + margin: 0 0 10px; + color: var(--fg_color); + } + + .article-author { + margin-bottom: 12px; + padding-bottom: 10px; + border-bottom: 1px solid var(--border_grey); + font-size: 14px; + + .article-author-row { + display: flex; + align-items: center; + gap: 8px; + } + + .article-avatar { + display: flex; + } + + .avatar { + width: 40px; + height: 40px; + } + + .article-author-name { + display: flex; + align-items: center; + margin-bottom: 2px; + + .fullname { + font-size: 15px; + } + + .verified-icon { + margin-left: 2px; + } + } + + .article-author-meta { + display: flex; + align-items: center; + } + + .fullname { + font-weight: 700; + color: var(--fg_color); + max-width: unset; + text-overflow: unset; + overflow: visible; + white-space: normal; + } + + .username, + .article-date-sep, + .article-date { + color: var(--fg_dark); + } + + .username { + margin-left: 0; + } + + .article-date-sep { + margin: 0 4px; + } + + .article-date { + text-decoration: none; + + &:hover { + text-decoration: underline; + } + } + + .tweet-stats { + margin-top: 6px; + + .tweet-stat { + padding-top: 0; + } + } + } + + > h1 { + display: block; + font-size: 1.8rem; + margin: 25px 0 15px; + } + + > h2 { + font-size: 1.4rem; + font-weight: bold; + margin: 20px 0 12px; + } + + > h3 { + font-size: 1.2rem; + font-weight: bold; + margin: 18px 0 10px; + } + + > p { + font-size: 16px; + line-height: 1.7; + margin: 16px 0; + word-wrap: break-word; + } + + .blockquote-attribution { + display: block; + margin-top: 0.5em; + } + + > blockquote { + border-left: 3px solid var(--accent); + padding-left: 16px; + margin: 16px 0; + color: var(--fg_faded); + font-size: 16px; + line-height: 1.7; + } + + > pre { + background-color: var(--bg_elements); + padding: 12px 16px; + border-radius: 6px; + overflow-x: auto; + margin: 16px 0; + + code { + font-family: monospace; + font-size: 14px; + color: var(--fg_color); + } + } + + code { + background-color: var(--bg_elements); + padding: 2px 5px; + border-radius: 3px; + font-family: monospace; + font-size: 0.9em; + } + + > ul, + > ol { + margin: 16px 0; + padding-left: 2em; + + li { + font-size: 16px; + line-height: 1.7; + margin: 6px 0; + } + } + + .article-media { + text-align: center; + margin: 20px 0; + + img, + video { + max-width: 100%; + border-radius: 12px; + } + } + + > a, + > p a, + > h1 a, + > h2 a, + > h3 a, + > blockquote a, + > ul a, + > ol a { + color: var(--accent); + text-decoration: none; + + &:hover { + text-decoration: underline; + } + } + + .article-divider { + border: none; + border-top: 1px solid var(--border_grey); + margin: 30px 0; + } + + .timeline-item { + margin: 20px 0; + border: 1px solid var(--border_grey); + border-radius: 12px; + overflow: hidden; + } + } +} + +.conversation .article-page { + max-width: 100%; + margin-bottom: 0; +} + +.article-card { + .card-image-container { + position: relative; + } + + .card-image img { + height: auto; + } + + .article-card-badge { + position: absolute; + bottom: 8px; + left: 8px; + background: rgba(0, 0, 0, 0.75); + color: #fff; + font-size: 13px; + font-weight: 700; + padding: 2px 8px; + border-radius: 4px; + } +} + +.quote .article-card { + margin: 0; + + .card-container { + border: none; + border-radius: 0; + border-top: solid 1px var(--dark_grey); + } +} + +@media (max-width: 700px) { + .article-page { + .article-body { + padding: 12px 15px 25px; + + .article-title { + font-size: 1.6rem; + } + } + } +} diff --git a/src/sass/index.scss b/src/sass/index.scss index e60b9c4..8126fe9 100644 --- a/src/sass/index.scss +++ b/src/sass/index.scss @@ -8,6 +8,7 @@ @import "timeline"; @import "search"; @import "broadcast"; +@import "_article"; body { // colors diff --git a/src/types.nim b/src/types.nim index 3604efb..05b7fb3 100644 --- a/src/types.nim +++ b/src/types.nim @@ -198,6 +198,43 @@ type PhotoRail* = seq[GalleryPhoto] + Article* = ref object + title*: string + coverImage*: string + user*: User + time*: DateTime + stats*: TweetStats + paragraphs*: seq[ArticleParagraph] + entities*: Table[int, ArticleEntity] + media*: Table[string, ArticleMedia] + + ArticleParagraph* = object + text*: string + kind*: string + inlineStyles*: seq[ArticleStyle] + entityRanges*: seq[ArticleEntityRange] + + ArticleStyle* = object + offset*: int + length*: int + style*: string + + ArticleEntityRange* = object + offset*: int + length*: int + key*: int + + ArticleEntity* = object + kind*: string + url*: string + mediaIds*: seq[string] + tweetId*: string + markdown*: string + + ArticleMedia* = object + kind*: string + url*: string + Poll* = object options*: seq[string] values*: seq[int] @@ -247,6 +284,12 @@ type likes*: int views*: int + ArticlePreview* = object + title*: string + previewText*: string + coverImage*: string + tweetId*: int64 + Tweet* = ref object id*: int64 threadId*: int64 @@ -275,6 +318,7 @@ type note*: string isAd*: bool isAI*: bool + articlePreview*: Option[ArticlePreview] Tweets* = seq[Tweet] diff --git a/src/views/article.nim b/src/views/article.nim new file mode 100644 index 0000000..1ba9c8b --- /dev/null +++ b/src/views/article.nim @@ -0,0 +1,246 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import strutils, strformat, tables, unicode, bitops, uri +import karax/[karaxdsl, vdom] + +import renderutils, tweet, timeline +import ".."/[types, utils, formatters] + +proc renderAtomicParagraph(paragraph: ArticleParagraph; article: Article; + tweets: Table[int64, Tweet]; path: string; + prefs: Prefs): VNode = + if paragraph.entityRanges.len == 0: + return text "" + + let er = paragraph.entityRanges[0] + if er.key notin article.entities: + return text "" + + let entity = article.entities[er.key] + + case entity.kind + of "MEDIA": + buildHtml(tdiv(class="article-media")): + for id in entity.mediaIds: + let media = article.media.getOrDefault(id) + if media.url.len == 0: + continue + case media.kind + of "ApiGif": + video(src=getVidUrl(media.url), controls="", autoplay="", loop="", + muted="") + of "ApiVideo": + video(src=getVidUrl(media.url), controls="") + else: + a(href=getOrigPicUrl(media.url), target="_blank"): + img(src=getSmallPic(media.url), alt="", loading="lazy") + of "TWEET": + let tweet = tweets.getOrDefault( + try: parseBiggestInt(entity.tweetId) + except ValueError: 0, nil) + if tweet != nil: + renderTweet(tweet, prefs, path) + else: + text "" + of "MARKDOWN": + var content = entity.markdown + if content.startsWith("```"): + let firstNl = content.find('\n') + if firstNl >= 0: content = content[firstNl + 1 .. ^1] + if content.endsWith("```"): content = content[0 .. ^4] + content = content.strip + buildHtml(pre()): + code(): text content + of "DIVIDER": + buildHtml(hr(class="article-divider")) + else: + text "" + +proc wrapStyle(node: VNode; style: int): VNode = + result = node + if style.testBit(4): result = buildHtml(code()): result + if style.testBit(0): result = buildHtml(strong()): result + if style.testBit(1): result = buildHtml(em()): result + if style.testBit(2): result = buildHtml(del()): result + if style.testBit(3): result = buildHtml(underlined()): result + +proc addContent(target: VNode; content: string; style = 0) = + var first = true + for line in content.split('\n'): + if not first: + target.add VNode(kind: VNodeKind.br) + first = false + var pos = 0 + while pos < line.len: + let atPos = line.find('@', pos) + if atPos == -1: + target.add wrapStyle(text line[pos .. ^1], style) + break + if atPos > 0 and line[atPos - 1] in Letters + Digits + {'_'}: + target.add wrapStyle(text line[pos .. atPos], style) + pos = atPos + 1 + continue + var j = atPos + 1 + while j < line.len and j - atPos - 1 < 15 and + line[j] in Letters + Digits + {'_'}: + inc j + if j == atPos + 1: + target.add wrapStyle(text line[pos .. atPos], style) + pos = atPos + 1 + continue + if atPos > pos: + target.add wrapStyle(text line[pos ..< atPos], style) + let username = line[atPos + 1 ..< j] + let link = a.newVNode() + link.setAttr("href", "/" & username) + link.add wrapStyle(text ("@" & username), style) + target.add link + pos = j + +proc applyInlineStyles(target: VNode; runes: seq[Rune]; start, length: int; + styles: seq[ArticleStyle]) = + if styles.len == 0: + target.addContent($runes[start ..< start + length]) + return + + var + lastStyle = 0 + lastStart = start + let endPos = start + length + + for i in start ..< endPos: + var style = 0 + for sr in styles: + let + sStart = sr.offset + sEnd = sStart + sr.length + if sStart <= i and sEnd > i: + case sr.style + of "Bold": style.setBit(0) + of "Italic": style.setBit(1) + of "Strikethrough": style.setBit(2) + of "Underline": style.setBit(3) + of "Code": style.setBit(4) + else: discard + + if style != lastStyle: + if i > lastStart: + addContent(target, $runes[lastStart ..< i], lastStyle) + lastStyle = style + lastStart = i + + if lastStart < endPos: + addContent(target, $runes[lastStart ..< endPos], lastStyle) + +proc renderTextParagraph(paragraph: ArticleParagraph; article: Article): VNode = + let text = paragraph.text + + result = case paragraph.kind + of "header-one": h1.newVNode() + of "header-two": h2.newVNode() + of "header-three": h3.newVNode() + of "ordered-list-item", "unordered-list-item": li.newVNode() + of "blockquote": VNode(kind: VNodeKind.blockquote) + of "code-block": + let pre = pre.newVNode() + let code = code.newVNode() + code.add text text + pre.add code + return pre + else: p.newVNode() + + let + runes = text.toRunes + textLen = runes.len + var last = 0 + for er in paragraph.entityRanges: + if er.offset > last: + applyInlineStyles(result, runes, last, er.offset - last, + paragraph.inlineStyles) + + last = er.offset + er.length + + var target = result + if er.key in article.entities: + let entity = article.entities[er.key] + if entity.kind == "LINK": + let parsed = parseUri(entity.url) + if parsed.scheme in ["http", "https"]: + target = a.newVNode() + if parsed.isTwitterUrl: + target.setAttr("href", parsed.path) + else: + target.setAttr("href", entity.url) + + applyInlineStyles(target, runes, er.offset, er.length, + paragraph.inlineStyles) + if target != result: + result.add target + + if last < textLen: + applyInlineStyles(result, runes, last, textLen - last, + paragraph.inlineStyles) + + if paragraph.kind == "blockquote" and result.len > 0: + let lastChild = result[result.len - 1] + if lastChild.kind == VNodeKind.strong and lastChild.len > 0 and + lastChild[0].kind == VNodeKind.text: + lastChild.setAttr("class", "blockquote-attribution") + +proc renderArticle*(article: Article; tweets: Table[int64, Tweet]; + path: string; prefs: Prefs; tweetId=""): VNode = + let author = article.user + + let main = buildHtml(article(class="article-body")): + h1(class="article-title"): text article.title + + tdiv(class="article-author"): + tdiv(class="article-author-row"): + a(class="article-avatar", href=("/" & author.username)): + genImg(author.getUserPic("_bigger"), class=prefs.getAvatarClass) + tdiv(class="article-author-info"): + tdiv(class="article-author-name"): + linkUser(author, class="fullname") + verifiedIcon(author) + tdiv(class="article-author-meta"): + linkUser(author, class="username") + span(class="article-date-sep"): text " · " + a(class="article-date", + href=("/" & author.username & "/status/" & tweetId)): + text article.time.getShortTime + if not prefs.hideTweetStats: + renderStats(article.stats) + + var listKind = "" + var list: VNode = nil + + for paragraph in article.paragraphs: + let isListItem = paragraph.kind in [ + "ordered-list-item", "unordered-list-item"] + + if not isListItem and list != nil: + main.add list + list = nil + listKind = "" + + if paragraph.kind == "atomic": + main.add renderAtomicParagraph(paragraph, article, tweets, path, prefs) + elif isListItem: + if paragraph.kind != listKind: + if list != nil: + main.add list + list = if paragraph.kind == "ordered-list-item": ol.newVNode() + else: ul.newVNode() + listKind = paragraph.kind + list.add renderTextParagraph(paragraph, article) + else: + main.add renderTextParagraph(paragraph, article) + + if list != nil: + main.add list + + buildHtml(tdiv(class="article-page")): + if article.coverImage.len > 0: + a(href=getOrigPicUrl(article.coverImage), target="_blank"): + img(class="article-cover", src=getSmallPic(article.coverImage), alt="") + main + renderToTop() diff --git a/src/views/general.nim b/src/views/general.nim index f753610..13ac10d 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -50,7 +50,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; let opensearchUrl = getUrlPrefix(cfg) & "/opensearch" buildHtml(head): - link(rel="stylesheet", type="text/css", href="/css/style.css?v=35") + link(rel="stylesheet", type="text/css", href="/css/style.css?v=38") link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=5") if theme.len > 0: @@ -122,9 +122,12 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; proc renderMain*(body: VNode; req: Request; cfg: Config; prefs=defaultPrefs; titleText=""; desc=""; ogTitle=""; rss=""; video=""; - images: seq[string] = @[]; banner=""): string = + images: seq[string] = @[]; banner=""; + twitterLink=""): string = - let twitterLink = getTwitterLink(req.path, req.params) + let twitterLink = + if twitterLink.len > 0: twitterLink + else: getTwitterLink(req.path, req.params) let node = buildHtml(html(lang="en")): renderHead(prefs, cfg, req, titleText, desc, video, images, banner, ogTitle, diff --git a/src/views/tweet.nim b/src/views/tweet.nim index 5546eb4..9bf6c9b 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -9,9 +9,24 @@ import general const doctype = "\n" -proc renderMiniAvatar(user: User; prefs: Prefs): VNode = +proc renderMiniAvatar*(user: User; prefs: Prefs): VNode = genImg(user.getUserPic("_mini"), class=(prefs.getAvatarClass & " mini")) +proc renderArticleCard(preview: ArticlePreview; prefs: Prefs): VNode = + let url = "/i/article/" & $preview.tweetId + buildHtml(tdiv(class="article-card card large")): + a(class="card-container", href=url): + if preview.coverImage.len > 0: + tdiv(class="card-image-container"): + tdiv(class="card-image"): + genImg(preview.coverImage) + span(class="article-card-badge"): text "Article" + tdiv(class="card-content-container"): + tdiv(class="card-content"): + h2(class="card-title"): text preview.title + if preview.previewText.len > 0: + p(class="card-description"): text preview.previewText + proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs): VNode = buildHtml(tdiv): if pinned: @@ -225,7 +240,7 @@ func formatStat(stat: int): string = if stat > 0: insertSep($stat, ',') else: "" -proc renderStats(stats: TweetStats): VNode = +proc renderStats*(stats: TweetStats): VNode = buildHtml(tdiv(class="tweet-stats")): span(class="tweet-stat"): icon "comment", formatStat(stats.replies) span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets) @@ -308,6 +323,9 @@ proc renderQuote(quote: Tweet; prefs: Prefs; path: string): VNode = if quote.media.len > 0: renderQuoteMedia(quote, prefs, path) + if quote.articlePreview.isSome: + renderArticleCard(quote.articlePreview.get(), prefs) + if quote.note.len > 0 and not prefs.hideCommunityNotes: renderCommunityNote(quote.note, prefs) @@ -392,6 +410,9 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; if tweet.card.isSome and tweet.card.get().kind != hidden: renderCard(tweet.card.get(), prefs, path) + if tweet.articlePreview.isSome: + renderArticleCard(tweet.articlePreview.get(), prefs) + if tweet.media.len > 0: renderMedia(tweet.media, prefs, path, bigThumb) diff --git a/tests/test_article.py b/tests/test_article.py new file mode 100644 index 0000000..15a87c7 --- /dev/null +++ b/tests/test_article.py @@ -0,0 +1,307 @@ +from base import BaseTestCase +from parameterized import parameterized + + +class ArticleSelectors: + page = '.article-page' + cover = '.article-cover' + body = '.article-body' + title = '.article-title' + author = '.article-author' + fullname = '.article-author .fullname' + username = '.article-author .username' + date = '.article-author .article-date' + avatar = '.article-author img.avatar' + verified = '.article-author .verified-icon' + media = '.article-media' + divider = '.article-divider' + + +articles = [ + ['2064166507438059759', + '1s,秒杀一切,开源一个 X 文章发布 Skill【重磅升级】', + 'punk2898', 'Punk'], + + ['2064689664213041529', + 'SpaceX Thesis & Valuation Memorandum', + 'Dialectic_Group', 'Dialectic'], + + ['2064691088636424322', + 'Consciousness and AI: The Problem of Inner Experience', + 'CosmicOrFun', 'Cosmic Orphan'], + + ['2064696491948777658', + 'NC Push for Data Centers + Stablecoin Crypto= Data Centers are defacto BAILOUT OF Fed Reserve System', + 'June_12_1776', 'June_12_1776'], + + ['2064755789391110154', + 'DeFi Markets Update 2026-06-10', + 'SteakhouseFi', 'Steakhouse Financial'], + + ['2064755231901319527', + 'The machine economy has a killswitch and somebody just pulled it.', + '1914ad', 'Justin Bechler HMP-028'], + + ['2062858677149675788', + 'Yakshinis', + 'CosmicOrFun', 'Cosmic Orphan'], +] + +articles_with_media = [ + ['2064166507438059759', 6], + ['2064689664213041529', 11], + ['2064755789391110154', 5], +] + +articles_with_dividers = [ + ['2064166507438059759', 1], + ['2064689664213041529', 6], +] + + +class ArticleBasicTest(BaseTestCase): + @parameterized.expand(articles) + def test_article_loads(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + self.assert_element_visible(ArticleSelectors.page) + self.assert_element_visible(ArticleSelectors.body) + self.assert_text(title, ArticleSelectors.title) + + @parameterized.expand(articles) + def test_article_author(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + self.assert_element_visible(ArticleSelectors.author) + self.assert_text(f'@{username}', ArticleSelectors.username) + + @parameterized.expand(articles) + def test_article_has_cover(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + self.assert_element_visible(ArticleSelectors.cover) + src = self.get_attribute(ArticleSelectors.cover, 'src') + self.assertIn('/pic/', src) + + @parameterized.expand(articles) + def test_article_has_date(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + date_text = self.get_text(ArticleSelectors.date) + self.assertTrue(len(date_text) > 3) + + @parameterized.expand(articles) + def test_article_author_avatar(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + self.assert_element_visible(ArticleSelectors.avatar) + src = self.get_attribute(ArticleSelectors.avatar, 'src') + self.assertIn('/pic/', src) + self.assertGreater(len(src), len('/pic/')) + + @parameterized.expand(articles) + def test_article_author_verified(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + self.assert_element_visible(ArticleSelectors.verified) + + def test_article_author_verified_business(self): + self.open_nitter('i/article/2064755789391110154') + self.assert_element_visible('.article-author .verified-icon.business') + + +class ArticleContentTest(BaseTestCase): + def test_article_has_paragraphs(self): + self.open_nitter('i/article/2064689664213041529') + paragraphs = self.find_elements('.article-body p') + self.assertGreater(len(paragraphs), 10) + + def test_article_has_headers(self): + self.open_nitter('i/article/2064689664213041529') + headers = self.find_elements('.article-body h1, .article-body h2') + self.assertGreater(len(headers), 5) + + def test_article_has_bold_text(self): + self.open_nitter('i/article/2064166507438059759') + bold = self.find_elements('.article-body strong') + self.assertGreater(len(bold), 0) + + def test_article_has_italic_text(self): + self.open_nitter('i/article/2064166507438059759') + italic = self.find_elements('.article-body em') + self.assertGreater(len(italic), 0) + + def test_article_has_blockquotes(self): + self.open_nitter('i/article/2064696491948777658') + self.assert_element_visible('.article-body blockquote') + + def test_article_has_lists(self): + self.open_nitter('i/article/2064696491948777658') + self.assert_element_visible('.article-body ul') + + def test_article_has_emoji_text(self): + self.open_nitter('i/article/2064166507438059759') + body = self.get_text(ArticleSelectors.body) + self.assertTrue(any(ord(c) > 0x1F000 for c in body)) + + def test_article_has_links(self): + self.open_nitter('i/article/2064691088636424322') + links = self.find_elements('.article-body a[href]') + self.assertGreater(len(links), 0) + + def test_article_twitter_links_localized(self): + self.open_nitter('i/article/2064755789391110154') + links = self.find_elements('.article-body a[href^="https://x.com"]') + self.assertEqual(len(links), 0, 'x.com links should be converted to local paths') + + @parameterized.expand(articles_with_media) + def test_article_media_count(self, tweet_id, expected_count): + self.open_nitter(f'i/article/{tweet_id}') + media = self.find_elements(ArticleSelectors.media) + self.assertEqual(len(media), expected_count) + + @parameterized.expand(articles_with_dividers) + def test_article_divider_count(self, tweet_id, expected_count): + self.open_nitter(f'i/article/{tweet_id}') + dividers = self.find_elements(ArticleSelectors.divider) + self.assertEqual(len(dividers), expected_count) + + +class ArticleMediaTest(BaseTestCase): + def test_media_images_proxied(self): + self.open_nitter('i/article/2064689664213041529') + self.assert_element_visible(ArticleSelectors.media) + img = self.find_element(f'{ArticleSelectors.media} img') + src = img.get_attribute('src') + self.assertIn('/pic/', src) + self.assertFalse(src.startswith('https://pbs.twimg.com')) + + def test_cover_image_proxied(self): + self.open_nitter('i/article/2064689664213041529') + self.assert_element_visible(ArticleSelectors.cover) + src = self.get_attribute(ArticleSelectors.cover, 'src') + self.assertIn('/pic/', src) + self.assertFalse(src.startswith('https://pbs.twimg.com')) + + def test_embedded_tweet(self): + self.open_nitter('i/article/2064755789391110154') + self.assert_element_visible('.article-body .timeline-item') + + def test_multiple_embedded_tweets(self): + self.open_nitter('i/article/2064755231901319527') + tweets = self.find_elements('.article-body .timeline-item') + self.assertGreaterEqual(len(tweets), 3) + + +class ArticleMentionTest(BaseTestCase): + def test_mention_linkified(self): + self.open_nitter('i/article/2064755231901319527') + link = self.find_element('.article-body a[href="/ZachXBT"]') + self.assertEqual(link.text, '@ZachXBT') + + def test_multiple_mentions_linkified(self): + self.open_nitter('i/article/2064755231901319527') + links = self.find_elements('.article-body a[href^="/"]') + mention_hrefs = [l.get_attribute('href') for l in links + if l.text.startswith('@')] + usernames = [h.split('/')[-1] for h in mention_hrefs] + self.assertIn('ZachXBT', usernames) + self.assertIn('River', usernames) + + def test_mention_in_different_article(self): + self.open_nitter('i/article/2064689664213041529') + link = self.find_element('.article-body a[href="/FutureJurvetson"]') + self.assertEqual(link.text, '@FutureJurvetson') + + def test_no_spurious_whitespace_in_styled_paragraph(self): + """Styled paragraphs should not have extra whitespace from VNode serialization.""" + self.open_nitter('i/article/2064696491948777658') + source = self.get_page_source() + self.assertNotIn('white-space: pre-wrap', source) + self.assertNotIn('white-space:pre-wrap', source) + + +class ArticleCardTest(BaseTestCase): + @parameterized.expand(articles) + def test_status_page_shows_article_card(self, tweet_id, title, username, fullname): + self.open_nitter(f'{username}/status/{tweet_id}') + self.assert_element_visible('.article-card') + self.assert_text(title, '.article-card .card-title') + + def test_article_card_has_cover_image(self): + self.open_nitter('Dialectic_Group/status/2064689664213041529') + self.assert_element_visible('.article-card .card-image img') + src = self.get_attribute('.article-card .card-image img', 'src') + self.assertIn('/pic/', src) + + def test_article_card_has_badge(self): + self.open_nitter('Dialectic_Group/status/2064689664213041529') + self.assert_element_visible('.article-card-badge') + self.assert_text('Article', '.article-card-badge') + + def test_article_card_has_preview_text(self): + self.open_nitter('CosmicOrFun/status/2064691088636424322') + self.assert_element_visible('.article-card .card-description') + + def test_article_card_links_to_article(self): + self.open_nitter('punk2898/status/2064166507438059759') + href = self.get_attribute('.article-card .card-container', 'href') + self.assertIn('/article/', href) + + def test_article_url_stripped_from_tweet_text(self): + self.open_nitter('punk2898/status/2064166507438059759') + self.assert_element_visible('.article-card') + source = self.get_page_source() + # Main tweet text should not contain article URL + import re + main = re.search(r'id="m".*?tweet-content[^>]*>(.*?)', source, re.DOTALL) + self.assertIsNotNone(main) + self.assertNotIn('/article/', main.group(1)) + + +class ArticleQuotedCardTest(BaseTestCase): + """Article cards inside quoted tweets (1914ad quoting own article).""" + quoted_tweet = '1914ad/status/2064789532071891085' + quoted_article_id = '2063677483548102688' + + def test_quoted_card_visible(self): + self.open_nitter(self.quoted_tweet) + self.assert_element_visible('.quote .article-card') + + def test_quoted_card_has_title(self): + self.open_nitter(self.quoted_tweet) + self.assert_text('David Bailey Already Won', '.quote .article-card .card-title') + + def test_quoted_card_has_badge(self): + self.open_nitter(self.quoted_tweet) + self.assert_element_visible('.quote .article-card-badge') + self.assert_text('Article', '.quote .article-card-badge') + + def test_quoted_card_has_cover_image(self): + self.open_nitter(self.quoted_tweet) + self.assert_element_visible('.quote .article-card .card-image img') + src = self.get_attribute('.quote .article-card .card-image img', 'src') + self.assertIn('/pic/', src) + + def test_quoted_card_has_description(self): + self.open_nitter(self.quoted_tweet) + self.assert_element_visible('.quote .article-card .card-description') + + def test_quoted_card_links_to_article(self): + self.open_nitter(self.quoted_tweet) + href = self.get_attribute('.quote .article-card .card-container', 'href') + self.assertIn(f'/article/{self.quoted_article_id}', href) + + +class ArticleRoutingTest(BaseTestCase): + def test_username_article_route_redirects(self): + self.open_nitter('punk2898/article/2064166507438059759') + self.assert_element_visible(ArticleSelectors.page) + self.assert_text('1s', ArticleSelectors.title) + + def test_status_article_route_redirects(self): + self.open_nitter('punk2898/status/2064166507438059759/article') + self.assert_element_visible(ArticleSelectors.page) + self.assert_text('1s', ArticleSelectors.title) + + def test_invalid_id_returns_404(self): + self.open_nitter('i/article/notanumber') + self.assert_element_not_visible(ArticleSelectors.page) + + def test_nonexistent_article(self): + self.open_nitter('i/article/1') + self.assert_element_visible('.error-panel') From 7b27c2c629a55661ad5b4ad91fb1a7b2d3f9910e Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 18 Jun 2026 13:32:20 +0200 Subject: [PATCH 26/47] Add support for Communities Fixes #1270 --- src/api.nim | 51 ++++++++ src/consts.nim | 37 ++++++ src/nitter.nim | 4 +- src/parser.nim | 76 +++++++++++ src/redis_cache.nim | 23 ++++ src/routes/community.nim | 89 +++++++++++++ src/sass/index.scss | 1 + src/sass/profile/_base.scss | 1 + src/sass/profile/_community.scss | 203 +++++++++++++++++++++++++++++ src/types.nim | 17 +++ src/views/community.nim | 128 +++++++++++++++++++ src/views/general.nim | 2 +- src/views/renderutils.nim | 6 +- src/views/timeline.nim | 3 +- src/views/tweet.nim | 15 ++- temp | 0 tests/test_community.py | 211 +++++++++++++++++++++++++++++++ 17 files changed, 856 insertions(+), 11 deletions(-) create mode 100644 src/routes/community.nim create mode 100644 src/sass/profile/_community.scss create mode 100644 src/views/community.nim delete mode 100644 temp create mode 100644 tests/test_community.py diff --git a/src/api.nim b/src/api.nim index 2702f7c..9aa6c50 100644 --- a/src/api.nim +++ b/src/api.nim @@ -94,6 +94,57 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi js = await fetch(url) result = parseGraphTimeline(js, after) +proc getGraphCommunity*(id: string): Future[Community] {.async.} = + if id.len == 0: return + let + url = apiReq(graphCommunity, communityVars % id) + js = await fetch(url) + result = parseGraphCommunity(js) + +proc getGraphCommunityTweets*(id: string; rankingMode: string; after=""): Future[Timeline] {.async.} = + if id.len == 0: return + let + cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + url = apiReq(graphCommunityTweets, communityTweetsVars % [id, cursor, rankingMode]) + js = await fetch(url) + result = parseGraphCommunityTimeline(js, after) + +proc getGraphCommunityMedia*(id: string; after=""): Future[Timeline] {.async.} = + if id.len == 0: return + let + cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + url = apiReq(graphCommunityMedia, communityMediaVars % [id, cursor]) + js = await fetch(url) + result = parseGraphCommunityTimeline(js, after) + +proc communitySliceReq(endpoint, variables: string): ApiReq = + let url = ApiUrl(endpoint: endpoint, params: @[("variables", variables)]) + ApiReq(cookie: url, oauth: url) + +proc getGraphCommunityMembers*(id: string; after=""): Future[Result[User]] {.async.} = + if id.len == 0: return + let + cursor = if after.len > 0: "\"$1\"" % after else: "null" + url = communitySliceReq(graphCommunityMembers, communityMembersVars % [id, cursor]) + js = await fetch(url) + result = parseGraphCommunityMembers(js, after) + +proc getGraphCommunityModerators*(id: string): Future[Result[User]] {.async.} = + if id.len == 0: return + let + url = communitySliceReq(graphCommunityModerators, communityMembersVars % [id, "null"]) + js = await fetch(url) + result = parseGraphCommunityMembers(js) + +proc getGraphCommunityHashtags*(id, hashtag: string; after=""): Future[Timeline] {.async.} = + if id.len == 0 or hashtag.len == 0: return + let + safeTag = multiReplace(hashtag, ("\"", ""), ("\\", "")) + cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + url = apiReq(graphCommunityHashtags, communityHashtagsVars % [id, cursor, safeTag]) + js = await fetch(url) + result = parseGraphCommunityTimeline(js, after) + proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return let diff --git a/src/consts.nim b/src/consts.nim index 5ac3c32..59b986c 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -28,6 +28,13 @@ const graphListTweets* = "0QJtcuMzVywHGAWD6Dtjlw/ListTimeline" graphAboutAccount* = "zUnx-DLN9dkwOkNhTLySjg/AboutAccountQuery" + graphCommunity* = "-ElI1vg3dYbttVMhBhGdLw/CommunityQuery" + graphCommunityTweets* = "Mvs5UOOEkpXVMDZtUcxR-Q/CommunityTweetsTimeline" + graphCommunityMedia* = "Bt9XYnY7D3OcmZE5lhdx-A/CommunityMediaTimeline" + graphCommunityMembers* = "WSbJGJjZaVasSj9bnqSZSA/membersSliceTimeline_Query" + graphCommunityModerators* = "GBMT3GOWy5dYsYC4XJfvow/moderatorsSliceTimeline_Query" + graphCommunityHashtags* = "40DyrMxfCknGuZwE-keW_Q/CommunityHashtagsTimeline" + graphTweetResultByRestId* = "qtXMy1p5Y62uCskc_NUPJw/TweetResultByRestId" graphTweetResultsByRestIds* = "Sc9EUQTZNEH-wzegn-nHvQ/TweetResultsByRestIds" @@ -47,6 +54,8 @@ const "premium_content_api_read_enabled": false, "communities_web_enable_tweet_community_results_fetch": true, "c9s_tweet_anatomy_moderator_badge_enabled": true, + "c9s_list_members_action_api_enabled": false, + "c9s_superc9s_indication_enabled": false, "responsive_web_grok_analyze_button_fetch_trends_enabled": false, "responsive_web_grok_analyze_post_followups_enabled": true, "rweb_cashtags_composer_attachment_enabled": true, @@ -152,6 +161,34 @@ const articleFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withArticleSummaryText":true,"withArticleVoiceOver":true}""" + communityVars* = """{"communityId":"$1"}""" + + communityTweetsVars* = """{ + "communityId": "$1", $2 + "count": 20, + "displayLocation": "Community", + "rankingMode": "$3", + "withCommunity": true +}""".replace(" ", "").replace("\n", "") + + communityMediaVars* = """{ + "communityId": "$1", $2 + "count": 20, + "withCommunity": true +}""".replace(" ", "").replace("\n", "") + + communityMembersVars* = """{ + "communityId": "$1", + "cursor": $2 +}""".replace(" ", "").replace("\n", "") + + communityHashtagsVars* = """{ + "communityId": "$1", $2 + "count": 20, + "hashtags": ["$3"], + "withCommunity": true +}""".replace(" ", "").replace("\n", "") + userFieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}""" userTweetsFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false}""" tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}""" diff --git a/src/nitter.nim b/src/nitter.nim index 78cace4..955849b 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -9,7 +9,7 @@ import jester import types, config, prefs, formatters, redis_cache, http_pool, auth, apiutils import views/[general, about] import routes/[ - preferences, timeline, status, media, search, rss, list, debug, + preferences, timeline, status, media, search, rss, list, community, debug, unsupported, embed, resolver, broadcast, article, router_utils] const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances" @@ -54,6 +54,7 @@ createResolverRouter(cfg) createPrefRouter(cfg) createTimelineRouter(cfg) createListRouter(cfg) +createCommunityRouter(cfg) createStatusRouter(cfg) createSearchRouter(cfg) createMediaRouter(cfg) @@ -126,6 +127,7 @@ routes: extend timeline, "" extend media, "" extend list, "" + extend community, "" extend preferences, "" extend resolver, "" extend embed, "" diff --git a/src/parser.nim b/src/parser.nim index d7b3f8d..c810f76 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -131,6 +131,36 @@ proc parseBroadcastInfo*(js: JsonNode): Broadcast = user: parseGraphUser(bc) ) +proc parseGraphCommunity*(js: JsonNode): Community = + if js.isNull: return + let c = ? js{"data", "communityResults", "result"} + + result = Community( + id: c{"rest_id"}.getStr(c{"id_str"}.getStr), + name: c{"name"}.getStr, + description: c{"description"}.getStr, + memberCount: c{"member_count"}.getInt, + joinPolicy: c{"join_policy"}.getStr, + category: c{"primary_community_topic", "topic_name"}.getStr, + banner: c{"custom_banner_media", "media_info", "original_img_url"}.getImageStr, + creator: parseGraphUser(c{"creator_results", "result"}), + ) + + let createdMs = c{"created_at"}.getInt(0) + if createdMs > 0: + result.createdAt = fromUnix(createdMs div 1000).utc() + + for rule in c{"rules"}: + result.rules.add CommunityRule( + name: rule{"name"}.getStr, + description: rule{"description"}.getStr + ) + + for item in c{"trending_hashtags_slice", "items"}: + let tag = item{"hashtag"}.getStr + if tag.len > 0: + result.hashtags.add tag + proc parseGraphList*(js: JsonNode): List = if js.isNull: return @@ -855,3 +885,49 @@ proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] = elif typ == "TimelineReplaceEntry": if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"): result.bottom = instruction{"entry", "content", "value"}.getStr + +proc parseGraphCommunityTimeline*(js: JsonNode; after=""): Timeline = + result = Timeline(beginning: after.len == 0) + + let communityResult = js{"data", "communityResults", "result"} + let instructions = ? select( + communityResult{"ranked_community_timeline", "timeline", "instructions"}, + communityResult{"community_media_timeline", "timeline", "instructions"}, + communityResult{"community_filtered_timeline", "timeline", "instructions"} + ) + if instructions.len == 0: + return + + for i in instructions: + if i{"entries"}.notNull: + for e in i{"entries"}: + let entryId = e.getEntryId + if entryId.startsWith("tweet") or entryId.startsWith("profile-grid") or + entryId.startsWith("communities-grid"): + for tweet in extractTweetsFromEntry(e): + result.content.add tweet + elif entryId.startsWith("cursor-bottom"): + result.bottom = e{"content", "value"}.getStr + + if after.len == 0 and i.getTypeName == "TimelinePinEntry": + var tweets = extractTweetsFromEntry(i{"entry"}) + for tweet in tweets.mitems: + tweet.pinned = true + if tweets.len > 0: + result.content.insert(tweets, 0) + +proc parseGraphCommunityMembers*(js: JsonNode; after=""): Result[User] = + result = Result[User](beginning: after.len == 0) + + let r = js{"data", "communityResults", "result"} + let slice = if not r{"members_slice"}.isNull: r{"members_slice"} + else: r{"moderators_slice"} + for item in slice{"items_results"}: + let user = parseGraphUser(item{"result"}) + if user.username.len > 0: + result.content.add user + + let cursor = slice{"slice_info", "next_cursor"}.getStr + if cursor.len > 0: + result.bottom = cursor + diff --git a/src/redis_cache.nim b/src/redis_cache.nim index e503e46..4d5bf28 100644 --- a/src/redis_cache.nim +++ b/src/redis_cache.nim @@ -195,6 +195,29 @@ proc getCachedPhotoRail*(id: string): Future[PhotoRail] {.async.} = result = await getPhotoRail(id) await cache(result, id) +proc cache*(data: Community) {.async.} = + if data.id.len == 0: return + await setEx("cm:" & data.id, listCacheTime, compress(toFlatty(data))) + +proc getCachedCommunity*(id: string): Future[Community] {.async.} = + if id.len == 0: return + let cached = await get("cm:" & id) + if cached != redisNil: + cached.deserialize(Community) + else: + result = await getGraphCommunity(id) + await cache(result) + +proc getCachedCommunityModerators*(id: string): Future[seq[User]] {.async.} = + if id.len == 0: return + let cached = await get("cmm:" & id) + if cached != redisNil: + cached.deserialize(seq[User]) + else: + let mods = await getGraphCommunityModerators(id) + result = mods.content + await setEx("cmm:" & id, listCacheTime, compress(toFlatty(result))) + proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} = let list = if id.len == 0: redisNil else: await get("l:" & id) diff --git a/src/routes/community.nim b/src/routes/community.nim new file mode 100644 index 0000000..b850b6b --- /dev/null +++ b/src/routes/community.nim @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import strformat + +import jester + +import router_utils +import ".."/[types, redis_cache, api] +import ../views/[general, timeline, community] + +export community + +template respCommunity*(cmty: Community; title: string; nav, vnode: typed) = + if cmty.id.len == 0 or cmty.name.len == 0: + resp Http404, showError(&"""Community "{@"id"}" not found""", cfg) + + let html = renderCommunity(vnode, nav, cmty) + resp renderMain(html, request, cfg, prefs, titleText=title, banner=cmty.banner) + +proc createCommunityRouter*(cfg: Config) = + router community: + get "/i/communities/@id/?": + cond '.' notin @"id" + let + prefs = requestPrefs() + cmty = await getCachedCommunity(@"id") + tl = await getGraphCommunityTweets(cmty.id, "Relevance", getCursor()) + respCommunity(cmty, cmty.name, + renderCommunityTabs(QueryKind.posts, cmty), + renderTimelineTweets(tl, prefs, request.path)) + + get "/i/communities/@id/latest": + cond '.' notin @"id" + let + prefs = requestPrefs() + cmty = await getCachedCommunity(@"id") + tl = await getGraphCommunityTweets(cmty.id, "Recency", getCursor()) + respCommunity(cmty, cmty.name & " - Latest", + renderCommunityTabs(QueryKind.replies, cmty), + renderTimelineTweets(tl, prefs, request.path)) + + get "/i/communities/@id/media": + cond '.' notin @"id" + let + prefs = requestPrefs() + cmty = await getCachedCommunity(@"id") + tl = await getGraphCommunityMedia(cmty.id, getCursor()) + respCommunity(cmty, cmty.name & " - Media", + renderCommunityTabs(QueryKind.media, cmty), + renderTimelineTweets(tl, prefs, request.path)) + + get "/i/communities/@id/about": + cond '.' notin @"id" + let + prefs = requestPrefs() + cmty = await getCachedCommunity(@"id") + mods = await getCachedCommunityModerators(cmty.id) + respCommunity(cmty, cmty.name & " - About", + renderCommunityTabs(QueryKind.userList, cmty), + renderCommunityAbout(cmty, mods)) + + get "/i/communities/@id/members": + cond '.' notin @"id" + let + prefs = requestPrefs() + cmty = await getCachedCommunity(@"id") + members = await getGraphCommunityMembers(cmty.id, getCursor()) + respCommunity(cmty, cmty.name & " - Members", + renderMemberTabs(cmty, false), + renderTimelineUsers(members, prefs, request.path)) + + get "/i/communities/@id/moderators": + cond '.' notin @"id" + let + prefs = requestPrefs() + cmty = await getCachedCommunity(@"id") + mods = await getCachedCommunityModerators(cmty.id) + respCommunity(cmty, cmty.name & " - Moderators", + renderMemberTabs(cmty, true), + renderTimelineUsers(Result[User](content: mods), prefs, request.path)) + + get "/i/communities/@id/hashtag/@tag": + cond '.' notin @"id" + let + prefs = requestPrefs() + cmty = await getCachedCommunity(@"id") + tl = await getGraphCommunityHashtags(cmty.id, @"tag", getCursor()) + respCommunity(cmty, cmty.name & " - #" & @"tag", + renderHashtagHeader(cmty, @"tag"), + renderTimelineTweets(tl, prefs, request.path)) diff --git a/src/sass/index.scss b/src/sass/index.scss index 8126fe9..b8c9a8c 100644 --- a/src/sass/index.scss +++ b/src/sass/index.scss @@ -160,6 +160,7 @@ body.fixed-nav .container { .verified-icon { display: inline-block; + position: relative; width: 14px; height: 14px; margin-bottom: 2px; diff --git a/src/sass/profile/_base.scss b/src/sass/profile/_base.scss index 2482460..81b3d78 100644 --- a/src/sass/profile/_base.scss +++ b/src/sass/profile/_base.scss @@ -4,6 +4,7 @@ @import "card"; @import "about-account"; @import "photo-rail"; +@import "community"; .profile-tabs { @include panel(auto, 900px); diff --git a/src/sass/profile/_community.scss b/src/sass/profile/_community.scss new file mode 100644 index 0000000..92d13c9 --- /dev/null +++ b/src/sass/profile/_community.scss @@ -0,0 +1,203 @@ +.community-header { + padding: 12px 15px; + border-bottom: 1px solid var(--border_grey); + background-color: var(--bg_panel); + + .community-name { + font-size: 22px; + margin-bottom: 6px; + + a { + color: inherit; + } + } + + .community-category { + display: inline-block; + background-color: var(--bg_elements); + border: 1px solid var(--border_grey); + border-radius: 16px; + padding: 2px 12px; + font-size: 13px; + color: var(--fg_faded); + margin-bottom: 8px; + } + + .community-description { + color: var(--fg_faded); + margin-bottom: 8px; + line-height: 1.4; + } + + .community-member-count { + font-weight: bold; + color: inherit; + } + + .community-stats { + color: var(--grey); + font-size: 14px; + } +} + +.community-about { + padding: 16px 15px 15px; + background-color: var(--bg_panel); + + h2 { + font-size: 18px; + margin: 0 0 12px; + } + + .community-info { + border-bottom: 1px solid var(--border_grey); + padding-bottom: 12px; + } + + .community-info-item { + display: flex; + gap: 10px; + padding: 8px 0; + align-items: center; + + .verified-icon { + margin-left: 2px; + } + + > .icon-container { + display: flex; + align-items: center; + justify-content: center; + font-size: 16px; + color: var(--grey); + flex-shrink: 0; + width: 26px; + height: 26px; + } + + strong { + color: var(--fg_color); + } + + a { + color: var(--accent); + } + } + + .community-rules { + border-bottom: 1px solid var(--border_grey); + padding: 16px 0 12px; + } + + .community-rules-intro { + color: var(--fg_faded); + font-size: 14px; + margin: 0 0 12px; + } + + .community-rule { + display: flex; + gap: 10px; + padding: 10px 0; + align-items: flex-start; + + .community-rule-number { + display: flex; + align-items: center; + justify-content: center; + min-width: 26px; + height: 26px; + border-radius: 50%; + background-color: var(--accent); + color: var(--fg_color); + font-weight: bold; + font-size: 13px; + flex-shrink: 0; + } + + .community-rule-content p { + margin: 4px 0 0; + color: var(--fg_faded); + font-size: 14px; + } + } + + .community-moderators { + padding-top: 16px; + + h2 { + display: flex; + align-items: center; + justify-content: space-between; + } + + .community-mods-link { + font-size: 14px; + font-weight: normal; + color: var(--accent); + } + } + + .community-moderator { + display: flex; + gap: 10px; + padding: 8px 0; + align-items: center; + + .community-mod-avatar { + width: 40px; + height: 40px; + border-radius: 50%; + } + + .community-mod-info { + display: flex; + flex-direction: column; + } + + .community-mod-name { + display: flex; + align-items: center; + font-weight: bold; + color: var(--fg_color); + + .verified-icon { + margin-left: 2px; + } + } + + .community-mod-username { + color: var(--fg_faded); + font-size: 14px; + } + } +} + +.community-tags { + display: flex; + flex-wrap: wrap; + gap: 8px; + padding: 10px 15px; + border-bottom: 1px solid var(--border_grey); + + .community-tag { + display: inline-block; + background-color: var(--bg_elements); + border: 1px solid var(--border_grey); + border-radius: 16px; + padding: 4px 12px; + font-size: 13px; + color: var(--accent); + } +} + +.community-hashtag-header { + padding: 12px 15px; + border-bottom: 1px solid var(--border_grey); + + .community-hashtag-title { + font-size: 20px; + color: var(--accent); + margin: 0; + } +} diff --git a/src/types.nim b/src/types.nim index 05b7fb3..79a99c0 100644 --- a/src/types.nim +++ b/src/types.nim @@ -361,6 +361,23 @@ type members*: int banner*: string + CommunityRule* = object + name*: string + description*: string + + Community* = object + id*: string + name*: string + description*: string + memberCount*: int + banner*: string + creator*: User + category*: string + joinPolicy*: string + createdAt*: DateTime + rules*: seq[CommunityRule] + hashtags*: seq[string] + GlobalObjects* = ref object tweets*: Table[string, Tweet] users*: Table[string, User] diff --git a/src/views/community.nim b/src/views/community.nim new file mode 100644 index 0000000..52f9041 --- /dev/null +++ b/src/views/community.nim @@ -0,0 +1,128 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import strutils, strformat, times +import karax/[karaxdsl, vdom] + +import renderutils +import ".."/[types, utils, formatters] + +proc renderCommunityTabs*(kind: QueryKind; community: Community): VNode = + let + path = &"/i/communities/{community.id}" + q = Query(kind: kind) + buildHtml(tdiv): + ul(class="tab"): + li(class=q.getTabClass(posts)): + a(href=path): text "Top" + li(class=q.getTabClass(replies)): + a(href=(path & "/latest")): text "Latest" + li(class=q.getTabClass(media)): + a(href=(path & "/media")): text "Media" + li(class=q.getTabClass(userList)): + a(href=(path & "/about")): text "About" + if community.hashtags.len > 0: + tdiv(class="community-tags"): + for tag in community.hashtags: + let bare = tag.strip(chars={'#'}) + a(class="community-tag", + href=(&"/i/communities/{community.id}/hashtag/{bare}")): + text tag + +proc renderMemberTabs*(community: Community; isModerators: bool): VNode = + let path = &"/i/communities/{community.id}" + buildHtml(ul(class="tab")): + li(class=(if not isModerators: "tab-item active" else: "tab-item")): + a(href=(path & "/members")): text "All" + li(class=(if isModerators: "tab-item active" else: "tab-item")): + a(href=(path & "/moderators")): text "Moderators" + +proc renderHashtagHeader*(community: Community; tag: string): VNode = + buildHtml(tdiv(class="community-hashtag-header")): + h2(class="community-hashtag-title"): text "#" & tag + +proc renderCommunityAbout*(community: Community; moderators: seq[User]): VNode = + buildHtml(tdiv(class="community-about")): + tdiv(class="community-info"): + h2: text "Community Info" + tdiv(class="community-info-item"): + icon "group" + if community.joinPolicy == "Open": + text "Anyone can join this Community." + else: + text "Membership is by approval only." + + tdiv(class="community-info-item"): + icon "info" + text "All Communities are publicly visible." + + tdiv(class="community-info-item"): + icon "calendar" + let + date = community.createdAt.format("MMMM d, yyyy") + creator = community.creator.username + span: + text &"Created {date} by " + a(href=(&"/{creator}")): text &"@{creator}" + if community.creator.verifiedType != none: + verifiedIcon(community.creator) + + if community.rules.len > 0: + tdiv(class="community-rules"): + h2: text "Rules" + p(class="community-rules-intro"): + text "These are set and enforced by Community admins and are in addition to " + a(href="https://help.x.com/rules-and-policies/x-rules"): text "X's rules" + text "." + + for i, rule in community.rules: + tdiv(class="community-rule"): + span(class="community-rule-number"): text $(i + 1) + tdiv(class="community-rule-content"): + strong: text rule.name + if rule.description.len > 0: + p: text rule.description + + if moderators.len > 0: + tdiv(class="community-moderators"): + h2: + text "Moderators" + a(class="community-mods-link", + href=(&"/i/communities/{community.id}/moderators")): + text "See all" + for user in moderators: + tdiv(class="community-moderator"): + a(href=(&"/{user.username}")): + genImg(user.getUserPic("_bigger"), class="community-mod-avatar") + tdiv(class="community-mod-info"): + a(href=(&"/{user.username}"), class="community-mod-name"): + text user.fullname + if user.verifiedType != none: + verifiedIcon(user) + a(href=(&"/{user.username}"), class="community-mod-username"): + text &"@{user.username}" + +proc renderCommunity*(body, nav: VNode; community: Community): VNode = + buildHtml(tdiv(class="timeline-container")): + if community.banner.len > 0: + tdiv(class="timeline-banner"): + a(href=getPicUrl(community.banner), target="_blank"): + genImg(community.banner) + + tdiv(class="community-header"): + h1(class="community-name"): + a(href=(&"/i/communities/{community.id}")): text community.name + + if community.category.len > 0: + span(class="community-category"): text community.category + + if community.description.len > 0: + tdiv(class="community-description"): + text community.description + + tdiv(class="community-stats"): + a(class="community-member-count", + href=(&"/i/communities/{community.id}/members")): + text insertSep($community.memberCount, ',') + text " Members" + + nav + body diff --git a/src/views/general.nim b/src/views/general.nim index 13ac10d..2e515bb 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -50,7 +50,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; let opensearchUrl = getUrlPrefix(cfg) & "/opensearch" buildHtml(head): - link(rel="stylesheet", type="text/css", href="/css/style.css?v=38") + link(rel="stylesheet", type="text/css", href="/css/style.css?v=39") link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=5") if theme.len > 0: diff --git a/src/views/renderutils.nim b/src/views/renderutils.nim index 0bd9789..8774f9d 100644 --- a/src/views/renderutils.nim +++ b/src/views/renderutils.nim @@ -18,7 +18,7 @@ proc getMediumPic*(url: string): string = result &= mediumWebp result = getPicUrl(result) -proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode = +proc icon*(icon: string; label=""; title=""; class=""; href=""): VNode = var c = "icon-" & icon if class.len > 0: c = &"{c} {class}" buildHtml(tdiv(class="icon-container")): @@ -27,8 +27,8 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode = else: span(class=c, title=title) - if text.len > 0: - text " " & text + if label.len > 0: + text " " & label template verifiedIcon*(user: User): untyped {.dirty.} = if user.verifiedType != VerifiedType.none: diff --git a/src/views/timeline.nim b/src/views/timeline.nim index b911765..9456bf9 100644 --- a/src/views/timeline.nim +++ b/src/views/timeline.nim @@ -154,7 +154,8 @@ proc renderTimelineTweets*(results: Timeline; prefs: Prefs; path: string; else: renderThread(thread, prefs, path, bigThumb) else: for thread in filtered: - if thread.len == 1: renderTweet(thread[0], prefs, path) + if thread.len == 1: + renderTweet(thread[0], prefs, path) else: renderThread(thread, prefs, path) var cursor = getSearchMaxId(results, path) diff --git a/src/views/tweet.nim b/src/views/tweet.nim index 9bf6c9b..e658c8e 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -27,14 +27,18 @@ proc renderArticleCard(preview: ArticlePreview; prefs: Prefs): VNode = if preview.previewText.len > 0: p(class="card-description"): text preview.previewText -proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs): VNode = +proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs; + path = ""): VNode = buildHtml(tdiv): if pinned: + let pinnedLabel = + if "/i/communities/" in path: "Pinned by Community mods" + else: "Pinned Tweet" tdiv(class="pinned"): - span: icon "pin", "Pinned Tweet" + span: icon("pin", pinnedLabel) elif retweet.len > 0: tdiv(class="retweet-header"): - span: icon "retweet", retweet & " retweeted" + span: icon("retweet", retweet & " retweeted") tdiv(class="tweet-header"): a(class="tweet-avatar", href=("/" & tweet.user.username)): @@ -358,7 +362,8 @@ proc renderLocation*(tweet: Tweet): string = return $node proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; - last=false; mainTweet=false; afterTweet=false; bigThumb=false): VNode = + last=false; mainTweet=false; afterTweet=false; + bigThumb=false): VNode = var divClass = class if index == -1 or last: divClass = "thread-last " & class @@ -391,7 +396,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; a(class="tweet-link", href=getLink(tweet)) tdiv(class="tweet-body"): - renderHeader(tweet, retweet, pinned, prefs) + renderHeader(tweet, retweet, pinned, prefs, path) if not afterTweet and index == 0 and tweet.reply.len > 0 and (tweet.reply.len > 1 or tweet.reply[0] != tweet.user.username or pinned): diff --git a/temp b/temp deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_community.py b/tests/test_community.py new file mode 100644 index 0000000..b266044 --- /dev/null +++ b/tests/test_community.py @@ -0,0 +1,211 @@ +from base import BaseTestCase +from parameterized import parameterized + + +COMMUNITY_ID = '1493446837214187523' +COMMUNITY_PATH = f'i/communities/{COMMUNITY_ID}' + + +class CommunityTest(BaseTestCase): + def test_top_page_loads(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.community-header') + self.assert_text('Build in Public', '.community-name') + + def test_banner_visible(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.timeline-banner img') + + def test_member_count(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.community-member-count') + self.assert_text('Members', '.community-member-count') + + def test_description_visible(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.community-description') + + def test_tabs_present(self): + self.open_nitter(COMMUNITY_PATH) + tabs = self.find_elements('.tab a') + labels = [t.text for t in tabs] + self.assertEqual(labels, ['Top', 'Latest', 'Media', 'About']) + + def test_top_tab_active(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.tab .active a[href$="/' + COMMUNITY_ID + '"]') + + def test_top_has_tweets(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.timeline-item .tweet-body') + + def test_top_has_pagination(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.show-more') + self.assert_text('Load more', '.show-more') + + def test_latest_has_tweets(self): + self.open_nitter(f'{COMMUNITY_PATH}/latest') + self.assert_element_visible('.timeline-item .tweet-body') + + def test_latest_tab_active(self): + self.open_nitter(f'{COMMUNITY_PATH}/latest') + self.assert_element_visible('.tab .active a[href$="/latest"]') + + def test_media_has_tweets(self): + self.open_nitter(f'{COMMUNITY_PATH}/media') + self.assert_element_visible('.timeline-item .tweet-body') + + def test_media_tab_active(self): + self.open_nitter(f'{COMMUNITY_PATH}/media') + self.assert_element_visible('.tab .active a[href$="/media"]') + + def test_about_page(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + self.assert_element_visible('.community-about') + self.assert_text('Community Info', '.community-info h2') + + def test_about_rules(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + self.assert_element_visible('.community-rules') + self.assert_text('Rules', '.community-rules h2') + rules = self.find_elements('.community-rule') + self.assertGreater(len(rules), 0) + + def test_about_creator(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + self.assert_text('Created', '.community-info') + link = self.find_element('.community-info-item a') + self.assertTrue(link.text.startswith('@')) + self.assertGreater(len(link.text), 1) + + def test_about_tab_active(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + self.assert_element_visible('.tab .active a[href$="/about"]') + + def test_about_moderators(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + self.assert_element_visible('.community-moderators') + self.assert_text('Moderators', '.community-moderators h2') + mods = self.find_elements('.community-moderator') + self.assertGreater(len(mods), 0) + + def test_about_moderators_have_avatars(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + avatars = self.find_elements('.community-mod-avatar') + self.assertGreater(len(avatars), 0) + + def test_about_moderators_link_to_profiles(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + links = self.find_elements('.community-mod-username') + self.assertGreater(len(links), 0) + for link in links: + self.assertTrue(link.text.startswith('@')) + self.assertTrue(link.get_attribute('href').startswith('http')) + + def test_about_see_all_link(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + link = self.find_element('.community-mods-link') + self.assertEqual(link.text, 'See all') + self.assertIn('/moderators', link.get_attribute('href')) + + def test_members_page(self): + self.open_nitter(f'{COMMUNITY_PATH}/members') + self.assert_element_visible('.timeline-item') + users = self.find_elements('.timeline-item .username') + self.assertGreater(len(users), 0) + + def test_members_has_member_tabs(self): + self.open_nitter(f'{COMMUNITY_PATH}/members') + tabs = self.find_elements('.tab a') + labels = [t.text for t in tabs] + self.assertEqual(labels, ['All', 'Moderators']) + + def test_members_all_tab_active(self): + self.open_nitter(f'{COMMUNITY_PATH}/members') + self.assert_element_visible('.tab .active a[href$="/members"]') + + def test_members_count_is_link(self): + self.open_nitter(COMMUNITY_PATH) + link = self.find_element('.community-member-count') + self.assertIn('Members', link.text) + self.assertIn('/members', link.get_attribute('href')) + + def test_moderators_page(self): + self.open_nitter(f'{COMMUNITY_PATH}/moderators') + self.assert_element_visible('.timeline-item') + users = self.find_elements('.timeline-item .username') + self.assertGreater(len(users), 0) + + def test_moderators_tab_active(self): + self.open_nitter(f'{COMMUNITY_PATH}/moderators') + self.assert_element_visible('.tab .active a[href$="/moderators"]') + + def test_moderators_has_member_tabs(self): + self.open_nitter(f'{COMMUNITY_PATH}/moderators') + tabs = self.find_elements('.tab a') + labels = [t.text for t in tabs] + self.assertEqual(labels, ['All', 'Moderators']) + + def test_pinned_tweet_label(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.pinned') + self.assert_text('Pinned by Community mods', '.pinned') + + def test_hashtags_visible(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.community-tags') + tags = self.find_elements('.community-tag') + self.assertGreater(len(tags), 0) + + def test_hashtags_are_links(self): + self.open_nitter(COMMUNITY_PATH) + tags = self.find_elements('.community-tag') + for tag in tags: + href = tag.get_attribute('href') + self.assertIn('/hashtag/', href) + self.assertTrue(tag.text.startswith('#')) + + def test_hashtag_page(self): + self.open_nitter(f'{COMMUNITY_PATH}/hashtag/buildinpublic') + self.assert_element_visible('.timeline-item .tweet-body') + + def test_hashtag_shows_header(self): + self.open_nitter(f'{COMMUNITY_PATH}/hashtag/buildinpublic') + self.assert_element_visible('.community-header') + self.assert_text('Build in Public', '.community-name') + + def test_hashtag_shows_tag_title(self): + self.open_nitter(f'{COMMUNITY_PATH}/hashtag/buildinpublic') + self.assert_element_visible('.community-hashtag-header') + self.assert_text('#buildinpublic', '.community-hashtag-title') + + def test_hashtag_no_main_tabs(self): + self.open_nitter(f'{COMMUNITY_PATH}/hashtag/buildinpublic') + tabs = self.find_elements('.tab a') + tab_labels = [t.text for t in tabs] + self.assertNotIn('Top', tab_labels) + self.assertNotIn('About', tab_labels) + + def test_category_visible(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.community-category') + + def test_about_join_policy(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + self.assert_text('Anyone can join', '.community-info') + + def test_about_visibility_note(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + self.assert_text('publicly visible', '.community-info') + + def test_404_invalid_id(self): + self.open_nitter('i/communities/999') + self.assert_element_visible('.error-panel') + self.assert_text('not found', '.error-panel') + + @parameterized.expand(['', '/latest', '/media', '/about', + '/members', '/moderators']) + def test_page_no_error(self, suffix): + self.open_nitter(f'{COMMUNITY_PATH}{suffix}') + self.assert_element_not_visible('.error-panel') From 44b2f096f67da2cc257a0e262a94a7ae79e95d47 Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 18 Jun 2026 15:01:48 +0200 Subject: [PATCH 27/47] Fix SSRF in /video proxy and API JSON injection Validate the target host on the /video media route with isTwitterUrl() (mirroring /pic) and reject non-http(s) schemes, and stop the media proxy following redirects off the validated host. JSON-escape user-controlled GraphQL cursors and build id variables with packedjson so untrusted input can't break out of the query. Warn on startup when the insecure default hmacKey is in use. Fixes #1411 --- nitter.example.conf | 2 +- src/api.nim | 33 +++++++++++++++++++-------------- src/consts.nim | 2 -- src/nitter.nim | 4 ++++ src/routes/media.nim | 8 +++++--- src/utils.nim | 4 ++-- tests/test_ssrf_1411.nim | 38 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 69 insertions(+), 22 deletions(-) create mode 100644 tests/test_ssrf_1411.nim diff --git a/nitter.example.conf b/nitter.example.conf index a58c63c..32fb618 100644 --- a/nitter.example.conf +++ b/nitter.example.conf @@ -20,7 +20,7 @@ redisMaxConnections = 30 # you receive tons of requests per second [Config] -hmacKey = "secretkey" # random key for cryptographic signing of video urls +hmacKey = "secretkey" # CHANGE THIS to a unique random value (e.g. `openssl rand -hex 32`); signs media urls base64Media = false # use base64 encoding for proxied media urls enableRSS = true # master switch, set to false to disable all RSS feeds enableRSSUserTweets = true # /@user/rss diff --git a/src/api.nim b/src/api.nim index 9aa6c50..4dd6a3d 100644 --- a/src/api.nim +++ b/src/api.nim @@ -18,6 +18,11 @@ proc apiReq(endpoint, variables: string; fieldToggles = ""; skipTid = false): Ap let url = apiUrl(endpoint, variables, fieldToggles, skipTid) return ApiReq(cookie: url, oauth: url) +proc cursorParam(after: string): string = + ## JSON-escape the user-supplied cursor so it cannot break out of the GraphQL + ## variables object (same input-validation class as the #1411 media SSRF). + if after.len > 0: "\"cursor\":" & $(%after) & "," else: "" + proc mediaUrl(id, cursor: string; count=20): ApiReq = result = ApiReq( cookie: apiUrl(graphUserMedia, userMediaVars % [id, cursor, $count]), @@ -39,10 +44,10 @@ proc tweetDetailUrl(id: string; cursor: string): ApiReq = # ) proc userUrl(username: string): ApiReq = - let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username + let cookieVars = $(%*{"screen_name": username, "withGrokTranslatedBio": false}) result = ApiReq( cookie: apiUrl(graphUser, cookieVars, tweetDetailFieldToggles), - oauth: apiUrl(graphUserV2, """{"screen_name": "$1"}""" % username) + oauth: apiUrl(graphUserV2, $(%*{"screen_name": username})) ) proc getGraphUser*(username: string): Future[User] {.async.} = @@ -60,7 +65,7 @@ proc getGraphUserById*(id: string): Future[User] {.async.} = proc getAboutAccount*(username: string): Future[AccountInfo] {.async.} = if username.len == 0: return let - url = apiReq(graphAboutAccount, """{"screenName":"$1"}""" % username) + url = apiReq(graphAboutAccount, $(%*{"screenName": username})) js = await fetch(url) result = parseAboutAccount(js) @@ -71,7 +76,7 @@ proc restReq(endpoint: string; params: seq[(string, string)] = @[]): ApiReq = proc getBroadcastInfo*(id: string): Future[Broadcast] {.async.} = if id.len == 0: return let - req = apiReq(graphBroadcast, """{"id":"$1"}""" % id) + req = apiReq(graphBroadcast, $(%*{"id": id})) js = await fetch(req) result = parseBroadcastInfo(js) @@ -86,7 +91,7 @@ proc fetchBroadcastStream*(mediaKey: string): Future[string] {.async.} = proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} = if id.len == 0: return let - cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + cursor = cursorParam(after) url = case kind of TimelineKind.tweets: userTweetsUrl(id, cursor) of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor) @@ -97,14 +102,14 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi proc getGraphCommunity*(id: string): Future[Community] {.async.} = if id.len == 0: return let - url = apiReq(graphCommunity, communityVars % id) + url = apiReq(graphCommunity, $(%*{"communityId": id})) js = await fetch(url) result = parseGraphCommunity(js) proc getGraphCommunityTweets*(id: string; rankingMode: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return let - cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + cursor = cursorParam(after) url = apiReq(graphCommunityTweets, communityTweetsVars % [id, cursor, rankingMode]) js = await fetch(url) result = parseGraphCommunityTimeline(js, after) @@ -112,7 +117,7 @@ proc getGraphCommunityTweets*(id: string; rankingMode: string; after=""): Future proc getGraphCommunityMedia*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return let - cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + cursor = cursorParam(after) url = apiReq(graphCommunityMedia, communityMediaVars % [id, cursor]) js = await fetch(url) result = parseGraphCommunityTimeline(js, after) @@ -124,7 +129,7 @@ proc communitySliceReq(endpoint, variables: string): ApiReq = proc getGraphCommunityMembers*(id: string; after=""): Future[Result[User]] {.async.} = if id.len == 0: return let - cursor = if after.len > 0: "\"$1\"" % after else: "null" + cursor = if after.len > 0: $(%after) else: "null" url = communitySliceReq(graphCommunityMembers, communityMembersVars % [id, cursor]) js = await fetch(url) result = parseGraphCommunityMembers(js, after) @@ -140,7 +145,7 @@ proc getGraphCommunityHashtags*(id, hashtag: string; after=""): Future[Timeline] if id.len == 0 or hashtag.len == 0: return let safeTag = multiReplace(hashtag, ("\"", ""), ("\\", "")) - cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + cursor = cursorParam(after) url = apiReq(graphCommunityHashtags, communityHashtagsVars % [id, cursor, safeTag]) js = await fetch(url) result = parseGraphCommunityTimeline(js, after) @@ -148,7 +153,7 @@ proc getGraphCommunityHashtags*(id, hashtag: string; after=""): Future[Timeline] proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return let - cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + cursor = cursorParam(after) url = apiReq(graphListTweets, restIdVars % [id, cursor, "20"]) js = await fetch(url) result = parseGraphTimeline(js, after).tweets @@ -162,7 +167,7 @@ proc getGraphListBySlug*(name, list: string): Future[List] {.async.} = proc getGraphList*(id: string): Future[List] {.async.} = let - url = apiReq(graphListById, """{"listId": "$1"}""" % id) + url = apiReq(graphListById, $(%*{"listId": id})) js = await fetch(url) result = parseGraphList(js) @@ -186,14 +191,14 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} = if id.len == 0: return let - url = apiReq(graphTweetResult, """{"rest_id": "$1"}""" % id) + url = apiReq(graphTweetResult, $(%*{"rest_id": id})) js = await fetch(url) result = parseGraphTweetResult(js) proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} = if id.len == 0: return let - cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + cursor = cursorParam(after) js = await fetch(tweetDetailUrl(id, cursor)) result = parseGraphConversation(js, id) diff --git a/src/consts.nim b/src/consts.nim index 59b986c..fead5bf 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -161,8 +161,6 @@ const articleFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withArticleSummaryText":true,"withArticleVoiceOver":true}""" - communityVars* = """{"communityId":"$1"}""" - communityTweetsVars* = """{ "communityId": "$1", $2 "count": 20, diff --git a/src/nitter.nim b/src/nitter.nim index 955849b..d629f6d 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -34,6 +34,10 @@ stdout.flushFile updateDefaultPrefs(fullCfg) setCacheTimes(cfg) setHmacKey(cfg.hmacKey) +if cfg.hmacKey.len == 0 or cfg.hmacKey == "secretkey": + stderr.write "WARNING: insecure default 'hmacKey' in nitter.conf; " & + "set a unique random value to stop media URL signatures being forgeable.\n" + stderr.flushFile setProxyEncoding(cfg.base64Media) setMaxHttpConns(cfg.httpMaxConns) setHttpProxy(cfg.proxy, cfg.proxyAuth) diff --git a/src/routes/media.nim b/src/routes/media.nim index df30d5f..40f5a6c 100644 --- a/src/routes/media.nim +++ b/src/routes/media.nim @@ -15,7 +15,9 @@ const maxAge* = "max-age=604800" proc safeFetch*(url: string): Future[string] {.async.} = - let client = newAsyncHttpClient() + # maxRedirects=0: the caller already validated the host, so never follow a + # redirect off the allowlisted host (would re-open the #1411 SSRF). + let client = newAsyncHttpClient(maxRedirects = 0) try: result = await client.getContent(url) except: discard finally: client.close() @@ -32,7 +34,7 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} = result = Http200 let request = req.getNativeReq() - client = newAsyncHttpClient() + client = newAsyncHttpClient(maxRedirects = 0) try: let res = await client.get(url) @@ -122,7 +124,7 @@ proc createMediaRouter*(cfg: Config) = get re"^\/video\/(enc)?\/?(.+)\/(.+)$": let url = decoded(request, 2) - cond "http" in url + cond isTwitterUrl(url) if getHmac(url) != request.matches[1]: resp Http403, showError("Failed to verify signature", cfg) diff --git a/src/utils.nim b/src/utils.nim index 391e2a3..f4ae560 100644 --- a/src/utils.nim +++ b/src/utils.nim @@ -57,8 +57,8 @@ proc filterParams*(params: Table): seq[(string, string)] = result.add p proc isTwitterUrl*(uri: Uri): bool = - uri.hostname in twitterDomains or - uri.hostname.endsWith(".video.pscp.tv") + uri.scheme in ["http", "https"] and + (uri.hostname in twitterDomains or uri.hostname.endsWith(".video.pscp.tv")) proc isTwitterUrl*(url: string): bool = isTwitterUrl(parseUri(url)) diff --git a/tests/test_ssrf_1411.nim b/tests/test_ssrf_1411.nim new file mode 100644 index 0000000..0cb4bf2 --- /dev/null +++ b/tests/test_ssrf_1411.nim @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Reproduction + regression test for issue #1411: +# SSRF via /video proxy with default HMAC key and missing host validation. +import std/[unittest, uri] +import ".."/src/utils + +suite "issue #1411 SSRF via /video proxy": + setup: + # The default key shipped in nitter.example.conf / config.nim. + setHmacKey("secretkey") + + test "HMAC for arbitrary SSRF URLs is forgeable with the default key": + # These signatures were independently computed (Python hmac-sha256, uppercase + # hex, first 13 chars) and observed live in the issue report. + check getHmac("http://172.17.0.1:19999/secret_data.m3u8") == "BBD19ACC6C012" + check getHmac("http://172.17.0.1:19999/secret_data.mp4") == "0780F00DDF3E7" + + test "isTwitterUrl rejects SSRF targets (the guard /video is missing)": + # Internal / metadata hosts an attacker would target. + check isTwitterUrl(parseUri("http://172.17.0.1:19999/secret_data.m3u8")) == false + check isTwitterUrl(parseUri("http://169.254.169.254/latest/meta-data/x.m3u8")) == false + check isTwitterUrl(parseUri("http://localhost/x.mp4")) == false + check isTwitterUrl(parseUri("http://[::1]/x.mp4")) == false + + test "isTwitterUrl rejects userinfo / look-alike host bypass attempts": + check isTwitterUrl(parseUri("http://video.twimg.com@169.254.169.254/x.mp4")) == false + check isTwitterUrl(parseUri("http://video.twimg.com.evil.com/x.mp4")) == false + check isTwitterUrl(parseUri("http://evilvideo.twimg.com.attacker/x.mp4")) == false + + test "isTwitterUrl rejects non-http schemes even on a Twitter host": + check isTwitterUrl(parseUri("gopher://video.twimg.com/x.mp4")) == false + check isTwitterUrl(parseUri("file:///etc/passwd")) == false + check isTwitterUrl(parseUri("ftp://video.twimg.com/x.mp4")) == false + + test "isTwitterUrl still allows legitimate Twitter video hosts": + check isTwitterUrl(parseUri("https://video.twimg.com/ext_tw_video/1/pu/pl/x.m3u8")) == true + check isTwitterUrl(parseUri("https://video.twimg.com/amplify_video/1/vid/x.mp4")) == true + check isTwitterUrl(parseUri("https://prod-fastly-us-east-1.video.pscp.tv/x.m3u8")) == true From bfcf75af7fbe7a17274b32edb611eee148969522 Mon Sep 17 00:00:00 2001 From: Zed Date: Thu, 18 Jun 2026 15:42:17 +0200 Subject: [PATCH 28/47] Fix invalid/empty image hrefs Fixes #1413 --- src/utils.nim | 2 ++ src/views/general.nim | 1 + src/views/renderutils.nim | 2 ++ 3 files changed, 5 insertions(+) diff --git a/src/utils.nim b/src/utils.nim index f4ae560..95b46de 100644 --- a/src/utils.nim +++ b/src/utils.nim @@ -40,12 +40,14 @@ proc getVidUrl*(link: string): string = &"/video/{sig}/{encodeUrl(link)}" proc getPicUrl*(link: string): string = + if link.len == 0: return if base64Media: &"/pic/enc/{encode(link, safe=true)}" else: &"/pic/{encodeUrl(link)}" proc getOrigPicUrl*(link: string): string = + if link.len == 0: return if base64Media: &"/pic/orig/enc/{encode(link, safe=true)}" else: diff --git a/src/views/general.nim b/src/views/general.nim index 2e515bb..d3ba088 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -97,6 +97,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; link(rel="preload", type="image/png", href=bannerUrl, `as`="image") for url in images: + if url.len == 0: continue let preloadUrl = if "400x400" in url: getPicUrl(url) else: getSmallPic(url) link(rel="preload", type="image/png", href=preloadUrl, `as`="image") diff --git a/src/views/renderutils.nim b/src/views/renderutils.nim index 8774f9d..af2f05d 100644 --- a/src/views/renderutils.nim +++ b/src/views/renderutils.nim @@ -7,12 +7,14 @@ const smallWebp* = "?name=small&format=webp" const mediumWebp* = "?name=medium&format=webp" proc getSmallPic*(url: string): string = + if url.len == 0: return result = url if "?" notin url and not url.endsWith("placeholder.png"): result &= smallWebp result = getPicUrl(result) proc getMediumPic*(url: string): string = + if url.len == 0: return result = url if "?" notin url and not url.endsWith("placeholder.png"): result &= mediumWebp From 581c6f1714b64e0c6e24bf47c238821803638966 Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 19 Jun 2026 22:00:28 +0200 Subject: [PATCH 29/47] Add retry logic and timeout to media proxy --- src/routes/media.nim | 98 +++++++++++++++++++++++++++----------------- 1 file changed, 61 insertions(+), 37 deletions(-) diff --git a/src/routes/media.nim b/src/routes/media.nim index 40f5a6c..0fa02aa 100644 --- a/src/routes/media.nim +++ b/src/routes/media.nim @@ -1,6 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only import uri, strutils, httpclient, os, hashes, base64, re import asynchttpserver, asyncstreams, asyncfile, asyncnet +import asyncdispatch import jester @@ -32,47 +33,70 @@ template respond*(req: asynchttpserver.Request; headers) = proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} = result = Http200 - let - request = req.getNativeReq() - client = newAsyncHttpClient(maxRedirects = 0) + let request = req.getNativeReq() - try: - let res = await client.get(url) - if res.status != "200 OK": - if res.status != "404 Not Found": - echo "[media] Proxying failed, status: $1, url: $2" % [res.status, url] - return Http404 - - let hashed = $hash(url) - if request.headers.getOrDefault("If-None-Match") == hashed: - return Http304 - - let contentLength = - if res.headers.hasKey("content-length"): - res.headers["content-length", 0] + for attempt in 0 .. 2: + let client = newAsyncHttpClient(maxRedirects = 0) + var shouldRetry = false + try: + let resFut = client.get(url) + let completed = await withTimeout(resFut, 5000) + if not completed: + if attempt < 2: + echo "[media] Retry $1/2, timeout after 5s, url: $2" % [$(attempt + 1), url] + shouldRetry = true + else: + echo "[media] Proxying timeout after 5s, url: $1" % [url] + return Http504 else: - "" + let res = resFut.read() + if res.status != "200 OK": + if res.status == "404 Not Found": + return Http404 + if attempt < 2: + echo "[media] Retry $1/2, status: $2, url: $3" % [$(attempt + 1), res.status, url] + shouldRetry = true + else: + echo "[media] Proxying failed, status: $1, url: $2" % [res.status, url] + return Http404 + else: + let hashed = $hash(url) + if request.headers.getOrDefault("If-None-Match") == hashed: + return Http304 - let headers = newHttpHeaders({ - "content-type": res.headers["content-type", 0], - "content-length": contentLength, - "cache-control": maxAge, - "etag": hashed - }) + let contentLength = + if res.headers.hasKey("content-length"): + res.headers["content-length", 0] + else: + "" - respond(request, headers) + let headers = newHttpHeaders({ + "content-type": res.headers["content-type", 0], + "content-length": contentLength, + "cache-control": maxAge, + "etag": hashed + }) - var (hasValue, data) = (true, "") - while hasValue: - (hasValue, data) = await res.bodyStream.read() - if hasValue: - await request.client.send(data) - data.setLen 0 - except HttpRequestError, ProtocolError, OSError: - echo "[media] Proxying exception, error: $1, url: $2" % [getCurrentExceptionMsg(), url] - result = Http404 - finally: - client.close() + respond(request, headers) + + var (hasValue, data) = (true, "") + while hasValue: + (hasValue, data) = await res.bodyStream.read() + if hasValue: + await request.client.send(data) + data.setLen 0 + return Http200 + except CatchableError: + if attempt < 2: + echo "[media] Retry $1/2, error: $2, url: $3" % [$(attempt + 1), getCurrentExceptionMsg(), url] + shouldRetry = true + else: + echo "[media] Proxying exception, error: $1, url: $2" % [getCurrentExceptionMsg(), url] + result = Http404 + finally: + client.close() + if not shouldRetry: + break template check*(code): untyped = if code != Http200: @@ -129,7 +153,7 @@ proc createMediaRouter*(cfg: Config) = if getHmac(url) != request.matches[1]: resp Http403, showError("Failed to verify signature", cfg) - if ".mp4" in url or ".ts" in url or ".m4s" in url: + if ".mp4" in url or ".ts" in url or ".m4s" in url or ".aac" in url: let code = await proxyMedia(request, url) check code From 17ad0fec34bf1aff97efcf9079adc01fa44119ba Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 19 Jun 2026 22:00:31 +0200 Subject: [PATCH 30/47] Add audio playback support to HLS.js handler --- public/js/hlsPlayback.js | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/public/js/hlsPlayback.js b/public/js/hlsPlayback.js index 9919fec..5970011 100644 --- a/public/js/hlsPlayback.js +++ b/public/js/hlsPlayback.js @@ -1,27 +1,30 @@ // @license http://www.gnu.org/licenses/agpl-3.0.html AGPL-3.0 // SPDX-License-Identifier: AGPL-3.0-only -function playVideo(overlay) { - const video = overlay.parentElement.querySelector('video'); - const url = video.getAttribute("data-url"); - const startTime = parseFloat(video.getAttribute("data-start") || "0"); - video.setAttribute("controls", ""); +function playMedia(overlay, tagName) { + const media = overlay.parentElement.querySelector(tagName); + const url = media.getAttribute("data-url"); + const startTime = parseFloat(media.getAttribute("data-start") || "0"); + media.setAttribute("controls", ""); overlay.style.display = "none"; if (Hls.isSupported()) { var hls = new Hls({autoStartLoad: false}); hls.loadSource(url); - hls.attachMedia(video); + hls.attachMedia(media); hls.on(Hls.Events.MANIFEST_PARSED, function () { hls.loadLevel = hls.levels.length - 1; hls.startLoad(startTime); - video.play(); + media.play(); }); - } else if (video.canPlayType('application/vnd.apple.mpegurl')) { - video.src = url; - video.addEventListener('canplay', function() { - if (startTime > 0) video.currentTime = startTime; - video.play(); + } else if (media.canPlayType('application/vnd.apple.mpegurl')) { + media.src = url; + media.addEventListener('canplay', function() { + if (startTime > 0) media.currentTime = startTime; + media.play(); }); } } + +function playVideo(overlay) { playMedia(overlay, 'video'); } +function playAudio(overlay) { playMedia(overlay, 'audio'); } // @license-end From db5a229bfe2b2c376cc452dd0266557bcf25aae7 Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 19 Jun 2026 22:00:36 +0200 Subject: [PATCH 31/47] Add Twitter Spaces support Fixes #459 --- src/api.nim | 13 ++++ src/consts.nim | 1 + src/nitter.nim | 4 +- src/parser.nim | 53 ++++++++++++++- src/redis_cache.nim | 15 +++++ src/routes/space.nim | 36 ++++++++++ src/sass/_space.scss | 149 ++++++++++++++++++++++++++++++++++++++++++ src/sass/index.scss | 1 + src/types.nim | 22 +++++++ src/views/general.nim | 2 +- src/views/space.nim | 86 ++++++++++++++++++++++++ 11 files changed, 379 insertions(+), 3 deletions(-) create mode 100644 src/routes/space.nim create mode 100644 src/sass/_space.scss create mode 100644 src/views/space.nim diff --git a/src/api.nim b/src/api.nim index 4dd6a3d..f2085df 100644 --- a/src/api.nim +++ b/src/api.nim @@ -88,6 +88,19 @@ proc fetchBroadcastStream*(mediaKey: string): Future[string] {.async.} = result = streamJs{"source", "noRedirectPlaybackUrl"}.getStr( streamJs{"source", "location"}.getStr) +proc getAudioSpace*(id: string): Future[AudioSpace] {.async.} = + if id.len == 0: return + let + variables = %*{ + "id": id, + "isMetatagsQuery": false, + "withReplays": true, + "withListeners": true + } + req = apiReq(graphAudioSpace, $variables) + js = await fetch(req) + result = parseAudioSpace(js) + proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} = if id.len == 0: return let diff --git a/src/consts.nim b/src/consts.nim index fead5bf..8ec4f04 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -39,6 +39,7 @@ const graphTweetResultsByRestIds* = "Sc9EUQTZNEH-wzegn-nHvQ/TweetResultsByRestIds" graphBroadcast* = "FJLCzpXCLPM1jUZqmM7oEA/BroadcastQuery" + graphAudioSpace* = "rWRLsOhNJ2xjpI1tREYurQ/AudioSpaceById" restLiveStream* = "1.1/live_video_stream/status/" gqlFeatures* = """{ diff --git a/src/nitter.nim b/src/nitter.nim index d629f6d..dc38161 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -10,7 +10,7 @@ import types, config, prefs, formatters, redis_cache, http_pool, auth, apiutils import views/[general, about] import routes/[ preferences, timeline, status, media, search, rss, list, community, debug, - unsupported, embed, resolver, broadcast, article, router_utils] + unsupported, embed, resolver, broadcast, space, article, router_utils] const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances" const issuesUrl = "https://github.com/zedeus/nitter/issues" @@ -65,6 +65,7 @@ createMediaRouter(cfg) createEmbedRouter(cfg) createRssRouter(cfg) createBroadcastRouter(cfg) +createSpaceRouter(cfg) createDebugRouter(cfg) settings: @@ -136,5 +137,6 @@ routes: extend resolver, "" extend embed, "" extend broadcastRoute, "" + extend spaceRoute, "" extend debug, "" extend unsupported, "" diff --git a/src/parser.nim b/src/parser.nim index c810f76..04a7d33 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -131,6 +131,51 @@ proc parseBroadcastInfo*(js: JsonNode): Broadcast = user: parseGraphUser(bc) ) +proc parseSpaceParticipant(js: JsonNode): SpaceParticipant = + result = SpaceParticipant( + userId: js{"user_results", "rest_id"}.getStr, + username: js{"twitter_screen_name"}.getStr, + displayName: js{"display_name"}.getStr, + avatarUrl: js{"avatar_url"}.getStr, + isVerified: js{"is_verified"}.getBool or + js{"user_results", "result", "is_blue_verified"}.getBool + ) + +proc parseAudioSpace*(js: JsonNode): AudioSpace = + let space = ? js{"data", "audioSpace"} + let meta = space{"metadata"} + + result = AudioSpace( + id: meta{"rest_id"}.getStr, + title: meta{"title"}.getStr, + state: meta{"state"}.getStr.toUpperAscii, + mediaKey: meta{"media_key"}.getStr, + totalLiveListeners: meta{"total_live_listeners"}.getInt, + totalReplayWatched: meta{"total_replay_watched"}.getInt, + availableForReplay: meta{"is_space_available_for_replay"}.getBool + ) + + let startedAt = meta{"started_at"}.getInt(0) + if startedAt > 0: + result.startTime = fromUnix(startedAt div 1000).utc() + + let endedAtStr = meta{"ended_at"}.getStr + if endedAtStr.len > 0: + try: + let endedAt = parseBiggestInt(endedAtStr) + if endedAt > 0: + result.endTime = fromUnix(endedAt div 1000).utc() + except ValueError: + discard + + result.creator = parseGraphUser(meta{"creator_results", "result"}) + + for admin in space{"participants", "admins"}: + result.admins.add parseSpaceParticipant(admin) + + for speaker in space{"participants", "speakers"}: + result.speakers.add parseSpaceParticipant(speaker) + proc parseGraphCommunity*(js: JsonNode): Community = if js.isNull: return let c = ? js{"data", "communityResults", "result"} @@ -390,7 +435,13 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card = result.url = vals{"player_url"}.getStrVal if "youtube.com" in result.url: result.url = result.url.replace("/embed/", "/watch?v=") - of audiospace, unknown: + of audiospace: + let spaceId = vals{"id"}.getStrVal + if spaceId.len > 0: + result.url = "/i/spaces/" & spaceId + result.title = "Twitter Space" + result.text = "Click to view Space" + of unknown: result.title = "This card type is not supported." else: discard diff --git a/src/redis_cache.nim b/src/redis_cache.nim index 4d5bf28..b9ddbcc 100644 --- a/src/redis_cache.nim +++ b/src/redis_cache.nim @@ -173,6 +173,21 @@ proc getCachedBroadcast*(id: string): Future[Broadcast] {.async.} = await cache(result) result.m3u8Url = await fetchBroadcastStream(result.mediaKey) +proc cache*(data: AudioSpace) {.async.} = + if data.id.len == 0: return + let ttl = if data.state == "RUNNING": baseCacheTime div 6 else: baseCacheTime + await setEx("sp:" & data.id, ttl, compress(toFlatty(data))) + +proc getCachedAudioSpace*(id: string): Future[AudioSpace] {.async.} = + if id.len == 0: return + let cached = await get("sp:" & id) + if cached != redisNil: + cached.deserialize(AudioSpace) + else: + result = await getAudioSpace(id) + await cache(result) + result.m3u8Url = await fetchBroadcastStream(result.mediaKey) + proc cache*(data: AccountInfo; name: string) {.async.} = await setEx("ai:" & toLower(name), baseCacheTime * 24, compress(toFlatty(data))) diff --git a/src/routes/space.nim b/src/routes/space.nim new file mode 100644 index 0000000..bd956ea --- /dev/null +++ b/src/routes/space.nim @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import asyncdispatch, strutils +import jester + +import router_utils +import ".."/[types, formatters, redis_cache] +import ../views/[general, space] +import media + +export space + +proc createSpaceRouter*(cfg: Config) = + router spaceRoute: + get "/i/spaces/@id": + cond @"id".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9'}) + let sp = await getCachedAudioSpace(@"id") + + if sp.id.len == 0: + resp Http404, showError("Space not found", cfg) + + let prefs = requestPrefs() + resp renderMain(renderSpace(sp, prefs, request.path), request, cfg, prefs, + sp.title, ogTitle=sp.title) + + get "/i/spaces/@id/stream": + cond @"id".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9'}) + let sp = await getCachedAudioSpace(@"id") + + if sp.m3u8Url.len == 0: + resp Http404 + + let manifest = await safeFetch(sp.m3u8Url) + if manifest.len == 0: + resp Http502 + + resp proxifyVideo(manifest, requestPrefs().proxyVideos, sp.m3u8Url), m3u8Mime diff --git a/src/sass/_space.scss b/src/sass/_space.scss new file mode 100644 index 0000000..5fe2e7e --- /dev/null +++ b/src/sass/_space.scss @@ -0,0 +1,149 @@ +.space-page { + max-width: 800px; + width: 100%; + margin: 20px auto 0; +} + +.space-panel { + background-color: var(--bg_panel); + border: 1px solid var(--border_grey); + border-radius: 8px; + overflow: hidden; +} + +.space-player { + position: relative; + background: linear-gradient(135deg, #7b2a8c 0%, #9b3ab1 100%); + min-height: 140px; + display: flex; + align-items: center; + justify-content: center; + + audio { + width: 100%; + padding: 15px; + box-sizing: border-box; + + &:not([controls]) { + display: none; + } + } + + .video-overlay { + background-color: transparent; + } +} + +.space-live { + background: #e0245e; + color: white; + padding: 3px 8px; + border-radius: 4px; + font-weight: bold; + font-size: 12px; + text-transform: uppercase; + position: absolute; + top: 8px; + right: 8px; +} + +.space-info { + padding: 16px; +} + +.space-header { + display: flex; + justify-content: space-between; + align-items: flex-start; + gap: 12px; + margin-bottom: 16px; +} + +.space-title { + font-size: 18px; + font-weight: bold; + margin: 0; + line-height: 1.3; + flex: 1; +} + +.space-meta { + display: flex; + flex-direction: column; + align-items: flex-end; + gap: 4px; + flex-shrink: 0; + font-size: 14px; + color: var(--fg_faded); +} + +.listener-count { + color: var(--fg_color); +} + +.space-state { + color: var(--fg_dark); +} + +.space-participants { + border-top: 1px solid var(--border_grey); + padding-top: 12px; +} + +.space-participant { + margin-bottom: 10px; + + a { + display: flex; + align-items: center; + gap: 10px; + color: var(--fg_color); + padding: 6px 0; + } + + img { + width: 40px; + height: 40px; + border-radius: 50%; + flex-shrink: 0; + } +} + +.participant-info { + min-width: 0; +} + +.participant-name { + display: flex; + align-items: center; + gap: 4px; + + strong { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .verified-icon { + margin-bottom: 0; + position: relative; + top: -2px; + } +} + +.host-badge { + background: var(--accent); + color: white; + padding: 2px 7px; + border-radius: 3px; + font-size: 11px; + font-weight: 600; + line-height: 1; + position: relative; + top: 1px; +} + +.participant-username { + color: var(--fg_dark); + font-size: 13px; +} diff --git a/src/sass/index.scss b/src/sass/index.scss index b8c9a8c..404f7d5 100644 --- a/src/sass/index.scss +++ b/src/sass/index.scss @@ -8,6 +8,7 @@ @import "timeline"; @import "search"; @import "broadcast"; +@import "space"; @import "_article"; body { diff --git a/src/types.nim b/src/types.nim index 79a99c0..f51f349 100644 --- a/src/types.nim +++ b/src/types.nim @@ -128,6 +128,28 @@ type availableForReplay*: bool user*: User + SpaceParticipant* = object + userId*: string + username*: string + displayName*: string + avatarUrl*: string + isVerified*: bool + + AudioSpace* = object + id*: string + title*: string + state*: string + mediaKey*: string + m3u8Url*: string + totalLiveListeners*: int + totalReplayWatched*: int + startTime*: DateTime + endTime*: DateTime + availableForReplay*: bool + creator*: User + admins*: seq[SpaceParticipant] + speakers*: seq[SpaceParticipant] + VideoType* = enum m3u8 = "application/x-mpegURL" mp4 = "video/mp4" diff --git a/src/views/general.nim b/src/views/general.nim index d3ba088..c273479 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -50,7 +50,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; let opensearchUrl = getUrlPrefix(cfg) & "/opensearch" buildHtml(head): - link(rel="stylesheet", type="text/css", href="/css/style.css?v=39") + link(rel="stylesheet", type="text/css", href="/css/style.css?v=42") link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=5") if theme.len > 0: diff --git a/src/views/space.nim b/src/views/space.nim new file mode 100644 index 0000000..a5cac7b --- /dev/null +++ b/src/views/space.nim @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import strutils, times +import karax/[karaxdsl, vdom] + +import renderutils +import ".."/[types, utils, formatters] + +proc renderParticipant(p: SpaceParticipant; role: string): VNode = + buildHtml(tdiv(class="space-participant")): + a(href=("/" & p.username)): + genImg(p.avatarUrl.replace("_normal", "_bigger")) + tdiv(class="participant-info"): + tdiv(class="participant-name"): + strong: text p.displayName + if p.isVerified: + tdiv(class="verified-icon blue"): + icon "circle", class="verified-icon-circle", title="Verified account" + icon "ok", class="verified-icon-check", title="Verified account" + if role.len > 0: + span(class="host-badge"): text role + span(class="participant-username"): text "@" & p.username + +proc renderSpace*(sp: AudioSpace; prefs: Prefs; path: string): VNode = + let + isLive = sp.state == "RUNNING" + source = if prefs.proxyVideos and sp.m3u8Url.startsWith("http"): + getVidUrl(sp.m3u8Url) else: sp.m3u8Url + stateText = + if isLive: "LIVE" + elif sp.endTime.year > 1: "Ended " & sp.endTime.format("MMM d, YYYY") + elif sp.state.len > 0: sp.state + else: "Ended" + durationMs = + if sp.startTime.year > 1 and sp.endTime.year > 1: + int((sp.endTime - sp.startTime).inMilliseconds) + else: 0 + duration = if durationMs > 0: getDuration(durationMs) else: "" + totalListeners = + if sp.totalReplayWatched > 0: sp.totalReplayWatched + else: sp.totalLiveListeners + + buildHtml(tdiv(class="space-page")): + tdiv(class="space-panel"): + tdiv(class="space-player"): + if sp.m3u8Url.len > 0 and prefs.hlsPlayback: + audio(data-url=source, data-autoload="false") + verbatim "
" + tdiv(class="overlay-circle"): span(class="overlay-triangle") + if isLive: + tdiv(class="space-live"): text "LIVE" + elif duration.len > 0: + tdiv(class="overlay-duration"): text duration + verbatim "
" + elif sp.m3u8Url.len > 0: + tdiv(class="video-overlay"): + buttonReferer "/enablehls", "Enable hls playback", path + if isLive: + tdiv(class="space-live"): text "LIVE" + elif duration.len > 0: + tdiv(class="overlay-duration"): text duration + elif sp.availableForReplay: + tdiv(class="video-overlay"): + p: text "Audio stream unavailable" + else: + tdiv(class="video-overlay"): + p: text "Replay is not available" + + tdiv(class="space-info"): + tdiv(class="space-header"): + h2(class="space-title"): text sp.title + tdiv(class="space-meta"): + if totalListeners > 0: + span(class="listener-count"): text insertSep($totalListeners, ',') & " listeners" + if isLive: + span(class="space-live"): text stateText + else: + span(class="space-state"): text stateText + + if sp.admins.len > 0 or sp.speakers.len > 0: + tdiv(class="space-participants"): + for admin in sp.admins: + let role = if admin.username == sp.creator.username: "Host" + else: "Co-host" + renderParticipant(admin, role) + for speaker in sp.speakers: + renderParticipant(speaker, "") From 96a850c96336e25b518263bb8e155cbea44b4ece Mon Sep 17 00:00:00 2001 From: Zed Date: Fri, 19 Jun 2026 22:00:40 +0200 Subject: [PATCH 32/47] Add Space tests and fix media test selectors --- tests/test_space.py | 119 ++++++++++++++++++++++++++++++++++++++ tests/test_thread.py | 11 +--- tests/test_tweet_media.py | 12 ++-- 3 files changed, 128 insertions(+), 14 deletions(-) create mode 100644 tests/test_space.py diff --git a/tests/test_space.py b/tests/test_space.py new file mode 100644 index 0000000..d2ba672 --- /dev/null +++ b/tests/test_space.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: AGPL-3.0-only +"""Integration tests for Twitter Spaces support.""" +import pytest +from seleniumbase import BaseCase + + +class TestSpacePage(BaseCase): + """Tests for /i/spaces/@id route.""" + + SPACE_ID = "1mxPaaRAwYjKN" + SPACE_URL = f"http://localhost:8080/i/spaces/{SPACE_ID}" + + def test_space_page_loads(self): + """Space page should load with title.""" + self.open(self.SPACE_URL) + self.assert_element(".space-page") + self.assert_element(".space-panel") + self.assert_text_visible("INTEL WILL MOON NEXT WEEK", ".space-title") + + def test_space_host_info(self): + """Space should display host in participants.""" + self.open(self.SPACE_URL) + self.assert_element(".space-participants") + self.assert_text_visible("bubble boi") + self.assert_element(".host-badge") + + def test_space_metadata(self): + """Space should display listener count and state.""" + self.open(self.SPACE_URL) + self.assert_element(".space-meta") + # Should show listener count (number format) + meta_text = self.get_text(".space-meta") + assert any(c.isdigit() for c in meta_text), "Should show listener count" + # Should show ended state + assert "Ended" in meta_text or "Jun" in meta_text, "Should show ended state" + + def test_space_participants(self): + """Space should display host and speakers.""" + self.open(self.SPACE_URL) + self.assert_element(".space-participants") + # Host should have badge + self.assert_element(".host-badge") + self.assert_text_visible("Host", ".host-badge") + # Should show speakers + self.assert_text_visible("CANTELOPEPEEL") + self.assert_text_visible("Based Burner Account") + self.assert_text_visible("anon invests") + + def test_space_participant_avatars(self): + """Participant avatars should load correctly.""" + self.open(self.SPACE_URL) + # Check avatars in participants section + avatars = self.find_elements(".space-participant img") + assert len(avatars) >= 4, "Should have at least 4 participant avatars" + for avatar in avatars: + src = avatar.get_attribute("src") + # Should NOT be double-encoded + assert "%2Fpic%2F" not in src, f"Avatar URL double-encoded: {src}" + # Should have valid path + assert "/pic/" in src, f"Avatar URL missing /pic/: {src}" + + def test_space_player_hls_disabled(self): + """Without HLS, should show enable button with video-overlay style.""" + self.open(self.SPACE_URL) + self.assert_element(".space-player") + self.assert_element(".video-overlay") + # Should show duration in overlay-duration + self.assert_element(".overlay-duration") + # Should have enable button + source = self.get_page_source() + assert "Enable hls playback" in source + + def test_space_player_hls_enabled(self): + """With HLS enabled, should have audio element in DOM.""" + self.open(self.SPACE_URL) + # Set HLS preference via cookie + self.add_cookie({"name": "hlsPlayback", "value": "on"}) + self.refresh() + # Check page source for audio element (hidden until play clicked) + source = self.get_page_source() + assert '