From 35882ed88d422b1355b66a1ff8c1144bffdc7bdf Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 16 Jun 2026 14:37:23 +0200 Subject: [PATCH] Add Twitter Article rendering Fixes #646 --- src/api.nim | 25 ++- src/consts.nim | 25 ++- src/experimental/parser.nim | 4 +- src/experimental/parser/article.nim | 87 ++++++++ src/experimental/types/article.nim | 78 +++++++ src/nitter.nim | 4 +- src/parser.nim | 23 ++- src/parserutils.nim | 18 +- src/routes/article.nim | 48 +++++ src/sass/_article.scss | 272 ++++++++++++++++++++++++ src/sass/index.scss | 1 + src/types.nim | 44 ++++ src/views/article.nim | 246 ++++++++++++++++++++++ src/views/general.nim | 9 +- src/views/tweet.nim | 25 ++- tests/test_article.py | 307 ++++++++++++++++++++++++++++ 16 files changed, 1192 insertions(+), 24 deletions(-) create mode 100644 src/experimental/parser/article.nim create mode 100644 src/experimental/types/article.nim create mode 100644 src/routes/article.nim create mode 100644 src/sass/_article.scss create mode 100644 src/views/article.nim create mode 100644 tests/test_article.py diff --git a/src/api.nim b/src/api.nim index 404a01e..2702f7c 100644 --- a/src/api.nim +++ b/src/api.nim @@ -2,7 +2,7 @@ import asyncdispatch, httpclient, strutils, sequtils, sugar import packedjson import types, query, formatters, consts, apiutils, parser, utils -import experimental/parser as newParser +import experimental/parser # Helper to generate params object for GraphQL requests proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] = @@ -25,14 +25,10 @@ proc mediaUrl(id, cursor: string; count=20): ApiReq = ) proc userTweetsUrl(id: string; cursor: string): ApiReq = - return apiReq(graphUserTweetsV2, restIdVars % [id, cursor, "20"]) - # result = ApiReq( - # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles), - # oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"]) - # ) + return apiReq(graphUserTweetsV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles) proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq = - return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"], skipTid=true) + return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles, skipTid=true) proc tweetDetailUrl(id: string; cursor: string): ApiReq = return apiReq(graphTweet, tweetVars % [id, cursor]) @@ -228,6 +224,21 @@ proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} = let js = await fetch(mediaUrl(id, "", 30)) result = parseGraphPhotoRail(js) +proc getGraphArticle*(id: string): Future[Article] {.async.} = + if id.len == 0: return + let + url = apiReq(graphTweetResultByRestId, articleVars % id, articleFieldToggles) + json = await fetchRaw(url) + result = parseGraphArticle(json) + +proc getGraphTweetResults*(ids: seq[string]): Future[seq[Tweet]] {.async.} = + if ids.len == 0: return + let + idsJson = "[" & ids.mapIt("\"" & it & "\"").join(",") & "]" + url = apiReq(graphTweetResultsByRestIds, articleBatchVars % idsJson, articleFieldToggles) + js = await fetch(url) + result = parseGraphTweetResults(js) + proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} = let client = newAsyncHttpClient(maxRedirects=0) try: diff --git a/src/consts.nim b/src/consts.nim index beefa57..5ac3c32 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -10,7 +10,7 @@ const graphUser* = "IGgvgiOx4QZndDHuD3x9TQ/UserByScreenName" graphUserV2* = "-ZzAG_Bckx16LMbEvHC3lg/UserResultByScreenNameQuery" graphUserById* = "-DAaa9jPxPswYeI2fZ9rug/UserResultByIdQuery" - graphUserTweetsV2* = "PHTSTXqZYuHIeK4B1HQprQ/UserWithProfileTweetsQueryV2" + graphUserTweetsV2* = "LE3eTyeqhBh2g-fX85O2eQ/UserWithProfileTweetsQueryV2" graphUserTweetsAndRepliesV2* = "AcYHjc_YAx-9_rKWdMsKvA/UserWithProfileTweetsAndRepliesQueryV2" graphUserTweets* = "PNd0vlufvrcIwrAnBYKE9g/UserTweets" graphUserTweetsAndReplies* = "EqtpEwt0CoQXmDfq5DKH0A/UserTweetsAndReplies" @@ -28,6 +28,9 @@ const graphListTweets* = "0QJtcuMzVywHGAWD6Dtjlw/ListTimeline" graphAboutAccount* = "zUnx-DLN9dkwOkNhTLySjg/AboutAccountQuery" + graphTweetResultByRestId* = "qtXMy1p5Y62uCskc_NUPJw/TweetResultByRestId" + graphTweetResultsByRestIds* = "Sc9EUQTZNEH-wzegn-nHvQ/TweetResultsByRestIds" + graphBroadcast* = "FJLCzpXCLPM1jUZqmM7oEA/BroadcastQuery" restLiveStream* = "1.1/live_video_stream/status/" @@ -131,6 +134,24 @@ const "withVoice": true }""".replace(" ", "").replace("\n", "") + articleVars* = """{ + "tweetId": "$1", + "includePromotedContent": false, + "withBirdwatchNotes": true, + "withVoice": true, + "withCommunity": true +}""".replace(" ", "").replace("\n", "") + + articleBatchVars* = """{ + "tweetIds": $1, + "includePromotedContent": false, + "withBirdwatchNotes": true, + "withVoice": true, + "withCommunity": true +}""".replace(" ", "").replace("\n", "") + + articleFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withArticleSummaryText":true,"withArticleVoiceOver":true}""" + userFieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}""" - userTweetsFieldToggles* = """{"withArticlePlainText":false}""" + userTweetsFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false}""" tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}""" diff --git a/src/experimental/parser.nim b/src/experimental/parser.nim index 40986f5..e22a51f 100644 --- a/src/experimental/parser.nim +++ b/src/experimental/parser.nim @@ -1,2 +1,2 @@ -import parser/[user, graphql] -export user, graphql +import parser/[user, graphql, article] +export user, graphql, article diff --git a/src/experimental/parser/article.nim b/src/experimental/parser/article.nim new file mode 100644 index 0000000..2ae7f4a --- /dev/null +++ b/src/experimental/parser/article.nim @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import std/[strutils, tables, times, options] +import jsony +import utils, graphql, ../types/article +from ../../types import Article, ArticleParagraph, ArticleEntity, ArticleMedia, + User, TweetStats + +proc parseGraphArticle*(json: string): Article = + if json.len == 0 or json[0] != '{': + return + + var raw: GraphArticle + try: + raw = json.fromJson(GraphArticle) + except CatchableError: + return + + let + tweet = raw.data.tweetResult.result + article = tweet.article.articleResults.result + + if article.title.len == 0: + return + + let publishedAt = article.metadata.firstPublishedAtSecs + var articleTime: DateTime + if publishedAt > 0: + articleTime = publishedAt.int64.fromUnix.utc + elif tweet.legacy.createdAt.len > 0: + articleTime = parseTwitterDate(tweet.legacy.createdAt) + + result = Article( + title: article.title, + coverImage: getImageUrl(article.coverMedia.mediaInfo.originalImgUrl), + time: articleTime, + user: parseUserResult(tweet.core.userResults.result), + ) + + result.stats = TweetStats( + replies: tweet.legacy.replyCount, + retweets: tweet.legacy.retweetCount, + likes: tweet.legacy.favoriteCount, + ) + if tweet.views.count.len > 0: + try: result.stats.views = parseInt(tweet.views.count) + except ValueError: discard + + for blk in article.contentState.blocks: + result.paragraphs.add ArticleParagraph( + text: blk.text, + kind: blk.blockKind, + inlineStyles: blk.inlineStyleRanges, + entityRanges: blk.entityRanges, + ) + + for entry in article.contentState.entityMap: + let key = try: parseInt(entry.key) except ValueError: continue + var entity = ArticleEntity(kind: entry.value.entityKind) + case entity.kind + of "LINK": entity.url = entry.value.data.url + of "MEDIA": + for mi in entry.value.data.mediaItems: + entity.mediaIds.add mi.mediaId + of "TWEET": entity.tweetId = entry.value.data.tweetId + of "MARKDOWN": entity.markdown = entry.value.data.markdown + else: discard + result.entities[key] = entity + + for me in article.mediaEntities: + let typeName = me.mediaInfo.typeName + var media = ArticleMedia(kind: typeName) + if me.mediaInfo.videoInfo.isSome: + let variants = me.mediaInfo.videoInfo.get.variants + case typeName + of "ApiGif": + if variants.len > 0: + media.url = variants[0].url + of "ApiVideo": + var bestBitrate = -1 + for v in variants: + if v.bitrate > bestBitrate: + bestBitrate = v.bitrate + media.url = v.url + else: discard + elif typeName == "ApiImage": + media.url = getImageUrl(me.mediaInfo.originalImgUrl) + result.media[me.mediaId] = media diff --git a/src/experimental/types/article.nim b/src/experimental/types/article.nim new file mode 100644 index 0000000..024c69f --- /dev/null +++ b/src/experimental/types/article.nim @@ -0,0 +1,78 @@ +import std/options +import graphuser +from ../../types import ArticleStyle, ArticleEntityRange + +type + GraphArticle* = object + data*: tuple[tweetResult: tuple[result: TweetResultNode]] + + TweetResultNode* = object + article*: tuple[articleResults: tuple[result: ArticleResultNode]] + legacy*: TweetLegacy + core*: tuple[userResults: UserData] + views*: tuple[count: string] + + TweetLegacy* = object + createdAt*: string + replyCount*: int + retweetCount*: int + favoriteCount*: int + + ArticleResultNode* = object + title*: string + coverMedia*: tuple[mediaInfo: MediaInfoNode] + contentState*: ContentState + metadata*: tuple[firstPublishedAtSecs: int] + mediaEntities*: seq[RawMediaEntity] + + ContentState* = object + blocks*: seq[ContentBlock] + entityMap*: seq[EntityMapEntry] + + ContentBlock* = object + text*: string + blockKind*: string + inlineStyleRanges*: seq[ArticleStyle] + entityRanges*: seq[ArticleEntityRange] + + EntityMapEntry* = object + key*: string + value*: EntityMapValue + + EntityMapValue* = object + entityKind*: string + data*: EntityDataNode + + EntityDataNode* = object + url*: string + mediaItems*: seq[tuple[mediaId: string]] + tweetId*: string + markdown*: string + + RawMediaEntity* = object + mediaId*: string + mediaInfo*: MediaInfoNode + + MediaInfoNode* = object + typeName*: string + originalImgUrl*: string + videoInfo*: Option[VideoInfoNode] + + VideoInfoNode* = object + variants*: seq[VideoVariant] + + VideoVariant* = object + url*: string + bitrate*: int + +proc renameHook*(v: var ContentBlock; fieldName: var string) = + if fieldName == "type": + fieldName = "blockKind" + +proc renameHook*(v: var EntityMapValue; fieldName: var string) = + if fieldName == "type": + fieldName = "entityKind" + +proc renameHook*(v: var MediaInfoNode; fieldName: var string) = + if fieldName == "__typename": + fieldName = "typeName" diff --git a/src/nitter.nim b/src/nitter.nim index 685b608..78cace4 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -10,7 +10,7 @@ import types, config, prefs, formatters, redis_cache, http_pool, auth, apiutils import views/[general, about] import routes/[ preferences, timeline, status, media, search, rss, list, debug, - unsupported, embed, resolver, broadcast, router_utils] + unsupported, embed, resolver, broadcast, article, router_utils] const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances" const issuesUrl = "https://github.com/zedeus/nitter/issues" @@ -48,6 +48,7 @@ waitFor initRedisPool(cfg) stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n" stdout.flushFile +createArticleRouter(cfg) createUnsupportedRouter(cfg) createResolverRouter(cfg) createPrefRouter(cfg) @@ -118,6 +119,7 @@ routes: resp Http429, showError( &"Instance has no auth tokens, or is fully rate limited.
Use {link} or try again later.", cfg) + extend articleRoute, "" extend rss, "" extend status, "" extend search, "" diff --git a/src/parser.nim b/src/parser.nim index 78d40f7..d7b3f8d 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -542,7 +542,8 @@ proc parseGraphTweet*(js: JsonNode): Tweet = if result.attribution.isNone: parseLegacyMediaEntities(js{"legacy"}, result) - result.expandTweetEntitiesV2(js) + let hasArticle = js{"article", "article_results", "result", "title"}.getStr.len > 0 + result.expandTweetEntitiesV2(js, hasArticle) # Strip video source URL from text (for videos from other tweets) with mediaEntities, js{"media_entities"}: @@ -558,6 +559,16 @@ proc parseGraphTweet*(js: JsonNode): Tweet = result = parseTweet(js{"legacy"}, jsCard, replyId) result.id = js{"rest_id"}.getId + with artNode, js{"article", "article_results", "result"}: + let artTitle = artNode{"title"}.getStr + if artTitle.len > 0: + result.articlePreview = some ArticlePreview( + title: artTitle, + previewText: artNode{"preview_text"}.getStr, + coverImage: artNode{"cover_media_results", "result", "media_info", "original_img_url"}.getImageStr, + tweetId: result.id + ) + result.user = parseGraphUser(js{"core"}) if result.reply.len == 0: @@ -627,6 +638,16 @@ proc parseGraphTweetResult*(js: JsonNode): Tweet = with tweet, js{"data", "tweet_result", "result"}: result = parseGraphTweet(tweet) +proc parseGraphTweetResults*(js: JsonNode): seq[Tweet] = + let results = js{"data", "tweetResult"} + if results.kind != JArray: return + for item in results: + let tweet = item{"result"} + if tweet.isNull: continue + let t = parseGraphTweet(tweet) + if t != nil: + result.add t + proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = result = Conversation(replies: Result[Chain](beginning: true)) diff --git a/src/parserutils.nim b/src/parserutils.nim index 860c9d2..07aa088 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -185,12 +185,16 @@ proc extractSlice(js: JsonNode): Slice[int] = result = js["indices"][0].getInt ..< js["indices"][1].getInt proc extractUrls(result: var seq[ReplaceSlice]; js: JsonNode; - textLen: int; hideTwitter = false) = + textLen: int; hideTwitter = false; + hideArticle = false) = let url = js.getExpandedUrl slice = js.extractSlice - if hideTwitter and slice.b.succ >= textLen and url.isTwitterUrl: + if hideArticle and url.isTwitterUrl and "/article/" in url: + if slice.a < textLen: + result.add ReplaceSlice(kind: rkRemove, slice: slice) + elif hideTwitter and slice.b.succ >= textLen and url.isTwitterUrl: if slice.a < textLen: result.add ReplaceSlice(kind: rkRemove, slice: slice) else: @@ -343,7 +347,7 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = hasQuote or hasJobCard) proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: Slice[int]; - hasRedundantLink=false) = + hasRedundantLink=false; hasArticle=false) = let hasCard = tweet.card.isSome var replacements = newSeq[ReplaceSlice]() @@ -354,7 +358,8 @@ proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: S if urlStr.len == 0 or urlStr notin text: continue - replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink) + replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink, + hideArticle = hasArticle) if hasCard and u{"url"}.getStr == get(tweet.card).url: get(tweet.card).url = u.getExpandedUrl @@ -385,7 +390,7 @@ proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: S tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false) -proc expandTweetEntitiesV2*(tweet: Tweet; js: JsonNode) = +proc expandTweetEntitiesV2*(tweet: Tweet; js: JsonNode; hasArticle=false) = let textRange = js{"details", "display_text_range"} textSlice = textRange{0}.getInt .. textRange{1}.getInt @@ -394,7 +399,8 @@ proc expandTweetEntitiesV2*(tweet: Tweet; js: JsonNode) = hasAttribution = tweet.attribution.isSome tweet.expandTextEntitiesV2(js, tweet.text, textSlice, - hasQuote or hasJobCard or hasAttribution) + hasQuote or hasJobCard or hasAttribution, + hasArticle) proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) = let diff --git a/src/routes/article.nim b/src/routes/article.nim new file mode 100644 index 0000000..0a7d1dc --- /dev/null +++ b/src/routes/article.nim @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import asyncdispatch, tables, strutils +import jester, karax/vdom +import ".."/[types, api] +import ../views/[article, general] +import router_utils + +export api, article, vdom, general, router_utils + +proc createArticleRouter*(cfg: Config) = + router articleRoute: + get "/i/article/@id": + cond @"id".allCharsInSet(Digits) + + let article = await getGraphArticle(@"id") + if article == nil: + resp Http404, showError("Article not found", cfg) + + var tweetIds: seq[string] + for e in article.entities.values: + if e.kind == "TWEET": + tweetIds.add e.tweetId + + var tweets = initTable[int64, Tweet]() + if tweetIds.len > 0: + try: + for t in await getGraphTweetResults(tweetIds): + tweets[t.id] = t + except CatchableError: + discard + + let + prefs = requestPrefs() + path = getPath() + html = renderArticle(article, tweets, path, prefs, @"id") + twitterUrl = "https://x.com/" & article.user.username & "/article/" & @"id" + resp renderMain(html, request, cfg, prefs, titleText=article.title, + twitterLink=twitterUrl) + + get "/@name/article/@id/?": + cond '.' notin @"name" + cond @"id".allCharsInSet(Digits) + redirect("/i/article/" & @"id") + + get "/@name/status/@id/article": + cond '.' notin @"name" + cond @"id".allCharsInSet(Digits) + redirect("/i/article/" & @"id") diff --git a/src/sass/_article.scss b/src/sass/_article.scss new file mode 100644 index 0000000..9cc728a --- /dev/null +++ b/src/sass/_article.scss @@ -0,0 +1,272 @@ +.article-page { + max-width: 700px; + margin: 0 auto 20px; + background-color: var(--bg_panel); + + > .top-ref { + padding-top: 20px; + } + + .article-cover { + width: 100%; + display: block; + } + + .article-body { + padding: 20px; + + > :last-child { + margin-bottom: 0; + } + + .article-title { + display: block; + font-size: 2rem; + line-height: 1.3; + margin: 0 0 10px; + color: var(--fg_color); + } + + .article-author { + margin-bottom: 12px; + padding-bottom: 10px; + border-bottom: 1px solid var(--border_grey); + font-size: 14px; + + .article-author-row { + display: flex; + align-items: center; + gap: 8px; + } + + .article-avatar { + display: flex; + } + + .avatar { + width: 40px; + height: 40px; + } + + .article-author-name { + display: flex; + align-items: center; + margin-bottom: 2px; + + .fullname { + font-size: 15px; + } + + .verified-icon { + margin-left: 2px; + } + } + + .article-author-meta { + display: flex; + align-items: center; + } + + .fullname { + font-weight: 700; + color: var(--fg_color); + max-width: unset; + text-overflow: unset; + overflow: visible; + white-space: normal; + } + + .username, + .article-date-sep, + .article-date { + color: var(--fg_dark); + } + + .username { + margin-left: 0; + } + + .article-date-sep { + margin: 0 4px; + } + + .article-date { + text-decoration: none; + + &:hover { + text-decoration: underline; + } + } + + .tweet-stats { + margin-top: 6px; + + .tweet-stat { + padding-top: 0; + } + } + } + + > h1 { + display: block; + font-size: 1.8rem; + margin: 25px 0 15px; + } + + > h2 { + font-size: 1.4rem; + font-weight: bold; + margin: 20px 0 12px; + } + + > h3 { + font-size: 1.2rem; + font-weight: bold; + margin: 18px 0 10px; + } + + > p { + font-size: 16px; + line-height: 1.7; + margin: 16px 0; + word-wrap: break-word; + } + + .blockquote-attribution { + display: block; + margin-top: 0.5em; + } + + > blockquote { + border-left: 3px solid var(--accent); + padding-left: 16px; + margin: 16px 0; + color: var(--fg_faded); + font-size: 16px; + line-height: 1.7; + } + + > pre { + background-color: var(--bg_elements); + padding: 12px 16px; + border-radius: 6px; + overflow-x: auto; + margin: 16px 0; + + code { + font-family: monospace; + font-size: 14px; + color: var(--fg_color); + } + } + + code { + background-color: var(--bg_elements); + padding: 2px 5px; + border-radius: 3px; + font-family: monospace; + font-size: 0.9em; + } + + > ul, + > ol { + margin: 16px 0; + padding-left: 2em; + + li { + font-size: 16px; + line-height: 1.7; + margin: 6px 0; + } + } + + .article-media { + text-align: center; + margin: 20px 0; + + img, + video { + max-width: 100%; + border-radius: 12px; + } + } + + > a, + > p a, + > h1 a, + > h2 a, + > h3 a, + > blockquote a, + > ul a, + > ol a { + color: var(--accent); + text-decoration: none; + + &:hover { + text-decoration: underline; + } + } + + .article-divider { + border: none; + border-top: 1px solid var(--border_grey); + margin: 30px 0; + } + + .timeline-item { + margin: 20px 0; + border: 1px solid var(--border_grey); + border-radius: 12px; + overflow: hidden; + } + } +} + +.conversation .article-page { + max-width: 100%; + margin-bottom: 0; +} + +.article-card { + .card-image-container { + position: relative; + } + + .card-image img { + height: auto; + } + + .article-card-badge { + position: absolute; + bottom: 8px; + left: 8px; + background: rgba(0, 0, 0, 0.75); + color: #fff; + font-size: 13px; + font-weight: 700; + padding: 2px 8px; + border-radius: 4px; + } +} + +.quote .article-card { + margin: 0; + + .card-container { + border: none; + border-radius: 0; + border-top: solid 1px var(--dark_grey); + } +} + +@media (max-width: 700px) { + .article-page { + .article-body { + padding: 12px 15px 25px; + + .article-title { + font-size: 1.6rem; + } + } + } +} diff --git a/src/sass/index.scss b/src/sass/index.scss index e60b9c4..8126fe9 100644 --- a/src/sass/index.scss +++ b/src/sass/index.scss @@ -8,6 +8,7 @@ @import "timeline"; @import "search"; @import "broadcast"; +@import "_article"; body { // colors diff --git a/src/types.nim b/src/types.nim index 3604efb..05b7fb3 100644 --- a/src/types.nim +++ b/src/types.nim @@ -198,6 +198,43 @@ type PhotoRail* = seq[GalleryPhoto] + Article* = ref object + title*: string + coverImage*: string + user*: User + time*: DateTime + stats*: TweetStats + paragraphs*: seq[ArticleParagraph] + entities*: Table[int, ArticleEntity] + media*: Table[string, ArticleMedia] + + ArticleParagraph* = object + text*: string + kind*: string + inlineStyles*: seq[ArticleStyle] + entityRanges*: seq[ArticleEntityRange] + + ArticleStyle* = object + offset*: int + length*: int + style*: string + + ArticleEntityRange* = object + offset*: int + length*: int + key*: int + + ArticleEntity* = object + kind*: string + url*: string + mediaIds*: seq[string] + tweetId*: string + markdown*: string + + ArticleMedia* = object + kind*: string + url*: string + Poll* = object options*: seq[string] values*: seq[int] @@ -247,6 +284,12 @@ type likes*: int views*: int + ArticlePreview* = object + title*: string + previewText*: string + coverImage*: string + tweetId*: int64 + Tweet* = ref object id*: int64 threadId*: int64 @@ -275,6 +318,7 @@ type note*: string isAd*: bool isAI*: bool + articlePreview*: Option[ArticlePreview] Tweets* = seq[Tweet] diff --git a/src/views/article.nim b/src/views/article.nim new file mode 100644 index 0000000..1ba9c8b --- /dev/null +++ b/src/views/article.nim @@ -0,0 +1,246 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import strutils, strformat, tables, unicode, bitops, uri +import karax/[karaxdsl, vdom] + +import renderutils, tweet, timeline +import ".."/[types, utils, formatters] + +proc renderAtomicParagraph(paragraph: ArticleParagraph; article: Article; + tweets: Table[int64, Tweet]; path: string; + prefs: Prefs): VNode = + if paragraph.entityRanges.len == 0: + return text "" + + let er = paragraph.entityRanges[0] + if er.key notin article.entities: + return text "" + + let entity = article.entities[er.key] + + case entity.kind + of "MEDIA": + buildHtml(tdiv(class="article-media")): + for id in entity.mediaIds: + let media = article.media.getOrDefault(id) + if media.url.len == 0: + continue + case media.kind + of "ApiGif": + video(src=getVidUrl(media.url), controls="", autoplay="", loop="", + muted="") + of "ApiVideo": + video(src=getVidUrl(media.url), controls="") + else: + a(href=getOrigPicUrl(media.url), target="_blank"): + img(src=getSmallPic(media.url), alt="", loading="lazy") + of "TWEET": + let tweet = tweets.getOrDefault( + try: parseBiggestInt(entity.tweetId) + except ValueError: 0, nil) + if tweet != nil: + renderTweet(tweet, prefs, path) + else: + text "" + of "MARKDOWN": + var content = entity.markdown + if content.startsWith("```"): + let firstNl = content.find('\n') + if firstNl >= 0: content = content[firstNl + 1 .. ^1] + if content.endsWith("```"): content = content[0 .. ^4] + content = content.strip + buildHtml(pre()): + code(): text content + of "DIVIDER": + buildHtml(hr(class="article-divider")) + else: + text "" + +proc wrapStyle(node: VNode; style: int): VNode = + result = node + if style.testBit(4): result = buildHtml(code()): result + if style.testBit(0): result = buildHtml(strong()): result + if style.testBit(1): result = buildHtml(em()): result + if style.testBit(2): result = buildHtml(del()): result + if style.testBit(3): result = buildHtml(underlined()): result + +proc addContent(target: VNode; content: string; style = 0) = + var first = true + for line in content.split('\n'): + if not first: + target.add VNode(kind: VNodeKind.br) + first = false + var pos = 0 + while pos < line.len: + let atPos = line.find('@', pos) + if atPos == -1: + target.add wrapStyle(text line[pos .. ^1], style) + break + if atPos > 0 and line[atPos - 1] in Letters + Digits + {'_'}: + target.add wrapStyle(text line[pos .. atPos], style) + pos = atPos + 1 + continue + var j = atPos + 1 + while j < line.len and j - atPos - 1 < 15 and + line[j] in Letters + Digits + {'_'}: + inc j + if j == atPos + 1: + target.add wrapStyle(text line[pos .. atPos], style) + pos = atPos + 1 + continue + if atPos > pos: + target.add wrapStyle(text line[pos ..< atPos], style) + let username = line[atPos + 1 ..< j] + let link = a.newVNode() + link.setAttr("href", "/" & username) + link.add wrapStyle(text ("@" & username), style) + target.add link + pos = j + +proc applyInlineStyles(target: VNode; runes: seq[Rune]; start, length: int; + styles: seq[ArticleStyle]) = + if styles.len == 0: + target.addContent($runes[start ..< start + length]) + return + + var + lastStyle = 0 + lastStart = start + let endPos = start + length + + for i in start ..< endPos: + var style = 0 + for sr in styles: + let + sStart = sr.offset + sEnd = sStart + sr.length + if sStart <= i and sEnd > i: + case sr.style + of "Bold": style.setBit(0) + of "Italic": style.setBit(1) + of "Strikethrough": style.setBit(2) + of "Underline": style.setBit(3) + of "Code": style.setBit(4) + else: discard + + if style != lastStyle: + if i > lastStart: + addContent(target, $runes[lastStart ..< i], lastStyle) + lastStyle = style + lastStart = i + + if lastStart < endPos: + addContent(target, $runes[lastStart ..< endPos], lastStyle) + +proc renderTextParagraph(paragraph: ArticleParagraph; article: Article): VNode = + let text = paragraph.text + + result = case paragraph.kind + of "header-one": h1.newVNode() + of "header-two": h2.newVNode() + of "header-three": h3.newVNode() + of "ordered-list-item", "unordered-list-item": li.newVNode() + of "blockquote": VNode(kind: VNodeKind.blockquote) + of "code-block": + let pre = pre.newVNode() + let code = code.newVNode() + code.add text text + pre.add code + return pre + else: p.newVNode() + + let + runes = text.toRunes + textLen = runes.len + var last = 0 + for er in paragraph.entityRanges: + if er.offset > last: + applyInlineStyles(result, runes, last, er.offset - last, + paragraph.inlineStyles) + + last = er.offset + er.length + + var target = result + if er.key in article.entities: + let entity = article.entities[er.key] + if entity.kind == "LINK": + let parsed = parseUri(entity.url) + if parsed.scheme in ["http", "https"]: + target = a.newVNode() + if parsed.isTwitterUrl: + target.setAttr("href", parsed.path) + else: + target.setAttr("href", entity.url) + + applyInlineStyles(target, runes, er.offset, er.length, + paragraph.inlineStyles) + if target != result: + result.add target + + if last < textLen: + applyInlineStyles(result, runes, last, textLen - last, + paragraph.inlineStyles) + + if paragraph.kind == "blockquote" and result.len > 0: + let lastChild = result[result.len - 1] + if lastChild.kind == VNodeKind.strong and lastChild.len > 0 and + lastChild[0].kind == VNodeKind.text: + lastChild.setAttr("class", "blockquote-attribution") + +proc renderArticle*(article: Article; tweets: Table[int64, Tweet]; + path: string; prefs: Prefs; tweetId=""): VNode = + let author = article.user + + let main = buildHtml(article(class="article-body")): + h1(class="article-title"): text article.title + + tdiv(class="article-author"): + tdiv(class="article-author-row"): + a(class="article-avatar", href=("/" & author.username)): + genImg(author.getUserPic("_bigger"), class=prefs.getAvatarClass) + tdiv(class="article-author-info"): + tdiv(class="article-author-name"): + linkUser(author, class="fullname") + verifiedIcon(author) + tdiv(class="article-author-meta"): + linkUser(author, class="username") + span(class="article-date-sep"): text " · " + a(class="article-date", + href=("/" & author.username & "/status/" & tweetId)): + text article.time.getShortTime + if not prefs.hideTweetStats: + renderStats(article.stats) + + var listKind = "" + var list: VNode = nil + + for paragraph in article.paragraphs: + let isListItem = paragraph.kind in [ + "ordered-list-item", "unordered-list-item"] + + if not isListItem and list != nil: + main.add list + list = nil + listKind = "" + + if paragraph.kind == "atomic": + main.add renderAtomicParagraph(paragraph, article, tweets, path, prefs) + elif isListItem: + if paragraph.kind != listKind: + if list != nil: + main.add list + list = if paragraph.kind == "ordered-list-item": ol.newVNode() + else: ul.newVNode() + listKind = paragraph.kind + list.add renderTextParagraph(paragraph, article) + else: + main.add renderTextParagraph(paragraph, article) + + if list != nil: + main.add list + + buildHtml(tdiv(class="article-page")): + if article.coverImage.len > 0: + a(href=getOrigPicUrl(article.coverImage), target="_blank"): + img(class="article-cover", src=getSmallPic(article.coverImage), alt="") + main + renderToTop() diff --git a/src/views/general.nim b/src/views/general.nim index f753610..13ac10d 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -50,7 +50,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; let opensearchUrl = getUrlPrefix(cfg) & "/opensearch" buildHtml(head): - link(rel="stylesheet", type="text/css", href="/css/style.css?v=35") + link(rel="stylesheet", type="text/css", href="/css/style.css?v=38") link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=5") if theme.len > 0: @@ -122,9 +122,12 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; proc renderMain*(body: VNode; req: Request; cfg: Config; prefs=defaultPrefs; titleText=""; desc=""; ogTitle=""; rss=""; video=""; - images: seq[string] = @[]; banner=""): string = + images: seq[string] = @[]; banner=""; + twitterLink=""): string = - let twitterLink = getTwitterLink(req.path, req.params) + let twitterLink = + if twitterLink.len > 0: twitterLink + else: getTwitterLink(req.path, req.params) let node = buildHtml(html(lang="en")): renderHead(prefs, cfg, req, titleText, desc, video, images, banner, ogTitle, diff --git a/src/views/tweet.nim b/src/views/tweet.nim index 5546eb4..9bf6c9b 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -9,9 +9,24 @@ import general const doctype = "\n" -proc renderMiniAvatar(user: User; prefs: Prefs): VNode = +proc renderMiniAvatar*(user: User; prefs: Prefs): VNode = genImg(user.getUserPic("_mini"), class=(prefs.getAvatarClass & " mini")) +proc renderArticleCard(preview: ArticlePreview; prefs: Prefs): VNode = + let url = "/i/article/" & $preview.tweetId + buildHtml(tdiv(class="article-card card large")): + a(class="card-container", href=url): + if preview.coverImage.len > 0: + tdiv(class="card-image-container"): + tdiv(class="card-image"): + genImg(preview.coverImage) + span(class="article-card-badge"): text "Article" + tdiv(class="card-content-container"): + tdiv(class="card-content"): + h2(class="card-title"): text preview.title + if preview.previewText.len > 0: + p(class="card-description"): text preview.previewText + proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs): VNode = buildHtml(tdiv): if pinned: @@ -225,7 +240,7 @@ func formatStat(stat: int): string = if stat > 0: insertSep($stat, ',') else: "" -proc renderStats(stats: TweetStats): VNode = +proc renderStats*(stats: TweetStats): VNode = buildHtml(tdiv(class="tweet-stats")): span(class="tweet-stat"): icon "comment", formatStat(stats.replies) span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets) @@ -308,6 +323,9 @@ proc renderQuote(quote: Tweet; prefs: Prefs; path: string): VNode = if quote.media.len > 0: renderQuoteMedia(quote, prefs, path) + if quote.articlePreview.isSome: + renderArticleCard(quote.articlePreview.get(), prefs) + if quote.note.len > 0 and not prefs.hideCommunityNotes: renderCommunityNote(quote.note, prefs) @@ -392,6 +410,9 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; if tweet.card.isSome and tweet.card.get().kind != hidden: renderCard(tweet.card.get(), prefs, path) + if tweet.articlePreview.isSome: + renderArticleCard(tweet.articlePreview.get(), prefs) + if tweet.media.len > 0: renderMedia(tweet.media, prefs, path, bigThumb) diff --git a/tests/test_article.py b/tests/test_article.py new file mode 100644 index 0000000..15a87c7 --- /dev/null +++ b/tests/test_article.py @@ -0,0 +1,307 @@ +from base import BaseTestCase +from parameterized import parameterized + + +class ArticleSelectors: + page = '.article-page' + cover = '.article-cover' + body = '.article-body' + title = '.article-title' + author = '.article-author' + fullname = '.article-author .fullname' + username = '.article-author .username' + date = '.article-author .article-date' + avatar = '.article-author img.avatar' + verified = '.article-author .verified-icon' + media = '.article-media' + divider = '.article-divider' + + +articles = [ + ['2064166507438059759', + '1s，秒杀一切，开源一个 X 文章发布 Skill【重磅升级】', + 'punk2898', 'Punk'], + + ['2064689664213041529', + 'SpaceX Thesis & Valuation Memorandum', + 'Dialectic_Group', 'Dialectic'], + + ['2064691088636424322', + 'Consciousness and AI: The Problem of Inner Experience', + 'CosmicOrFun', 'Cosmic Orphan'], + + ['2064696491948777658', + 'NC Push for Data Centers + Stablecoin Crypto= Data Centers are defacto BAILOUT OF Fed Reserve System', + 'June_12_1776', 'June_12_1776'], + + ['2064755789391110154', + 'DeFi Markets Update 2026-06-10', + 'SteakhouseFi', 'Steakhouse Financial'], + + ['2064755231901319527', + 'The machine economy has a killswitch and somebody just pulled it.', + '1914ad', 'Justin Bechler HMP-028'], + + ['2062858677149675788', + 'Yakshinis', + 'CosmicOrFun', 'Cosmic Orphan'], +] + +articles_with_media = [ + ['2064166507438059759', 6], + ['2064689664213041529', 11], + ['2064755789391110154', 5], +] + +articles_with_dividers = [ + ['2064166507438059759', 1], + ['2064689664213041529', 6], +] + + +class ArticleBasicTest(BaseTestCase): + @parameterized.expand(articles) + def test_article_loads(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + self.assert_element_visible(ArticleSelectors.page) + self.assert_element_visible(ArticleSelectors.body) + self.assert_text(title, ArticleSelectors.title) + + @parameterized.expand(articles) + def test_article_author(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + self.assert_element_visible(ArticleSelectors.author) + self.assert_text(f'@{username}', ArticleSelectors.username) + + @parameterized.expand(articles) + def test_article_has_cover(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + self.assert_element_visible(ArticleSelectors.cover) + src = self.get_attribute(ArticleSelectors.cover, 'src') + self.assertIn('/pic/', src) + + @parameterized.expand(articles) + def test_article_has_date(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + date_text = self.get_text(ArticleSelectors.date) + self.assertTrue(len(date_text) > 3) + + @parameterized.expand(articles) + def test_article_author_avatar(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + self.assert_element_visible(ArticleSelectors.avatar) + src = self.get_attribute(ArticleSelectors.avatar, 'src') + self.assertIn('/pic/', src) + self.assertGreater(len(src), len('/pic/')) + + @parameterized.expand(articles) + def test_article_author_verified(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + self.assert_element_visible(ArticleSelectors.verified) + + def test_article_author_verified_business(self): + self.open_nitter('i/article/2064755789391110154') + self.assert_element_visible('.article-author .verified-icon.business') + + +class ArticleContentTest(BaseTestCase): + def test_article_has_paragraphs(self): + self.open_nitter('i/article/2064689664213041529') + paragraphs = self.find_elements('.article-body p') + self.assertGreater(len(paragraphs), 10) + + def test_article_has_headers(self): + self.open_nitter('i/article/2064689664213041529') + headers = self.find_elements('.article-body h1, .article-body h2') + self.assertGreater(len(headers), 5) + + def test_article_has_bold_text(self): + self.open_nitter('i/article/2064166507438059759') + bold = self.find_elements('.article-body strong') + self.assertGreater(len(bold), 0) + + def test_article_has_italic_text(self): + self.open_nitter('i/article/2064166507438059759') + italic = self.find_elements('.article-body em') + self.assertGreater(len(italic), 0) + + def test_article_has_blockquotes(self): + self.open_nitter('i/article/2064696491948777658') + self.assert_element_visible('.article-body blockquote') + + def test_article_has_lists(self): + self.open_nitter('i/article/2064696491948777658') + self.assert_element_visible('.article-body ul') + + def test_article_has_emoji_text(self): + self.open_nitter('i/article/2064166507438059759') + body = self.get_text(ArticleSelectors.body) + self.assertTrue(any(ord(c) > 0x1F000 for c in body)) + + def test_article_has_links(self): + self.open_nitter('i/article/2064691088636424322') + links = self.find_elements('.article-body a[href]') + self.assertGreater(len(links), 0) + + def test_article_twitter_links_localized(self): + self.open_nitter('i/article/2064755789391110154') + links = self.find_elements('.article-body a[href^="https://x.com"]') + self.assertEqual(len(links), 0, 'x.com links should be converted to local paths') + + @parameterized.expand(articles_with_media) + def test_article_media_count(self, tweet_id, expected_count): + self.open_nitter(f'i/article/{tweet_id}') + media = self.find_elements(ArticleSelectors.media) + self.assertEqual(len(media), expected_count) + + @parameterized.expand(articles_with_dividers) + def test_article_divider_count(self, tweet_id, expected_count): + self.open_nitter(f'i/article/{tweet_id}') + dividers = self.find_elements(ArticleSelectors.divider) + self.assertEqual(len(dividers), expected_count) + + +class ArticleMediaTest(BaseTestCase): + def test_media_images_proxied(self): + self.open_nitter('i/article/2064689664213041529') + self.assert_element_visible(ArticleSelectors.media) + img = self.find_element(f'{ArticleSelectors.media} img') + src = img.get_attribute('src') + self.assertIn('/pic/', src) + self.assertFalse(src.startswith('https://pbs.twimg.com')) + + def test_cover_image_proxied(self): + self.open_nitter('i/article/2064689664213041529') + self.assert_element_visible(ArticleSelectors.cover) + src = self.get_attribute(ArticleSelectors.cover, 'src') + self.assertIn('/pic/', src) + self.assertFalse(src.startswith('https://pbs.twimg.com')) + + def test_embedded_tweet(self): + self.open_nitter('i/article/2064755789391110154') + self.assert_element_visible('.article-body .timeline-item') + + def test_multiple_embedded_tweets(self): + self.open_nitter('i/article/2064755231901319527') + tweets = self.find_elements('.article-body .timeline-item') + self.assertGreaterEqual(len(tweets), 3) + + +class ArticleMentionTest(BaseTestCase): + def test_mention_linkified(self): + self.open_nitter('i/article/2064755231901319527') + link = self.find_element('.article-body a[href="/ZachXBT"]') + self.assertEqual(link.text, '@ZachXBT') + + def test_multiple_mentions_linkified(self): + self.open_nitter('i/article/2064755231901319527') + links = self.find_elements('.article-body a[href^="/"]') + mention_hrefs = [l.get_attribute('href') for l in links + if l.text.startswith('@')] + usernames = [h.split('/')[-1] for h in mention_hrefs] + self.assertIn('ZachXBT', usernames) + self.assertIn('River', usernames) + + def test_mention_in_different_article(self): + self.open_nitter('i/article/2064689664213041529') + link = self.find_element('.article-body a[href="/FutureJurvetson"]') + self.assertEqual(link.text, '@FutureJurvetson') + + def test_no_spurious_whitespace_in_styled_paragraph(self): + """Styled paragraphs should not have extra whitespace from VNode serialization.""" + self.open_nitter('i/article/2064696491948777658') + source = self.get_page_source() + self.assertNotIn('white-space: pre-wrap', source) + self.assertNotIn('white-space:pre-wrap', source) + + +class ArticleCardTest(BaseTestCase): + @parameterized.expand(articles) + def test_status_page_shows_article_card(self, tweet_id, title, username, fullname): + self.open_nitter(f'{username}/status/{tweet_id}') + self.assert_element_visible('.article-card') + self.assert_text(title, '.article-card .card-title') + + def test_article_card_has_cover_image(self): + self.open_nitter('Dialectic_Group/status/2064689664213041529') + self.assert_element_visible('.article-card .card-image img') + src = self.get_attribute('.article-card .card-image img', 'src') + self.assertIn('/pic/', src) + + def test_article_card_has_badge(self): + self.open_nitter('Dialectic_Group/status/2064689664213041529') + self.assert_element_visible('.article-card-badge') + self.assert_text('Article', '.article-card-badge') + + def test_article_card_has_preview_text(self): + self.open_nitter('CosmicOrFun/status/2064691088636424322') + self.assert_element_visible('.article-card .card-description') + + def test_article_card_links_to_article(self): + self.open_nitter('punk2898/status/2064166507438059759') + href = self.get_attribute('.article-card .card-container', 'href') + self.assertIn('/article/', href) + + def test_article_url_stripped_from_tweet_text(self): + self.open_nitter('punk2898/status/2064166507438059759') + self.assert_element_visible('.article-card') + source = self.get_page_source() + # Main tweet text should not contain article URL + import re + main = re.search(r'id="m".*?tweet-content[^>]*>(.*?)', source, re.DOTALL) + self.assertIsNotNone(main) + self.assertNotIn('/article/', main.group(1)) + + +class ArticleQuotedCardTest(BaseTestCase): + """Article cards inside quoted tweets (1914ad quoting own article).""" + quoted_tweet = '1914ad/status/2064789532071891085' + quoted_article_id = '2063677483548102688' + + def test_quoted_card_visible(self): + self.open_nitter(self.quoted_tweet) + self.assert_element_visible('.quote .article-card') + + def test_quoted_card_has_title(self): + self.open_nitter(self.quoted_tweet) + self.assert_text('David Bailey Already Won', '.quote .article-card .card-title') + + def test_quoted_card_has_badge(self): + self.open_nitter(self.quoted_tweet) + self.assert_element_visible('.quote .article-card-badge') + self.assert_text('Article', '.quote .article-card-badge') + + def test_quoted_card_has_cover_image(self): + self.open_nitter(self.quoted_tweet) + self.assert_element_visible('.quote .article-card .card-image img') + src = self.get_attribute('.quote .article-card .card-image img', 'src') + self.assertIn('/pic/', src) + + def test_quoted_card_has_description(self): + self.open_nitter(self.quoted_tweet) + self.assert_element_visible('.quote .article-card .card-description') + + def test_quoted_card_links_to_article(self): + self.open_nitter(self.quoted_tweet) + href = self.get_attribute('.quote .article-card .card-container', 'href') + self.assertIn(f'/article/{self.quoted_article_id}', href) + + +class ArticleRoutingTest(BaseTestCase): + def test_username_article_route_redirects(self): + self.open_nitter('punk2898/article/2064166507438059759') + self.assert_element_visible(ArticleSelectors.page) + self.assert_text('1s', ArticleSelectors.title) + + def test_status_article_route_redirects(self): + self.open_nitter('punk2898/status/2064166507438059759/article') + self.assert_element_visible(ArticleSelectors.page) + self.assert_text('1s', ArticleSelectors.title) + + def test_invalid_id_returns_404(self): + self.open_nitter('i/article/notanumber') + self.assert_element_not_visible(ArticleSelectors.page) + + def test_nonexistent_article(self): + self.open_nitter('i/article/1') + self.assert_element_visible('.error-panel')