diff --git a/nitter.example.conf b/nitter.example.conf index 32fb618..4e040f8 100644 --- a/nitter.example.conf +++ b/nitter.example.conf @@ -26,6 +26,7 @@ enableRSS = true # master switch, set to false to disable all RSS fe enableRSSUserTweets = true # /@user/rss enableRSSUserReplies = true # /@user/with_replies/rss enableRSSUserMedia = true # /@user/media/rss +enableRSSUserArticles = true # /@user/articles/rss enableRSSSearch = true # /search/rss and /@user/search/rss enableRSSList = true # list RSS feeds enableDebug = false # enable request logs and debug endpoints (/.sessions) diff --git a/src/api.nim b/src/api.nim index 1133b9e..555a699 100644 --- a/src/api.nim +++ b/src/api.nim @@ -38,6 +38,12 @@ proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq = oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles, skipTid=true) ) +proc userArticlesUrl(id: string; cursor: string): ApiReq = + result = ApiReq( + cookie: apiUrl(graphUserArticles, userArticlesVars % [id, cursor], userTweetsFieldToggles), + oauth: apiUrl(graphUserArticlesV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles) + ) + proc tweetDetailUrl(id, cursor: string; mode = Relevance): ApiReq = return apiReq(graphTweet, tweetVars % [id, cursor, $mode]) # let cookieVars = tweetDetailVars % [id, cursor] @@ -112,6 +118,7 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi of TimelineKind.tweets: userTweetsUrl(id, cursor) of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor) of TimelineKind.media: mediaUrl(id, cursor) + of TimelineKind.articles: userArticlesUrl(id, cursor) js = await fetch(url) result = parseGraphTimeline(js, after) diff --git a/src/config.nim b/src/config.nim index 800224f..b46a979 100644 --- a/src/config.nim +++ b/src/config.nim @@ -42,6 +42,7 @@ proc getConfig*(path: string): (Config, parseCfg.Config) = enableRSSUserTweets: masterRss and cfg.get("Config", "enableRSSUserTweets", true), enableRSSUserReplies: masterRss and cfg.get("Config", "enableRSSUserReplies", true), enableRSSUserMedia: masterRss and cfg.get("Config", "enableRSSUserMedia", true), + enableRSSUserArticles: masterRss and cfg.get("Config", "enableRSSUserArticles", true), enableRSSSearch: masterRss and cfg.get("Config", "enableRSSSearch", true), enableRSSList: masterRss and cfg.get("Config", "enableRSSList", true), enableDebug: cfg.get("Config", "enableDebug", false), diff --git a/src/consts.nim b/src/consts.nim index 0e2c536..c88ae68 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -16,6 +16,8 @@ const graphUserTweetsAndReplies* = "qUpkZU6eN8MbtQb7rC_pYg/UserTweetsAndReplies" graphUserMedia* = "VyudDWQnr9vJNw7GasFz2g/UserMedia" graphUserMediaV2* = "WK111rbR0vM0ZX4lyZCYjw/MediaTimelineV2" + graphUserArticles* = "ZmMjUyrTpwYfTGAdylEyMw/UserArticlesTweets" + graphUserArticlesV2* = "PsGixN38UZz2RheyayNB5Q/UserProfileArticlesTimelineQuery" graphTweet* = "OZMbEnEa96AN8Pq6HyTWdw/ConversationTimeline" graphTweetDetail* = "XMOz5h24KAZ86qKffKTLdQ/TweetDetail" graphTweetResult* = "xYOrBQoTlfKJJPsX76MZEw/TweetResultByIdQuery" @@ -148,6 +150,13 @@ const "withVoice": true }""".replace(" ", "").replace("\n", "") + userArticlesVars* = """{ + "userId": "$1", $2 + "count": 20, + "includePromotedContent": false, + "withVoice": true +}""".replace(" ", "").replace("\n", "") + articleVars* = """{ "tweetId": "$1", "includePromotedContent": false, diff --git a/src/parser.nim b/src/parser.nim index fb5cca4..eebca2d 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -479,7 +479,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card = result.url = getPicUrl(result.image) proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull(); - replyId: int64 = 0): Tweet = + replyId: int64 = 0; hasArticle = false): Tweet = if js.isNull: return Tweet() let time = @@ -547,7 +547,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull(); elif name.len > 0 and jsCard{"binding_values"}.notNull: result.card = some parseCard(jsCard, js{"entities", "urls"}) - result.expandTweetEntities(js) + result.expandTweetEntities(js, hasArticle) parseLegacyMediaEntities(js, result) with jsWithheld, js{"withheld_in_countries"}: @@ -605,6 +605,8 @@ proc parseGraphTweet*(js: JsonNode): Tweet = with restId, js{"reply_to_results", "rest_id"}: replyId = restId.getId + let hasArticle = js{"article", "article_results", "result", "title"}.getStr.len > 0 + if "details" in js: result = Tweet( id: js{"rest_id"}.getId, @@ -639,7 +641,6 @@ proc parseGraphTweet*(js: JsonNode): Tweet = if result.attribution.isNone: parseLegacyMediaEntities(js{"legacy"}, result) - let hasArticle = js{"article", "article_results", "result", "title"}.getStr.len > 0 result.expandTweetEntitiesV2(js, hasArticle) # Strip video source URL from text (for videos from other tweets) @@ -653,7 +654,7 @@ proc parseGraphTweet*(js: JsonNode): Tweet = result.text = result.text[0 ..< idx].strip() break else: - result = parseTweet(js{"legacy"}, jsCard, replyId) + result = parseTweet(js{"legacy"}, jsCard, replyId, hasArticle) result.id = js{"rest_id"}.getId with artNode, js{"article", "article_results", "result"}: @@ -662,7 +663,10 @@ proc parseGraphTweet*(js: JsonNode): Tweet = result.articlePreview = some ArticlePreview( title: artTitle, previewText: artNode{"preview_text"}.getStr, - coverImage: artNode{"cover_media_results", "result", "media_info", "original_img_url"}.getImageStr, + coverImage: select( + artNode{"cover_media_results", "result", "media_info", "original_img_url"}, + artNode{"cover_media", "media_info", "original_img_url"} + ).getImageStr, tweetId: result.id ) diff --git a/src/parserutils.nim b/src/parserutils.nim index 6fbed4f..4b8e0b4 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -281,7 +281,7 @@ proc expandUserEntities*(user: var User; js: JsonNode) = .replacef(htRegex, htReplace) proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlice: Slice[int]; - replyTo=""; hasRedundantLink=false) = + replyTo=""; hasRedundantLink=false; hasArticle=false) = let hasCard = tweet.card.isSome var replacements = newSeq[ReplaceSlice]() @@ -292,7 +292,8 @@ proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlic if urlStr.len == 0 or urlStr notin text: continue - replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink) + replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink, + hideArticle = hasArticle) if hasCard and u{"url"}.getStr == get(tweet.card).url: get(tweet.card).url = u.getExpandedUrl @@ -329,7 +330,7 @@ proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlic tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false) -proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = +proc expandTweetEntities*(tweet: Tweet; js: JsonNode; hasArticle=false) = let entities = ? js{"entities"} textRange = js{"display_text_range"} @@ -344,7 +345,7 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = tweet.reply.add replyTo tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, - hasQuote or hasJobCard) + hasQuote or hasJobCard, hasArticle) proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: Slice[int]; hasRedundantLink=false; hasArticle=false) = diff --git a/src/query.nim b/src/query.nim index aeda0a1..ecb428d 100644 --- a/src/query.nim +++ b/src/query.nim @@ -29,6 +29,10 @@ proc initQuery*(pms: Table[string, string]; name=""): Query = minLikes: validateNumber(@"min_faves") ) + # articles is an internal tab kind, not a valid search filter + if result.kind == QueryKind.articles: + result.kind = tweets + if name.len > 0: result.fromUser = name.split(",") @@ -40,6 +44,12 @@ proc getMediaQuery*(name: string): Query = sep: "OR" ) +proc getArticlesQuery*(name: string): Query = + Query( + kind: QueryKind.articles, + fromUser: @[name] + ) + proc getReplyQuery*(name: string): Query = Query( kind: replies, diff --git a/src/routes/rss.nim b/src/routes/rss.nim index f56a2cb..038096c 100644 --- a/src/routes/rss.nim +++ b/src/routes/rss.nim @@ -106,24 +106,16 @@ proc createRssRouter*(cfg: Config) = get "/@name/@tab/rss": cond '.' notin @"name" - cond @"tab" in ["with_replies", "media", "search"] - let rssEnabled = case @"tab" - of "with_replies": cfg.enableRSSUserReplies - of "media": cfg.enableRSSUserMedia - of "search": cfg.enableRSSSearch - else: false - if not rssEnabled: + cond @"tab" in ["with_replies", "media", "search", "articles"] + # articles can't be approximated by search, so multi-user is unsupported + cond not (@"tab" == "articles" and ',' in @"name") + if not cfg.tabRssEnabled(@"tab"): resp Http403, showError("RSS feed is disabled", cfg) let prefs = requestPrefs() name = @"name" tab = @"tab" - query = - case tab - of "with_replies": getReplyQuery(name) - of "media": getMediaQuery(name) - of "search": initQuery(params(request), name=name) - else: Query(fromUser: @[name]) + query = request.getQuery(tab, name, prefs) let searchKey = if tab != "search": "" else: ":" & $hash(genQueryUrl(query)) diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 35ef1aa..c7949a2 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -12,11 +12,22 @@ export router_utils export redis_cache, formatters, query, api export profile, timeline, status, about_account +proc tabRssEnabled*(cfg: Config; tab: string): bool = + case tab + of "": cfg.enableRSSUserTweets + of "with_replies": cfg.enableRSSUserReplies + of "media": cfg.enableRSSUserMedia + of "articles": cfg.enableRSSUserArticles + of "search": cfg.enableRSSSearch + else: false + proc getQuery*(request: Request; tab, name: string; prefs: Prefs): Query = let view = request.params.getOrDefault("view") case tab of "with_replies": result = getReplyQuery(name) + of "articles": + result = getArticlesQuery(name) of "media": result = getMediaQuery(name) result.view = @@ -64,6 +75,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false): Future[Profile] of posts: await getGraphUserTweets(userId, TimelineKind.tweets, after) of replies: await getGraphUserTweets(userId, TimelineKind.replies, after) of media: await getGraphUserTweets(userId, TimelineKind.media, after) + of QueryKind.articles: await getGraphUserTweets(userId, TimelineKind.articles, after) else: Profile(tweets: await getGraphTweetSearch(query, after)) result.user = await user @@ -172,7 +184,9 @@ proc createTimelineRouter*(cfg: Config) = cond '.' notin @"name" cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"] cond @"name".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9', '_', ','}) - cond @"tab" in ["with_replies", "media", "search", ""] + cond @"tab" in ["with_replies", "media", "search", "articles", ""] + # articles can't be approximated by search, so multi-user is unsupported + cond not (@"tab" == "articles" and ',' in @"name") let prefs = requestPrefs() after = getCursor() @@ -196,15 +210,8 @@ proc createTimelineRouter*(cfg: Config) = profile.tweets.beginning = true resp $renderTimelineTweets(profile.tweets, prefs, getPath()) - let rssEnabled = - if @"tab".len == 0: cfg.enableRSSUserTweets - elif @"tab" == "with_replies": cfg.enableRSSUserReplies - elif @"tab" == "media": cfg.enableRSSUserMedia - elif @"tab" == "search": cfg.enableRSSSearch - else: false - let rss = - if not rssEnabled: + if not cfg.tabRssEnabled(@"tab"): "" elif @"tab".len == 0: "/$1/rss" % @"name" diff --git a/src/types.nim b/src/types.nim index bb521c7..0a748ba 100644 --- a/src/types.nim +++ b/src/types.nim @@ -11,7 +11,7 @@ type BadClientError* = object of CatchableError TimelineKind* {.pure.} = enum - tweets, replies, media + tweets, replies, media, articles ApiUrl* = object endpoint*: string @@ -174,7 +174,8 @@ type variants*: seq[VideoVariant] QueryKind* = enum - posts, replies, media, users, tweets, userList, followers, following, lists, top + posts, replies, media, users, tweets, userList, followers, following, lists, top, + articles RankingMode* = enum Relevance, Recency, Likes @@ -431,6 +432,7 @@ type enableRSSUserTweets*: bool enableRSSUserReplies*: bool enableRSSUserMedia*: bool + enableRSSUserArticles*: bool enableRSSSearch*: bool enableRSSList*: bool enableDebug*: bool diff --git a/src/views/rss.nimf b/src/views/rss.nimf index 9c703d5..4738705 100644 --- a/src/views/rss.nimf +++ b/src/views/rss.nimf @@ -17,6 +17,11 @@ ## text = unicode.runeSubStr(text, 0, 32) & "..." ##end if #text = xmltree.escape(text) +## article tweets' text is just the article link; the title says more +#if tweet.articlePreview.isSome and tweet.articlePreview.get().title.len > 0: +# result = prefix & xmltree.escape(tweet.articlePreview.get().title) +# return +#end if #if text.len > 0: # result = prefix & text # return @@ -34,12 +39,6 @@ # end case # end if #end if -#if result.len == 0 and tweet.articlePreview.isSome: -# let art = tweet.articlePreview.get() -# if art.title.len > 0: -# result = prefix & xmltree.escape(art.title) -# end if -#end if #if result.len == 0 and tweet.card.isSome: # let card = tweet.card.get() # if card.kind notin {hidden, unknown} and card.title.len > 0: diff --git a/src/views/search.nim b/src/views/search.nim index 1c29203..4d4ed5e 100644 --- a/src/views/search.nim +++ b/src/views/search.nim @@ -36,6 +36,9 @@ proc renderProfileTabs*(query: Query; username: string): VNode = a(href=(link & "/with_replies")): text "Tweets & Replies" li(class=query.getTabClass(media)): a(href=(link & "/media")): text "Media" + if query.fromUser.len == 1: + li(class=query.getTabClass(QueryKind.articles)): + a(href=(link & "/articles")): text "Articles" li(class=query.getTabClass(tweets)): a(href=(link & "/search")): text "Search" diff --git a/tests/test_timeline.py b/tests/test_timeline.py index 65c598f..e40dbc2 100644 --- a/tests/test_timeline.py +++ b/tests/test_timeline.py @@ -84,3 +84,31 @@ class TweetTest(BaseTestCase): #for i, url in enumerate(images): #img = self.get_attribute(Timeline.photo_rail + f' a:nth-child({i + 1}) img', 'src') #self.assertIn(url, img) + + +class ArticlesTabTest(BaseTestCase): + def test_articles_tab_on_profile(self): + self.open_nitter('satyanadella') + self.assert_element_present('.tab .tab-item a[href="/satyanadella/articles"]') + + def test_articles_timeline(self): + self.open_nitter('satyanadella/articles') + self.assert_text('Articles', '.tab .tab-item.active a') + self.assert_element_present('.timeline .article-card') + self.assert_element_present('.timeline .article-card a[href^="/i/article/"]') + + def test_articles_card_cover_and_no_raw_link(self): + self.open_nitter('jack/articles') + self.assert_element_present('.timeline .article-card .card-image img') + # the article link tweet text is redundant with the card and is stripped + self.assert_element_absent('.timeline .tweet-content a[href*="x.com/i/article"]') + + def test_articles_empty(self): + self.open_nitter('mobile_test/articles') + self.assert_text('No items found', Timeline.none) + + def test_articles_multi_user_unsupported(self): + self.open_nitter('jack,satyanadella') + self.assert_element_absent('.tab .tab-item a[href$="/articles"]') + self.open_nitter('jack,satyanadella/articles') + self.assert_text('Page not found')