Add Articles tab to profile pages

Fixes #1426

nitter.example.conf

@@ -26,6 +26,7 @@ enableRSS = true             # master switch, set to false to disable all RSS fe
 enableRSSUserTweets = true   # /@user/rss
 enableRSSUserReplies = true  # /@user/with_replies/rss
 enableRSSUserMedia = true    # /@user/media/rss
+enableRSSUserArticles = true # /@user/articles/rss
 enableRSSSearch = true       # /search/rss and /@user/search/rss
 enableRSSList = true         # list RSS feeds
 enableDebug = false          # enable request logs and debug endpoints (/.sessions)

src/api.nim

@@ -38,6 +38,12 @@ proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq =
     oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles, skipTid=true)
   )
 
+proc userArticlesUrl(id: string; cursor: string): ApiReq =
+  result = ApiReq(
+    cookie: apiUrl(graphUserArticles, userArticlesVars % [id, cursor], userTweetsFieldToggles),
+    oauth: apiUrl(graphUserArticlesV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles)
+  )
+
 proc tweetDetailUrl(id, cursor: string; mode = Relevance): ApiReq =
   return apiReq(graphTweet, tweetVars % [id, cursor, $mode])
   # let cookieVars = tweetDetailVars % [id, cursor]
@@ -112,6 +118,7 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi
       of TimelineKind.tweets: userTweetsUrl(id, cursor)
       of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor)
       of TimelineKind.media: mediaUrl(id, cursor)
+      of TimelineKind.articles: userArticlesUrl(id, cursor)
     js = await fetch(url)
   result = parseGraphTimeline(js, after)
 

src/config.nim

@@ -42,6 +42,7 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
     enableRSSUserTweets: masterRss and cfg.get("Config", "enableRSSUserTweets", true),
     enableRSSUserReplies: masterRss and cfg.get("Config", "enableRSSUserReplies", true),
     enableRSSUserMedia: masterRss and cfg.get("Config", "enableRSSUserMedia", true),
+    enableRSSUserArticles: masterRss and cfg.get("Config", "enableRSSUserArticles", true),
     enableRSSSearch: masterRss and cfg.get("Config", "enableRSSSearch", true),
     enableRSSList: masterRss and cfg.get("Config", "enableRSSList", true),
     enableDebug: cfg.get("Config", "enableDebug", false),

src/consts.nim

@@ -16,6 +16,8 @@ const
   graphUserTweetsAndReplies* = "qUpkZU6eN8MbtQb7rC_pYg/UserTweetsAndReplies"
   graphUserMedia* = "VyudDWQnr9vJNw7GasFz2g/UserMedia"
   graphUserMediaV2* = "WK111rbR0vM0ZX4lyZCYjw/MediaTimelineV2"
+  graphUserArticles* = "ZmMjUyrTpwYfTGAdylEyMw/UserArticlesTweets"
+  graphUserArticlesV2* = "PsGixN38UZz2RheyayNB5Q/UserProfileArticlesTimelineQuery"
   graphTweet* = "OZMbEnEa96AN8Pq6HyTWdw/ConversationTimeline"
   graphTweetDetail* = "XMOz5h24KAZ86qKffKTLdQ/TweetDetail"
   graphTweetResult* = "xYOrBQoTlfKJJPsX76MZEw/TweetResultByIdQuery"
@@ -148,6 +150,13 @@ const
   "withVoice": true
 }""".replace(" ", "").replace("\n", "")
 
+  userArticlesVars* = """{
+  "userId": "$1", $2
+  "count": 20,
+  "includePromotedContent": false,
+  "withVoice": true
+}""".replace(" ", "").replace("\n", "")
+
   articleVars* = """{
   "tweetId": "$1",
   "includePromotedContent": false,

src/parser.nim

@@ -479,7 +479,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
     result.url = getPicUrl(result.image)
 
 proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
-                replyId: int64 = 0): Tweet =
+                replyId: int64 = 0; hasArticle = false): Tweet =
   if js.isNull: return Tweet()
 
   let time =
@@ -547,7 +547,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
     elif name.len > 0 and jsCard{"binding_values"}.notNull:
       result.card = some parseCard(jsCard, js{"entities", "urls"})
 
-  result.expandTweetEntities(js)
+  result.expandTweetEntities(js, hasArticle)
   parseLegacyMediaEntities(js, result)
 
   with jsWithheld, js{"withheld_in_countries"}:
@@ -605,6 +605,8 @@ proc parseGraphTweet*(js: JsonNode): Tweet =
   with restId, js{"reply_to_results", "rest_id"}:
     replyId = restId.getId
 
+  let hasArticle = js{"article", "article_results", "result", "title"}.getStr.len > 0
+
   if "details" in js:
     result = Tweet(
       id: js{"rest_id"}.getId,
@@ -639,7 +641,6 @@ proc parseGraphTweet*(js: JsonNode): Tweet =
     if result.attribution.isNone:
       parseLegacyMediaEntities(js{"legacy"}, result)
 
-    let hasArticle = js{"article", "article_results", "result", "title"}.getStr.len > 0
     result.expandTweetEntitiesV2(js, hasArticle)
 
     # Strip video source URL from text (for videos from other tweets)
@@ -653,7 +654,7 @@ proc parseGraphTweet*(js: JsonNode): Tweet =
               result.text = result.text[0 ..< idx].strip()
             break
   else:
-    result = parseTweet(js{"legacy"}, jsCard, replyId)
+    result = parseTweet(js{"legacy"}, jsCard, replyId, hasArticle)
     result.id = js{"rest_id"}.getId
 
   with artNode, js{"article", "article_results", "result"}:
@@ -662,7 +663,10 @@ proc parseGraphTweet*(js: JsonNode): Tweet =
       result.articlePreview = some ArticlePreview(
         title: artTitle,
         previewText: artNode{"preview_text"}.getStr,
-        coverImage: artNode{"cover_media_results", "result", "media_info", "original_img_url"}.getImageStr,
+        coverImage: select(
+          artNode{"cover_media_results", "result", "media_info", "original_img_url"},
+          artNode{"cover_media", "media_info", "original_img_url"}
+        ).getImageStr,
         tweetId: result.id
       )
 

src/parserutils.nim

@@ -281,7 +281,7 @@ proc expandUserEntities*(user: var User; js: JsonNode) =
                      .replacef(htRegex, htReplace)
 
 proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlice: Slice[int];
-                        replyTo=""; hasRedundantLink=false) =
+                        replyTo=""; hasRedundantLink=false; hasArticle=false) =
   let hasCard = tweet.card.isSome
 
   var replacements = newSeq[ReplaceSlice]()
@@ -292,7 +292,8 @@ proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlic
       if urlStr.len == 0 or urlStr notin text:
         continue
 
-      replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink)
+      replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink,
+                               hideArticle = hasArticle)
 
       if hasCard and u{"url"}.getStr == get(tweet.card).url:
         get(tweet.card).url = u.getExpandedUrl
@@ -329,7 +330,7 @@ proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlic
 
   tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false)
 
-proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
+proc expandTweetEntities*(tweet: Tweet; js: JsonNode; hasArticle=false) =
   let
     entities = ? js{"entities"}
     textRange = js{"display_text_range"}
@@ -344,7 +345,7 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
       tweet.reply.add replyTo
 
   tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo,
-                           hasQuote or hasJobCard)
+                           hasQuote or hasJobCard, hasArticle)
 
 proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: Slice[int];
                           hasRedundantLink=false; hasArticle=false) =

src/query.nim

@@ -29,6 +29,10 @@ proc initQuery*(pms: Table[string, string]; name=""): Query =
     minLikes: validateNumber(@"min_faves")
   )
 
+  # articles is an internal tab kind, not a valid search filter
+  if result.kind == QueryKind.articles:
+    result.kind = tweets
+
   if name.len > 0:
     result.fromUser = name.split(",")
 
@@ -40,6 +44,12 @@ proc getMediaQuery*(name: string): Query =
     sep: "OR"
   )
 
+proc getArticlesQuery*(name: string): Query =
+  Query(
+    kind: QueryKind.articles,
+    fromUser: @[name]
+  )
+
 proc getReplyQuery*(name: string): Query =
   Query(
     kind: replies,

src/routes/rss.nim

@@ -106,24 +106,16 @@ proc createRssRouter*(cfg: Config) =
 
     get "/@name/@tab/rss":
       cond '.' notin @"name"
-      cond @"tab" in ["with_replies", "media", "search"]
+      cond @"tab" in ["with_replies", "media", "search", "articles"]
-      let rssEnabled = case @"tab"
+      # articles can't be approximated by search, so multi-user is unsupported
-        of "with_replies": cfg.enableRSSUserReplies
+      cond not (@"tab" == "articles" and ',' in @"name")
-        of "media": cfg.enableRSSUserMedia
+      if not cfg.tabRssEnabled(@"tab"):
-        of "search": cfg.enableRSSSearch
-        else: false
-      if not rssEnabled:
         resp Http403, showError("RSS feed is disabled", cfg)
       let
         prefs = requestPrefs()
         name = @"name"
         tab = @"tab"
-        query =
+        query = request.getQuery(tab, name, prefs)
-          case tab
-          of "with_replies": getReplyQuery(name)
-          of "media": getMediaQuery(name)
-          of "search": initQuery(params(request), name=name)
-          else: Query(fromUser: @[name])
 
       let searchKey = if tab != "search": ""
                       else: ":" & $hash(genQueryUrl(query))

src/routes/timeline.nim

@@ -12,11 +12,22 @@ export router_utils
 export redis_cache, formatters, query, api
 export profile, timeline, status, about_account
 
+proc tabRssEnabled*(cfg: Config; tab: string): bool =
+  case tab
+  of "": cfg.enableRSSUserTweets
+  of "with_replies": cfg.enableRSSUserReplies
+  of "media": cfg.enableRSSUserMedia
+  of "articles": cfg.enableRSSUserArticles
+  of "search": cfg.enableRSSSearch
+  else: false
+
 proc getQuery*(request: Request; tab, name: string; prefs: Prefs): Query =
   let view = request.params.getOrDefault("view")
   case tab
   of "with_replies":
     result = getReplyQuery(name)
+  of "articles":
+    result = getArticlesQuery(name)
   of "media":
     result = getMediaQuery(name)
     result.view =
@@ -64,6 +75,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false): Future[Profile]
     of posts: await getGraphUserTweets(userId, TimelineKind.tweets, after)
     of replies: await getGraphUserTweets(userId, TimelineKind.replies, after)
     of media: await getGraphUserTweets(userId, TimelineKind.media, after)
+    of QueryKind.articles: await getGraphUserTweets(userId, TimelineKind.articles, after)
     else: Profile(tweets: await getGraphTweetSearch(query, after))
 
   result.user = await user
@@ -172,7 +184,9 @@ proc createTimelineRouter*(cfg: Config) =
       cond '.' notin @"name"
       cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]
       cond @"name".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9', '_', ','})
-      cond @"tab" in ["with_replies", "media", "search", ""]
+      cond @"tab" in ["with_replies", "media", "search", "articles", ""]
+      # articles can't be approximated by search, so multi-user is unsupported
+      cond not (@"tab" == "articles" and ',' in @"name")
       let
         prefs = requestPrefs()
         after = getCursor()
@@ -196,15 +210,8 @@ proc createTimelineRouter*(cfg: Config) =
           profile.tweets.beginning = true
           resp $renderTimelineTweets(profile.tweets, prefs, getPath())
 
-      let rssEnabled =
-        if @"tab".len == 0: cfg.enableRSSUserTweets
-        elif @"tab" == "with_replies": cfg.enableRSSUserReplies
-        elif @"tab" == "media": cfg.enableRSSUserMedia
-        elif @"tab" == "search": cfg.enableRSSSearch
-        else: false
-
       let rss =
-        if not rssEnabled: 
+        if not cfg.tabRssEnabled(@"tab"): 
           ""
         elif @"tab".len == 0:
           "/$1/rss" % @"name"

src/types.nim

@@ -11,7 +11,7 @@ type
   BadClientError* = object of CatchableError
 
   TimelineKind* {.pure.} = enum
-    tweets, replies, media
+    tweets, replies, media, articles
 
   ApiUrl* = object
     endpoint*: string
@@ -174,7 +174,8 @@ type
     variants*: seq[VideoVariant]
 
   QueryKind* = enum
-    posts, replies, media, users, tweets, userList, followers, following, lists, top
+    posts, replies, media, users, tweets, userList, followers, following, lists, top,
+    articles
 
   RankingMode* = enum
     Relevance, Recency, Likes
@@ -431,6 +432,7 @@ type
     enableRSSUserTweets*: bool
     enableRSSUserReplies*: bool
     enableRSSUserMedia*: bool
+    enableRSSUserArticles*: bool
     enableRSSSearch*: bool
     enableRSSList*: bool
     enableDebug*: bool

src/views/rss.nimf

@@ -17,6 +17,11 @@
 ##  text = unicode.runeSubStr(text, 0, 32) & "..."
 ##end if
 #text = xmltree.escape(text)
+## article tweets' text is just the article link; the title says more
+#if tweet.articlePreview.isSome and tweet.articlePreview.get().title.len > 0:
+#  result = prefix & xmltree.escape(tweet.articlePreview.get().title)
+#  return
+#end if
 #if text.len > 0:
 #  result = prefix & text
 #  return
@@ -34,12 +39,6 @@
 #    end case
 #  end if
 #end if
-#if result.len == 0 and tweet.articlePreview.isSome:
-#  let art = tweet.articlePreview.get()
-#  if art.title.len > 0:
-#    result = prefix & xmltree.escape(art.title)
-#  end if
-#end if
 #if result.len == 0 and tweet.card.isSome:
 #  let card = tweet.card.get()
 #  if card.kind notin {hidden, unknown} and card.title.len > 0:

src/views/search.nim

@@ -36,6 +36,9 @@ proc renderProfileTabs*(query: Query; username: string): VNode =
       a(href=(link & "/with_replies")): text "Tweets & Replies"
     li(class=query.getTabClass(media)):
       a(href=(link & "/media")): text "Media"
+    if query.fromUser.len == 1:
+      li(class=query.getTabClass(QueryKind.articles)):
+        a(href=(link & "/articles")): text "Articles"
     li(class=query.getTabClass(tweets)):
       a(href=(link & "/search")): text "Search"
 

tests/test_timeline.py

@@ -84,3 +84,31 @@ class TweetTest(BaseTestCase):
         #for i, url in enumerate(images):
             #img = self.get_attribute(Timeline.photo_rail + f' a:nth-child({i + 1}) img', 'src')
             #self.assertIn(url, img)
+
+
+class ArticlesTabTest(BaseTestCase):
+    def test_articles_tab_on_profile(self):
+        self.open_nitter('satyanadella')
+        self.assert_element_present('.tab .tab-item a[href="/satyanadella/articles"]')
+
+    def test_articles_timeline(self):
+        self.open_nitter('satyanadella/articles')
+        self.assert_text('Articles', '.tab .tab-item.active a')
+        self.assert_element_present('.timeline .article-card')
+        self.assert_element_present('.timeline .article-card a[href^="/i/article/"]')
+
+    def test_articles_card_cover_and_no_raw_link(self):
+        self.open_nitter('jack/articles')
+        self.assert_element_present('.timeline .article-card .card-image img')
+        # the article link tweet text is redundant with the card and is stripped
+        self.assert_element_absent('.timeline .tweet-content a[href*="x.com/i/article"]')
+
+    def test_articles_empty(self):
+        self.open_nitter('mobile_test/articles')
+        self.assert_text('No items found', Timeline.none)
+
+    def test_articles_multi_user_unsupported(self):
+        self.open_nitter('jack,satyanadella')
+        self.assert_element_absent('.tab .tab-item a[href$="/articles"]')
+        self.open_nitter('jack,satyanadella/articles')
+        self.assert_text('Page not found')