Merge branch 'zedeus:master' into master

src/api.nim

@@ -2,7 +2,7 @@
 import asyncdispatch, httpclient, strutils, sequtils, sugar
 import packedjson
 import types, query, formatters, consts, apiutils, parser, utils
-import experimental/parser as newParser
+import experimental/parser
 
 # Helper to generate params object for GraphQL requests
 proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
@@ -11,13 +11,18 @@ proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
   if fieldToggles.len > 0:
     result.add ("fieldToggles", fieldToggles)
 
-proc apiUrl(endpoint, variables: string; fieldToggles = ""): ApiUrl =
-  return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles))
+proc apiUrl(endpoint, variables: string; fieldToggles = ""; skipTid = false): ApiUrl =
+  return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles), skipTid: skipTid)
 
-proc apiReq(endpoint, variables: string; fieldToggles = ""): ApiReq =
-  let url = apiUrl(endpoint, variables, fieldToggles)
+proc apiReq(endpoint, variables: string; fieldToggles = ""; skipTid = false): ApiReq =
+  let url = apiUrl(endpoint, variables, fieldToggles, skipTid)
   return ApiReq(cookie: url, oauth: url)
 
+proc cursorParam(after: string): string =
+  ## JSON-escape the user-supplied cursor so it cannot break out of the GraphQL
+  ## variables object (same input-validation class as the #1411 media SSRF).
+  if after.len > 0: "\"cursor\":" & $(%after) & "," else: ""
+
 proc mediaUrl(id, cursor: string; count=20): ApiReq =
   result = ApiReq(
     cookie: apiUrl(graphUserMedia, userMediaVars % [id, cursor, $count]),
@@ -25,33 +30,24 @@ proc mediaUrl(id, cursor: string; count=20): ApiReq =
   )
 
 proc userTweetsUrl(id: string; cursor: string): ApiReq =
-  result = ApiReq(
-    # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
-    oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"])
-  )
-  # might change this in the future pending testing
-  result.cookie = result.oauth
+  return apiReq(graphUserTweetsV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles)
 
 proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq =
-  let cookieVars = userTweetsAndRepliesVars % [id, cursor]
-  result = ApiReq(
-    cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
-    oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"])
-  )
+  return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles, skipTid=true)
 
 proc tweetDetailUrl(id: string; cursor: string): ApiReq =
-  let cookieVars = tweetDetailVars % [id, cursor]
-  result = ApiReq(
-    # cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
-    cookie: apiUrl(graphTweet, tweetVars % [id, cursor]),
-    oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
-  )
+  return apiReq(graphTweet, tweetVars % [id, cursor])
+  # let cookieVars = tweetDetailVars % [id, cursor]
+  # result = ApiReq(
+  #   cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
+  #   oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
+  # )
 
 proc userUrl(username: string): ApiReq =
-  let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
+  let cookieVars = $(%*{"screen_name": username, "withGrokTranslatedBio": false})
   result = ApiReq(
     cookie: apiUrl(graphUser, cookieVars, tweetDetailFieldToggles),
-    oauth: apiUrl(graphUserV2, """{"screen_name": "$1"}""" % username)
+    oauth: apiUrl(graphUserV2, $(%*{"screen_name": username}))
   )
 
 proc getGraphUser*(username: string): Future[User] {.async.} =
@@ -69,7 +65,7 @@ proc getGraphUserById*(id: string): Future[User] {.async.} =
 proc getAboutAccount*(username: string): Future[AccountInfo] {.async.} =
   if username.len == 0: return
   let
-    url = apiReq(graphAboutAccount, """{"screenName":"$1"}""" % username)
+    url = apiReq(graphAboutAccount, $(%*{"screenName": username}))
     js = await fetch(url)
   result = parseAboutAccount(js)
 
@@ -80,7 +76,7 @@ proc restReq(endpoint: string; params: seq[(string, string)] = @[]): ApiReq =
 proc getBroadcastInfo*(id: string): Future[Broadcast] {.async.} =
   if id.len == 0: return
   let
-    req = apiReq(graphBroadcast, """{"id":"$1"}""" % id)
+    req = apiReq(graphBroadcast, $(%*{"id": id}))
     js = await fetch(req)
   result = parseBroadcastInfo(js)
 
@@ -92,10 +88,23 @@ proc fetchBroadcastStream*(mediaKey: string): Future[string] {.async.} =
   result = streamJs{"source", "noRedirectPlaybackUrl"}.getStr(
     streamJs{"source", "location"}.getStr)
 
+proc getAudioSpace*(id: string): Future[AudioSpace] {.async.} =
+  if id.len == 0: return
+  let
+    variables = %*{
+      "id": id,
+      "isMetatagsQuery": false,
+      "withReplays": true,
+      "withListeners": true
+    }
+    req = apiReq(graphAudioSpace, $variables)
+    js = await fetch(req)
+  result = parseAudioSpace(js)
+
 proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
   if id.len == 0: return
   let
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    cursor = cursorParam(after)
     url = case kind
       of TimelineKind.tweets: userTweetsUrl(id, cursor)
       of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor)
@@ -103,10 +112,61 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi
     js = await fetch(url)
   result = parseGraphTimeline(js, after)
 
+proc getGraphCommunity*(id: string): Future[Community] {.async.} =
+  if id.len == 0: return
+  let
+    url = apiReq(graphCommunity, $(%*{"communityId": id}))
+    js = await fetch(url)
+  result = parseGraphCommunity(js)
+
+proc getGraphCommunityTweets*(id: string; rankingMode: string; after=""): Future[Timeline] {.async.} =
+  if id.len == 0: return
+  let
+    cursor = cursorParam(after)
+    url = apiReq(graphCommunityTweets, communityTweetsVars % [id, cursor, rankingMode])
+    js = await fetch(url)
+  result = parseGraphCommunityTimeline(js, after)
+
+proc getGraphCommunityMedia*(id: string; after=""): Future[Timeline] {.async.} =
+  if id.len == 0: return
+  let
+    cursor = cursorParam(after)
+    url = apiReq(graphCommunityMedia, communityMediaVars % [id, cursor])
+    js = await fetch(url)
+  result = parseGraphCommunityTimeline(js, after)
+
+proc communitySliceReq(endpoint, variables: string): ApiReq =
+  let url = ApiUrl(endpoint: endpoint, params: @[("variables", variables)])
+  ApiReq(cookie: url, oauth: url)
+
+proc getGraphCommunityMembers*(id: string; after=""): Future[Result[User]] {.async.} =
+  if id.len == 0: return
+  let
+    cursor = if after.len > 0: $(%after) else: "null"
+    url = communitySliceReq(graphCommunityMembers, communityMembersVars % [id, cursor])
+    js = await fetch(url)
+  result = parseGraphCommunityMembers(js, after)
+
+proc getGraphCommunityModerators*(id: string): Future[Result[User]] {.async.} =
+  if id.len == 0: return
+  let
+    url = communitySliceReq(graphCommunityModerators, communityMembersVars % [id, "null"])
+    js = await fetch(url)
+  result = parseGraphCommunityMembers(js)
+
+proc getGraphCommunityHashtags*(id, hashtag: string; after=""): Future[Timeline] {.async.} =
+  if id.len == 0 or hashtag.len == 0: return
+  let
+    safeTag = multiReplace(hashtag, ("\"", ""), ("\\", ""))
+    cursor = cursorParam(after)
+    url = apiReq(graphCommunityHashtags, communityHashtagsVars % [id, cursor, safeTag])
+    js = await fetch(url)
+  result = parseGraphCommunityTimeline(js, after)
+
 proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
   if id.len == 0: return
   let
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    cursor = cursorParam(after)
     url = apiReq(graphListTweets, restIdVars % [id, cursor, "20"])
     js = await fetch(url)
   result = parseGraphTimeline(js, after).tweets
@@ -120,7 +180,7 @@ proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
 
 proc getGraphList*(id: string): Future[List] {.async.} =
   let 
-    url = apiReq(graphListById, """{"listId": "$1"}""" % id)
+    url = apiReq(graphListById, $(%*{"listId": id}))
     js = await fetch(url)
   result = parseGraphList(js)
 
@@ -141,17 +201,39 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
     js = await fetchRaw(url)
   result = parseGraphListMembers(js, after)
 
+proc getGraphUserConnections(userId: string; endpoint: string; kind: QueryKind;
+                              after=""): Future[Result[User]] {.async.} =
+  if userId.len == 0: return
+  var variables = %*{
+    "userId": userId,
+    "count": 20,
+    "includePromotedContent": false,
+    "withGrokTranslatedBio": true
+  }
+  if after.len > 0:
+    variables["cursor"] = %after
+  let
+    url = apiReq(endpoint, $variables)
+    js = await fetchRaw(url)
+  result = parseGraphFollowers(js, after, kind)
+
+proc getGraphFollowers*(userId: string; after=""): Future[Result[User]] {.async.} =
+  result = await getGraphUserConnections(userId, graphFollowers, followers, after)
+
+proc getGraphFollowing*(userId: string; after=""): Future[Result[User]] {.async.} =
+  result = await getGraphUserConnections(userId, graphFollowing, following, after)
+
 proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
   if id.len == 0: return
   let
-    url = apiReq(graphTweetResult, """{"rest_id": "$1"}""" % id)
+    url = apiReq(graphTweetResult, $(%*{"rest_id": id}))
     js = await fetch(url)
   result = parseGraphTweetResult(js)
 
 proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
   if id.len == 0: return
   let
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    cursor = cursorParam(after)
     js = await fetch(tweetDetailUrl(id, cursor))
   result = parseGraphConversation(js, id)
 
@@ -184,13 +266,13 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
   var
     variables = %*{
       "rawQuery": q,
-      "query_source": "typedQuery",
       "count": 20,
+      "querySource": "typed_query",
       "product": "Latest",
-      "withDownvotePerspective": false,
-      "withReactionsMetadata": false,
-      "withReactionsPerspective": false
+      "withGrokTranslatedBio":true,
+      "withQuickPromoteEligibilityTweetFields":false
     }
+
   if after.len > 0 and maxId.len == 0:
     variables["cursor"] = % after
   let
@@ -212,12 +294,11 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
   var
     variables = %*{
       "rawQuery": query.text,
-      "query_source": "typedQuery",
       "count": 20,
+      "querySource": "typed_query",
       "product": "People",
-      "withDownvotePerspective": false,
-      "withReactionsMetadata": false,
-      "withReactionsPerspective": false
+      "withGrokTranslatedBio":true,
+      "withQuickPromoteEligibilityTweetFields":false
     }
   if after.len > 0:
     variables["cursor"] = % after
@@ -234,6 +315,21 @@ proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
   let js = await fetch(mediaUrl(id, "", 30))
   result = parseGraphPhotoRail(js)
 
+proc getGraphArticle*(id: string): Future[Article] {.async.} =
+  if id.len == 0: return
+  let
+    url = apiReq(graphTweetResultByRestId, articleVars % id, articleFieldToggles)
+    json = await fetchRaw(url)
+  result = parseGraphArticle(json)
+
+proc getGraphTweetResults*(ids: seq[string]): Future[seq[Tweet]] {.async.} =
+  if ids.len == 0: return
+  let
+    idsJson = "[" & ids.mapIt("\"" & it & "\"").join(",") & "]"
+    url = apiReq(graphTweetResultsByRestIds, articleBatchVars % idsJson, articleFieldToggles)
+    js = await fetch(url)
+  result = parseGraphTweetResults(js)
+
 proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
   let client = newAsyncHttpClient(maxRedirects=0)
   try:

src/apiutils.nim

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
-import jsony, packedjson, zippy, oauth1
+import jsony, packedjson, zippy, oauth/oauth1
 import types, auth, consts, parserutils, http_pool, tid
 import experimental/types/common
 
@@ -8,6 +8,7 @@ const
   rlRemaining = "x-rate-limit-remaining"
   rlReset = "x-rate-limit-reset"
   rlLimit = "x-rate-limit-limit"
+  npCache = "x-np-cache"
   errorsToSkip = {null, doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
 
 var
@@ -63,7 +64,7 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
 proc getCookieHeader(authToken, ct0: string): string =
   "auth_token=" & authToken & "; ct0=" & ct0
 
-proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
+proc genHeaders*(session: Session, url: Uri, skipTid: bool): Future[HttpHeaders] {.async.} =
   result = newHttpHeaders({
     "accept": "*/*",
     "accept-encoding": "gzip",
@@ -84,13 +85,14 @@ proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
     result["x-twitter-auth-type"] = "OAuth2Session"
     result["x-csrf-token"] = session.ct0
     result["cookie"] = getCookieHeader(session.authToken, session.ct0)
+    result["referer"] = "https://x.com/"
     result["sec-ch-ua"] = """"Google Chrome";v="142", "Chromium";v="142", "Not A(Brand";v="24""""
     result["sec-ch-ua-mobile"] = "?0"
     result["sec-ch-ua-platform"] = "Windows"
     result["sec-fetch-dest"] = "empty"
     result["sec-fetch-mode"] = "cors"
-    result["sec-fetch-site"] = "same-site"
-    if disableTid or "/1.1/" in url.path:
+    result["sec-fetch-site"] = "same-origin"
+    if disableTid or skipTid or "/1.1/" in url.path:
       result["authorization"] = bearerToken2
     else:
       result["authorization"] = bearerToken
@@ -114,7 +116,12 @@ template fetchImpl(result, fetchBody) {.dirty.} =
 
   try:
     var resp: AsyncResponse
-    pool.use(await genHeaders(session, url)):
+    let skipTid = case session.kind
+      of oauth: req.oauth.skipTid
+      of cookie: req.cookie.skipTid
+    let headers = await genHeaders(session, url, skipTid)
+
+    pool.use(headers):
       template getContent =
         # TODO: this is a temporary simple implementation
         if apiProxy.len > 0 and "/1.1/" notin url.path:
@@ -130,10 +137,11 @@ template fetchImpl(result, fetchBody) {.dirty.} =
         raise newException(BadClientError, "Bad client")
 
       if resp.status == $Http404 and result.len == 0:
-        echo "[sessions] transient 404 (empty body), retrying: ", url.path
+        echo "[sessions] transient 404 (empty body), retrying: ", url.path, ", session: ", session.pretty
         raise rateLimitError()
 
-    if resp.headers.hasKey(rlRemaining):
+    let cacheStatus = resp.headers.getOrDefault(npCache)
+    if cacheStatus notin ["HIT", "STALE"] and resp.headers.hasKey(rlRemaining):
       let
         remaining = parseInt(resp.headers[rlRemaining])
         reset = parseInt(resp.headers[rlReset])
@@ -147,7 +155,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
       if result.startsWith("{\"errors"):
         let errors = result.fromJson(Errors)
         if errors notin errorsToSkip:
-          echo "Fetch error, API: ", url.path, ", errors: ", errors
+          echo "Fetch error, API: ", url.path, ", errors: ", errors, ", session: ", session.pretty
           if errors in {expiredToken, badToken, locked}:
             invalidate(session)
             raise rateLimitError()
@@ -162,7 +170,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
     fetchBody
 
     if resp.status == $Http400:
-      echo "ERROR 400, ", url.path, ": ", result
+      echo "ERROR 400, ", url.path, ": ", result, ", session: ", session.pretty
       raise newException(InternalError, $url)
   except InternalError as e:
     raise e
@@ -177,22 +185,34 @@ template fetchImpl(result, fetchBody) {.dirty.} =
   finally:
     release(session)
 
-template retry(bod) =
+template retry(bod) {.dirty.} =
+  var session: Session
+  var retrySuccess = false
   for i in 0 ..< maxRetries:
     try:
+      session = nil
       bod
+      retrySuccess = true
       break
     except RateLimitError:
-      echo "[sessions] Rate limited, retrying ", req.cookie.endpoint,
-           " request (", i, "/", maxRetries, ")..."
+      let api = if session.isNil: req.cookie.endpoint
+                else: req.endpoint(session)
+      if session.isNil:
+        echo "[sessions] Rate limited, retrying ", api,
+             " request (", i, "/", maxRetries, ")..."
+      else:
+        echo "[sessions] Rate limited, retrying ", api,
+             " request (", i, "/", maxRetries, ")..., session: ", session.pretty
+      session = nil
       if retryDelayMs > 0:
         await sleepAsync(retryDelayMs)
+  if not retrySuccess:
+    raise rateLimitError()
 
 proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
   retry:
-    var 
-      body: string
-      session = await getAndValidateSession(req)
+    var body: string
+    session = await getAndValidateSession(req)
 
     let url = req.toUrl(session.kind)
 
@@ -200,22 +220,22 @@ proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
       if body.startsWith('{') or body.startsWith('['):
         result = parseJson(body)
       else:
-        echo resp.status, ": ", body, " --- url: ", url
+        echo resp.status, ": ", body, " --- url: ", url, ", session: ", session.pretty
         result = newJNull()
 
       let error = result.getError
       if error != null and error notin errorsToSkip:
-        echo "Fetch error, API: ", url.path, ", error: ", error
+        echo "Fetch error, API: ", url.path, ", error: ", error, ", session: ", session.pretty
         if error in {expiredToken, badToken, locked}:
           invalidate(session)
           raise rateLimitError()
 
 proc fetchRaw*(req: ApiReq): Future[string] {.async.} =
   retry:
-    var session = await getAndValidateSession(req)
+    session = await getAndValidateSession(req)
     let url = req.toUrl(session.kind)
 
     fetchImpl result:
       if not (result.startsWith('{') or result.startsWith('[')):
-        echo resp.status, ": ", result, " --- url: ", url
+        echo resp.status, ": ", result, " --- url: ", url, ", session: ", session.pretty
         result.setLen(0)

src/auth.nim

@@ -18,7 +18,7 @@ proc setMaxConcurrentReqs*(reqs: int) =
 template log(str: varargs[string, `$`]) =
   echo "[sessions] ", str.join("")
 
-proc endpoint(req: ApiReq; session: Session): string =
+proc endpoint*(req: ApiReq; session: Session): string =
   case session.kind
   of oauth: req.oauth.endpoint
   of cookie: req.cookie.endpoint
@@ -50,6 +50,8 @@ proc getSessionPoolHealth*(): JsonNode =
     oldest = now.int64
     newest = 0'i64
     average = 0'i64
+    oauthTotal, cookieTotal = 0
+    oauthLimited, cookieLimited = 0
 
   for session in sessionPool:
     let created = snowflakeToEpoch(session.id)
@@ -59,8 +61,15 @@ proc getSessionPoolHealth*(): JsonNode =
       oldest = created
     average += created
 
+    case session.kind
+    of oauth: inc oauthTotal
+    of cookie: inc cookieTotal
+
     if session.limited:
       limited.incl session.id
+      case session.kind
+      of oauth: inc oauthLimited
+      of cookie: inc cookieLimited
 
     for api in session.apis.keys:
       let
@@ -84,6 +93,8 @@ proc getSessionPoolHealth*(): JsonNode =
     "sessions": %*{
       "total": sessionPool.len,
       "limited": limited.card,
+      "oauth": %*{"total": oauthTotal, "limited": oauthLimited},
+      "cookie": %*{"total": cookieTotal, "limited": cookieLimited},
       "oldest": $fromUnix(oldest),
       "newest": $fromUnix(newest),
       "average": $fromUnix(average)
@@ -100,6 +111,7 @@ proc getSessionPoolDebug*(): JsonNode =
 
   for session in sessionPool:
     let sessionJson = %*{
+      "kind": $session.kind,
       "apis": newJObject(),
       "pending": session.pending,
     }
@@ -173,7 +185,10 @@ proc getSession*(req: ApiReq): Future[Session] {.async.} =
   if not result.isNil and result.isReady(req):
     inc result.pending
   else:
-    log "no sessions available for API: ", req.cookie.endpoint
+    if result.isNil:
+      log "no sessions available for API: ", req.cookie.endpoint
+    else:
+      log "no sessions available for API: ", req.endpoint(result), ", last tried: ", result.pretty
     raise noSessionsError()
 
 proc setLimited*(session: Session; req: ApiReq) =

src/consts.nim

@@ -7,109 +7,86 @@ const
   bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
   bearerToken2* = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
 
-  graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
-  graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery"
-  graphUserById* = "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery"
-  graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2"
-  graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2"
-  graphUserTweets* = "oRJs8SLCRNRbQzuZG93_oA/UserTweets"
-  graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
-  graphUserMedia* = "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
-  graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
-  graphTweet* = "b4pV7sWOe97RncwHcGESUA/ConversationTimeline"
-  graphTweetDetail* = "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail"
-  graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
-  graphTweetEditHistory* = "upS9teTSG45aljmP9oTuXA/TweetEditHistory"
-  graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
-  graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
-  graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
-  graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
-  graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
-  graphAboutAccount* = "zs_jFPFT78rBpXv9Z3U2YQ/AboutAccountQuery"
+  graphUser* = "IGgvgiOx4QZndDHuD3x9TQ/UserByScreenName"
+  graphUserV2* = "-ZzAG_Bckx16LMbEvHC3lg/UserResultByScreenNameQuery"
+  graphUserById* = "-DAaa9jPxPswYeI2fZ9rug/UserResultByIdQuery"
+  graphUserTweetsV2* = "LE3eTyeqhBh2g-fX85O2eQ/UserWithProfileTweetsQueryV2"
+  graphUserTweetsAndRepliesV2* = "AcYHjc_YAx-9_rKWdMsKvA/UserWithProfileTweetsAndRepliesQueryV2"
+  graphUserTweets* = "PNd0vlufvrcIwrAnBYKE9g/UserTweets"
+  graphUserTweetsAndReplies* = "EqtpEwt0CoQXmDfq5DKH0A/UserTweetsAndReplies"
+  graphUserMedia* = "g_rGPF0fLON-M9cyVjXuzA/UserMedia"
+  graphUserMediaV2* = "WK111rbR0vM0ZX4lyZCYjw/MediaTimelineV2"
+  graphTweet* = "OZMbEnEa96AN8Pq6HyTWdw/ConversationTimeline"
+  graphTweetDetail* = "6uCvnic3m5reVuehkvHa3w/TweetDetail"
+  graphTweetResult* = "xYOrBQoTlfKJJPsX76MZEw/TweetResultByIdQuery"
+  graphTweetEditHistory* = "MGElmrYILE8wUfI8GorUYA/TweetEditHistory"
+  graphSearchTimeline* = "-TFXKoMnMTKdEXcCn-eahw/SearchTimeline"
 
-  graphBroadcast* = "0nMmbMh-_JwwRRFNXkyH3Q/BroadcastQuery"
+  graphListById* = "t9AbdyHaJVfjL9jsODwgpQ/ListByRestId"
+  graphListBySlug* = "LDQpQ89B5ipR8izCKrWU0g/ListBySlug"
+  graphListMembers* = "EM7YRaM3gCnzDESmchA7RA/ListMembers"
+  graphListTweets* = "0QJtcuMzVywHGAWD6Dtjlw/ListTimeline"
+  graphAboutAccount* = "zUnx-DLN9dkwOkNhTLySjg/AboutAccountQuery"
+
+  graphCommunity* = "-ElI1vg3dYbttVMhBhGdLw/CommunityQuery"
+  graphCommunityTweets* = "Mvs5UOOEkpXVMDZtUcxR-Q/CommunityTweetsTimeline"
+  graphCommunityMedia* = "Bt9XYnY7D3OcmZE5lhdx-A/CommunityMediaTimeline"
+  graphCommunityMembers* = "WSbJGJjZaVasSj9bnqSZSA/membersSliceTimeline_Query"
+  graphCommunityModerators* = "GBMT3GOWy5dYsYC4XJfvow/moderatorsSliceTimeline_Query"
+  graphCommunityHashtags* = "40DyrMxfCknGuZwE-keW_Q/CommunityHashtagsTimeline"
+
+  graphTweetResultByRestId* = "qtXMy1p5Y62uCskc_NUPJw/TweetResultByRestId"
+  graphTweetResultsByRestIds* = "Sc9EUQTZNEH-wzegn-nHvQ/TweetResultsByRestIds"
+
+  graphBroadcast* = "FJLCzpXCLPM1jUZqmM7oEA/BroadcastQuery"
+  graphAudioSpace* = "rWRLsOhNJ2xjpI1tREYurQ/AudioSpaceById"
   restLiveStream* = "1.1/live_video_stream/status/"
 
+  graphFollowers* = "9jsVJ9l2uXUIKslHvJqIhw/Followers"
+  graphFollowing* = "OLm4oHZBfqWx8jbcEhWoFw/Following"
+
   gqlFeatures* = """{
-  "android_ad_formats_media_component_render_overlay_enabled": false,
-  "android_graphql_skip_api_media_color_palette": false,
-  "android_professional_link_spotlight_display_enabled": false,
-  "articles_api_enabled": false,
-  "articles_preview_enabled": true,
-  "blue_business_profile_image_shape_enabled": false,
-  "c9s_tweet_anatomy_moderator_badge_enabled": true,
-  "commerce_android_shop_module_enabled": false,
-  "communities_web_enable_tweet_community_results_fetch": true,
-  "creator_subscriptions_quote_tweet_preview_enabled": false,
-  "creator_subscriptions_subscription_count_enabled": false,
-  "creator_subscriptions_tweet_preview_api_enabled": true,
-  "freedom_of_speech_not_reach_fetch_enabled": true,
-  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
-  "grok_android_analyze_trend_fetch_enabled": false,
-  "grok_translations_community_note_auto_translation_is_enabled": false,
-  "grok_translations_community_note_translation_is_enabled": false,
-  "grok_translations_post_auto_translation_is_enabled": false,
-  "grok_translations_timeline_user_bio_auto_translation_is_enabled": false,
-  "hidden_profile_likes_enabled": false,
-  "highlights_tweets_tab_ui_enabled": false,
-  "immersive_video_status_linkable_timestamps": false,
-  "interactive_text_enabled": false,
-  "longform_notetweets_consumption_enabled": true,
-  "longform_notetweets_inline_media_enabled": true,
-  "longform_notetweets_richtext_consumption_enabled": true,
-  "longform_notetweets_rich_text_read_enabled": true,
-  "mobile_app_spotlight_module_enabled": false,
-  "payments_enabled": false,
-  "post_ctas_fetch_enabled": true,
-  "premium_content_api_read_enabled": false,
+  "rweb_video_screen_enabled": false,
+  "rweb_cashtags_enabled": true,
   "profile_label_improvements_pcf_label_in_post_enabled": true,
-  "profile_label_improvements_pcf_label_in_profile_enabled": false,
-  "responsive_web_edit_tweet_api_enabled": true,
-  "responsive_web_enhance_cards_enabled": false,
-  "responsive_web_graphql_exclude_directive_enabled": true,
-  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+  "responsive_web_profile_redirect_enabled": false,
+  "rweb_tipjar_consumption_enabled": false,
+  "verified_phone_label_enabled": false,
+  "creator_subscriptions_tweet_preview_api_enabled": true,
   "responsive_web_graphql_timeline_navigation_enabled": true,
-  "responsive_web_grok_analysis_button_from_backend": true,
+  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+  "premium_content_api_read_enabled": false,
+  "communities_web_enable_tweet_community_results_fetch": true,
+  "c9s_tweet_anatomy_moderator_badge_enabled": true,
+  "c9s_list_members_action_api_enabled": false,
+  "c9s_superc9s_indication_enabled": false,
   "responsive_web_grok_analyze_button_fetch_trends_enabled": false,
   "responsive_web_grok_analyze_post_followups_enabled": true,
+  "rweb_cashtags_composer_attachment_enabled": true,
+  "responsive_web_jetfuel_frame": true,
+  "responsive_web_grok_share_attachment_enabled": true,
   "responsive_web_grok_annotations_enabled": true,
-  "responsive_web_grok_community_note_auto_translation_is_enabled": false,
+  "articles_preview_enabled": true,
+  "responsive_web_edit_tweet_api_enabled": true,
+  "rweb_conversational_replies_downvote_enabled": false,
+  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
+  "view_counts_everywhere_api_enabled": true,
+  "longform_notetweets_consumption_enabled": true,
+  "responsive_web_twitter_article_tweet_consumption_enabled": true,
+  "content_disclosure_indicator_enabled": true,
+  "content_disclosure_ai_generated_indicator_enabled": true,
+  "responsive_web_grok_show_grok_translated_post": true,
+  "responsive_web_grok_analysis_button_from_backend": true,
+  "post_ctas_fetch_enabled": true,
+  "freedom_of_speech_not_reach_fetch_enabled": true,
+  "standardized_nudges_misinfo": true,
+  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
+  "longform_notetweets_rich_text_read_enabled": true,
+  "longform_notetweets_inline_media_enabled": false,
   "responsive_web_grok_image_annotation_enabled": true,
   "responsive_web_grok_imagine_annotation_enabled": true,
-  "responsive_web_grok_share_attachment_enabled": true,
-  "responsive_web_grok_show_grok_translated_post": false,
-  "responsive_web_jetfuel_frame": true,
-  "responsive_web_media_download_video_enabled": false,
-  "responsive_web_profile_redirect_enabled": false,
-  "responsive_web_text_conversations_enabled": false,
-  "responsive_web_twitter_article_notes_tab_enabled": false,
-  "responsive_web_twitter_article_tweet_consumption_enabled": true,
-  "responsive_web_twitter_blue_verified_badge_is_enabled": true,
-  "rweb_lists_timeline_redesign_enabled": true,
-  "rweb_tipjar_consumption_enabled": true,
-  "rweb_video_screen_enabled": false,
-  "rweb_video_timestamps_enabled": false,
-  "spaces_2022_h2_clipping": true,
-  "spaces_2022_h2_spaces_communities": true,
-  "standardized_nudges_misinfo": true,
-  "subscriptions_feature_can_gift_premium": false,
-  "subscriptions_verification_info_enabled": true,
-  "subscriptions_verification_info_is_identity_verified_enabled": false,
-  "subscriptions_verification_info_reason_enabled": true,
-  "subscriptions_verification_info_verified_since_enabled": true,
-  "super_follow_badge_privacy_enabled": false,
-  "super_follow_exclusive_tweet_notifications_enabled": false,
-  "super_follow_tweet_api_enabled": false,
-  "super_follow_user_api_enabled": false,
-  "tweet_awards_web_tipping_enabled": false,
-  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
-  "tweetypie_unmention_optimization_enabled": false,
-  "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
-  "unified_cards_destination_url_params_enabled": false,
-  "verified_phone_label_enabled": false,
-  "vibe_api_enabled": false,
-  "view_counts_everywhere_api_enabled": true,
-  "hidden_profile_subscriptions_enabled": false
+  "responsive_web_grok_community_note_auto_translation_is_enabled": true,
+  "responsive_web_enhance_cards_enabled": false
 }""".replace(" ", "").replace("\n", "")
 
   tweetVars* = """{
@@ -143,7 +120,7 @@ const
   restIdVars* = """{
   "rest_id": "$1", $2
   "count": $3
-}"""
+}""".replace(" ", "").replace("\n", "")
 
   userMediaVars* = """{
   "userId": "$1", $2
@@ -170,6 +147,50 @@ const
   "withVoice": true
 }""".replace(" ", "").replace("\n", "")
 
+  articleVars* = """{
+  "tweetId": "$1",
+  "includePromotedContent": false,
+  "withBirdwatchNotes": true,
+  "withVoice": true,
+  "withCommunity": true
+}""".replace(" ", "").replace("\n", "")
+
+  articleBatchVars* = """{
+  "tweetIds": $1,
+  "includePromotedContent": false,
+  "withBirdwatchNotes": true,
+  "withVoice": true,
+  "withCommunity": true
+}""".replace(" ", "").replace("\n", "")
+
+  articleFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withArticleSummaryText":true,"withArticleVoiceOver":true}"""
+
+  communityTweetsVars* = """{
+  "communityId": "$1", $2
+  "count": 20,
+  "displayLocation": "Community",
+  "rankingMode": "$3",
+  "withCommunity": true
+}""".replace(" ", "").replace("\n", "")
+
+  communityMediaVars* = """{
+  "communityId": "$1", $2
+  "count": 20,
+  "withCommunity": true
+}""".replace(" ", "").replace("\n", "")
+
+  communityMembersVars* = """{
+  "communityId": "$1",
+  "cursor": $2
+}""".replace(" ", "").replace("\n", "")
+
+  communityHashtagsVars* = """{
+  "communityId": "$1", $2
+  "count": 20,
+  "hashtags": ["$3"],
+  "withCommunity": true
+}""".replace(" ", "").replace("\n", "")
+
   userFieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}"""
-  userTweetsFieldToggles* = """{"withArticlePlainText":false}"""
+  userTweetsFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false}"""
   tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}"""

src/experimental/parser.nim

@@ -1,2 +1,2 @@
-import parser/[user, graphql]
-export user, graphql
+import parser/[user, graphql, article]
+export user, graphql, article

src/experimental/parser/article.nim

@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import std/[strutils, tables, times, options]
+import jsony
+import utils, graphql, ../types/article
+from ../../types import Article, ArticleParagraph, ArticleEntity, ArticleMedia,
+                        User, TweetStats
+
+proc parseGraphArticle*(json: string): Article =
+  if json.len == 0 or json[0] != '{':
+    return
+
+  var raw: GraphArticle
+  try:
+    raw = json.fromJson(GraphArticle)
+  except CatchableError:
+    return
+
+  let
+    tweet = raw.data.tweetResult.result
+    article = tweet.article.articleResults.result
+
+  if article.title.len == 0:
+    return
+
+  let publishedAt = article.metadata.firstPublishedAtSecs
+  var articleTime: DateTime
+  if publishedAt > 0:
+    articleTime = publishedAt.int64.fromUnix.utc
+  elif tweet.legacy.createdAt.len > 0:
+    articleTime = parseTwitterDate(tweet.legacy.createdAt)
+
+  result = Article(
+    title: article.title,
+    coverImage: getImageUrl(article.coverMedia.mediaInfo.originalImgUrl),
+    time: articleTime,
+    user: parseUserResult(tweet.core.userResults.result),
+  )
+
+  result.stats = TweetStats(
+    replies: tweet.legacy.replyCount,
+    retweets: tweet.legacy.retweetCount,
+    likes: tweet.legacy.favoriteCount,
+  )
+  if tweet.views.count.len > 0:
+    try: result.stats.views = parseInt(tweet.views.count)
+    except ValueError: discard
+
+  for blk in article.contentState.blocks:
+    result.paragraphs.add ArticleParagraph(
+      text: blk.text,
+      kind: blk.blockKind,
+      inlineStyles: blk.inlineStyleRanges,
+      entityRanges: blk.entityRanges,
+    )
+
+  for entry in article.contentState.entityMap:
+    let key = try: parseInt(entry.key) except ValueError: continue
+    var entity = ArticleEntity(kind: entry.value.entityKind)
+    case entity.kind
+    of "LINK": entity.url = entry.value.data.url
+    of "MEDIA":
+      for mi in entry.value.data.mediaItems:
+        entity.mediaIds.add mi.mediaId
+      entity.caption = entry.value.data.caption
+    of "TWEET": entity.tweetId = entry.value.data.tweetId
+    of "MARKDOWN": entity.markdown = entry.value.data.markdown
+    else: discard
+    result.entities[key] = entity
+
+  for me in article.mediaEntities:
+    let typeName = me.mediaInfo.typeName
+    var media = ArticleMedia(kind: typeName)
+    if me.mediaInfo.videoInfo.isSome:
+      let variants = me.mediaInfo.videoInfo.get.variants
+      case typeName
+      of "ApiGif":
+        if variants.len > 0:
+          media.url = variants[0].url
+      of "ApiVideo":
+        var bestBitrate = -1
+        for v in variants:
+          if v.bitrate > bestBitrate:
+            bestBitrate = v.bitrate
+            media.url = v.url
+      else: discard
+    elif typeName == "ApiImage":
+      media.url = getImageUrl(me.mediaInfo.originalImgUrl)
+    result.media[me.mediaId] = media

src/experimental/parser/graphql.nim

@@ -1,6 +1,6 @@
 import options, strutils
 import jsony
-import user, utils, ../types/[graphuser, graphlistmembers]
+import user, utils, ../types/[graphuser, graphlistmembers, graphfollowers]
 from ../../types import User, VerifiedType, Result, Query, QueryKind
 
 proc parseUserResult*(userResult: UserResult): User =
@@ -67,3 +67,25 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] =
         of TimelineTimelineCursor:
           if entry.content.cursorType == "Bottom":
             result.bottom = entry.content.value
+
+proc parseGraphFollowers*(json, cursor: string; kind: QueryKind): Result[User] =
+  result = Result[User](
+    beginning: cursor.len == 0,
+    query: Query(kind: kind)
+  )
+
+  if json.len == 0 or json[0] != '{':
+    return
+
+  let raw = json.fromJson(GraphFollowers)
+  for instruction in raw.data.user.result.timeline.timeline.instructions:
+    if instruction.kind == "TimelineAddEntries":
+      for entry in instruction.entries:
+        case entry.content.entryType
+        of TimelineTimelineItem:
+          let userResult = entry.content.itemContent.userResults.result
+          if userResult.restId.len > 0:
+            result.content.add parseUserResult(userResult)
+        of TimelineTimelineCursor:
+          if entry.content.cursorType == "Bottom":
+            result.bottom = entry.content.value

src/experimental/types/article.nim

@@ -0,0 +1,79 @@
+import std/options
+import graphuser
+from ../../types import ArticleStyle, ArticleEntityRange
+
+type
+  GraphArticle* = object
+    data*: tuple[tweetResult: tuple[result: TweetResultNode]]
+
+  TweetResultNode* = object
+    article*: tuple[articleResults: tuple[result: ArticleResultNode]]
+    legacy*: TweetLegacy
+    core*: tuple[userResults: UserData]
+    views*: tuple[count: string]
+
+  TweetLegacy* = object
+    createdAt*: string
+    replyCount*: int
+    retweetCount*: int
+    favoriteCount*: int
+
+  ArticleResultNode* = object
+    title*: string
+    coverMedia*: tuple[mediaInfo: MediaInfoNode]
+    contentState*: ContentState
+    metadata*: tuple[firstPublishedAtSecs: int]
+    mediaEntities*: seq[RawMediaEntity]
+
+  ContentState* = object
+    blocks*: seq[ContentBlock]
+    entityMap*: seq[EntityMapEntry]
+
+  ContentBlock* = object
+    text*: string
+    blockKind*: string
+    inlineStyleRanges*: seq[ArticleStyle]
+    entityRanges*: seq[ArticleEntityRange]
+
+  EntityMapEntry* = object
+    key*: string
+    value*: EntityMapValue
+
+  EntityMapValue* = object
+    entityKind*: string
+    data*: EntityDataNode
+
+  EntityDataNode* = object
+    url*: string
+    mediaItems*: seq[tuple[mediaId: string]]
+    tweetId*: string
+    markdown*: string
+    caption*: string
+
+  RawMediaEntity* = object
+    mediaId*: string
+    mediaInfo*: MediaInfoNode
+
+  MediaInfoNode* = object
+    typeName*: string
+    originalImgUrl*: string
+    videoInfo*: Option[VideoInfoNode]
+
+  VideoInfoNode* = object
+    variants*: seq[VideoVariant]
+
+  VideoVariant* = object
+    url*: string
+    bitrate*: int
+
+proc renameHook*(v: var ContentBlock; fieldName: var string) =
+  if fieldName == "type":
+    fieldName = "blockKind"
+
+proc renameHook*(v: var EntityMapValue; fieldName: var string) =
+  if fieldName == "type":
+    fieldName = "entityKind"
+
+proc renameHook*(v: var MediaInfoNode; fieldName: var string) =
+  if fieldName == "__typename":
+    fieldName = "typeName"

src/experimental/types/graphfollowers.nim

@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import graphlistmembers
+
+type
+  GraphFollowers* = object
+    data*: tuple[user: UserWrapper]
+
+  UserWrapper = object
+    result*: UserResultWrapper
+
+  UserResultWrapper = object
+    timeline*: tuple[timeline: graphlistmembers.Timeline]
+
+# Hook to normalize snake_case field from API to camelCase used by shared types
+proc renameHook*(v: var Content; fieldName: var string) =
+  if fieldName == "user_results":
+    fieldName = "userResults"

src/experimental/types/graphlistmembers.nim

@@ -7,10 +7,10 @@ type
   List = object
     membersTimeline*: tuple[timeline: Timeline]
 
-  Timeline = object
+  Timeline* = object
     instructions*: seq[Instruction]
 
-  Instruction = object
+  Instruction* = object
     kind*: string
     entries*: seq[tuple[content: Content]]
 
@@ -18,7 +18,7 @@ type
     TimelineTimelineItem
     TimelineTimelineCursor
 
-  Content = object
+  Content* = object
     case entryType*: ContentEntryType
     of TimelineTimelineItem:
       itemContent*: tuple[userResults: UserData]

src/formatters.nim

@@ -140,25 +140,30 @@ proc pageDesc*(user: User): string =
     "The latest tweets from " & user.fullname
 
 proc getJoinDate*(user: User): string =
+  if user.joinDate.year == 0: return ""
   user.joinDate.format("'Joined' MMMM YYYY")
 
 proc getJoinDateFull*(user: User): string =
+  if user.joinDate.year == 0: return ""
   user.joinDate.format("h:mm tt - d MMM YYYY")
 
 proc getTime*(tweet: Tweet): string =
+  if tweet.time.year == 0: return ""
   tweet.time.format("MMM d', 'YYYY' · 'h:mm tt' UTC'")
 
 proc getRfc822Time*(tweet: Tweet): string =
+  if tweet.time.year == 0: return ""
   tweet.time.format("ddd', 'dd MMM yyyy HH:mm:ss 'GMT'")
 
-proc getShortTime*(tweet: Tweet): string =
+proc getShortTime*(time: DateTime): string =
+  if time.year == 0: return ""
   let now = now()
-  let since = now - tweet.time
+  let since = now - time
 
-  if now.year != tweet.time.year:
-    result = tweet.time.format("d MMM yyyy")
+  if now.year != time.year:
+    result = time.format("d MMM yyyy")
   elif since.inDays >= 1:
-    result = tweet.time.format("MMM d")
+    result = time.format("MMM d")
   elif since.inHours >= 1:
     result = $since.inHours & "h"
   elif since.inMinutes >= 1:
@@ -168,6 +173,9 @@ proc getShortTime*(tweet: Tweet): string =
   else:
     result = "now"
 
+proc getShortTime*(tweet: Tweet): string =
+  getShortTime(tweet.time)
+
 proc getDuration*(ms: int): string =
   let
     sec = int(round(ms / 1000))

src/nitter.nim

@@ -2,15 +2,15 @@
 import asyncdispatch, strformat, logging
 from net import Port
 from htmlgen import a
-from os import getEnv
+from os import getEnv, normalizedPath
 
 import jester
 
 import types, config, prefs, formatters, redis_cache, http_pool, auth, apiutils
 import views/[general, about]
 import routes/[
-  preferences, timeline, status, media, search, rss, list, debug,
-  unsupported, embed, resolver, broadcast, router_utils]
+  preferences, timeline, status, media, search, rss, list, community, debug,
+  unsupported, embed, resolver, broadcast, space, article, router_utils]
 
 const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
 const issuesUrl = "https://github.com/zedeus/nitter/issues"
@@ -34,6 +34,10 @@ stdout.flushFile
 updateDefaultPrefs(fullCfg)
 setCacheTimes(cfg)
 setHmacKey(cfg.hmacKey)
+if cfg.hmacKey.len == 0 or cfg.hmacKey == "secretkey":
+  stderr.write "WARNING: insecure default 'hmacKey' in nitter.conf; " &
+    "set a unique random value to stop media URL signatures being forgeable.\n"
+  stderr.flushFile
 setProxyEncoding(cfg.base64Media)
 setMaxHttpConns(cfg.httpMaxConns)
 setHttpProxy(cfg.proxy, cfg.proxyAuth)
@@ -48,27 +52,35 @@ waitFor initRedisPool(cfg)
 stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n"
 stdout.flushFile
 
+createArticleRouter(cfg)
 createUnsupportedRouter(cfg)
 createResolverRouter(cfg)
 createPrefRouter(cfg)
 createTimelineRouter(cfg)
 createListRouter(cfg)
+createCommunityRouter(cfg)
 createStatusRouter(cfg)
 createSearchRouter(cfg)
 createMediaRouter(cfg)
 createEmbedRouter(cfg)
 createRssRouter(cfg)
 createBroadcastRouter(cfg)
+createSpaceRouter(cfg)
 createDebugRouter(cfg)
 
 settings:
   port = Port(cfg.port)
-  staticDir = cfg.staticDir
+  staticDir = normalizedPath(cfg.staticDir)
   bindAddr = cfg.address
   reusePort = true
+  maxBody = 64 * 1024
 
 routes:
   before:
+    # Reject malformed paths
+    if request.path.len == 0 or request.path[0] != '/':
+      halt Http400
+
     # skip all file URLs
     cond "." notin request.path
     applyUrlPrefs()
@@ -113,15 +125,18 @@ routes:
     resp Http429, showError(
       &"Instance has no auth tokens, or is fully rate limited.<br>Use {link} or try again later.", cfg)
 
+  extend articleRoute, ""
   extend rss, ""
   extend status, ""
   extend search, ""
   extend timeline, ""
   extend media, ""
   extend list, ""
+  extend community, ""
   extend preferences, ""
   extend resolver, ""
   extend embed, ""
   extend broadcastRoute, ""
+  extend spaceRoute, ""
   extend debug, ""
   extend unsupported, ""

src/parser.nim

@@ -1,10 +1,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, options, times, math, tables
+import strutils, options, times, math, tables, uri
 import packedjson, packedjson/deserialiser
 import types, parserutils, utils
 import experimental/parser/unifiedcard
 
-proc parseGraphTweet(js: JsonNode): Tweet
+proc parseGraphTweet*(js: JsonNode): Tweet
 
 proc parseVerifiedType(s: string; current: VerifiedType): VerifiedType =
   try: parseEnum[VerifiedType](s)
@@ -46,10 +46,10 @@ proc parseUser(js: JsonNode; id=""): User =
 proc parseGraphUser(js: JsonNode): User =
   var user = js{"user_result", "result"}
   if user.isNull:
-    user = ? js{"user_results", "result"}
+    user = js{"user_results", "result"}
 
   if user.isNull:
-    if js{"core"}.notNull and js{"legacy"}.notNull:
+    if js{"core"}.notNull:
       user = js
     else:
       return
@@ -61,6 +61,7 @@ proc parseGraphUser(js: JsonNode): User =
 
   # fallback to support UserMedia/recent GraphQL updates
   if result.username.len == 0:
+    result.id = user{"rest_id"}.getStr
     result.username = user{"core", "screen_name"}.getStr
     result.fullname = user{"core", "name"}.getStr
     result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
@@ -130,6 +131,81 @@ proc parseBroadcastInfo*(js: JsonNode): Broadcast =
     user: parseGraphUser(bc)
   )
 
+proc parseSpaceParticipant(js: JsonNode): SpaceParticipant =
+  result = SpaceParticipant(
+    userId: js{"user_results", "rest_id"}.getStr,
+    username: js{"twitter_screen_name"}.getStr,
+    displayName: js{"display_name"}.getStr,
+    avatarUrl: js{"avatar_url"}.getStr,
+    isVerified: js{"is_verified"}.getBool or
+                js{"user_results", "result", "is_blue_verified"}.getBool
+  )
+
+proc parseAudioSpace*(js: JsonNode): AudioSpace =
+  let space = ? js{"data", "audioSpace"}
+  let meta = space{"metadata"}
+
+  result = AudioSpace(
+    id: meta{"rest_id"}.getStr,
+    title: meta{"title"}.getStr,
+    state: meta{"state"}.getStr.toUpperAscii,
+    mediaKey: meta{"media_key"}.getStr,
+    totalLiveListeners: meta{"total_live_listeners"}.getInt,
+    totalReplayWatched: meta{"total_replay_watched"}.getInt,
+    availableForReplay: meta{"is_space_available_for_replay"}.getBool
+  )
+
+  let startedAt = meta{"started_at"}.getInt(0)
+  if startedAt > 0:
+    result.startTime = fromUnix(startedAt div 1000).utc()
+
+  let endedAtStr = meta{"ended_at"}.getStr
+  if endedAtStr.len > 0:
+    try:
+      let endedAt = parseBiggestInt(endedAtStr)
+      if endedAt > 0:
+        result.endTime = fromUnix(endedAt div 1000).utc()
+    except ValueError:
+      discard
+
+  result.creator = parseGraphUser(meta{"creator_results", "result"})
+
+  for admin in space{"participants", "admins"}:
+    result.admins.add parseSpaceParticipant(admin)
+
+  for speaker in space{"participants", "speakers"}:
+    result.speakers.add parseSpaceParticipant(speaker)
+
+proc parseGraphCommunity*(js: JsonNode): Community =
+  if js.isNull: return
+  let c = ? js{"data", "communityResults", "result"}
+
+  result = Community(
+    id: c{"rest_id"}.getStr(c{"id_str"}.getStr),
+    name: c{"name"}.getStr,
+    description: c{"description"}.getStr,
+    memberCount: c{"member_count"}.getInt,
+    joinPolicy: c{"join_policy"}.getStr,
+    category: c{"primary_community_topic", "topic_name"}.getStr,
+    banner: c{"custom_banner_media", "media_info", "original_img_url"}.getImageStr,
+    creator: parseGraphUser(c{"creator_results", "result"}),
+  )
+
+  let createdMs = c{"created_at"}.getInt(0)
+  if createdMs > 0:
+    result.createdAt = fromUnix(createdMs div 1000).utc()
+
+  for rule in c{"rules"}:
+    result.rules.add CommunityRule(
+      name: rule{"name"}.getStr,
+      description: rule{"description"}.getStr
+    )
+
+  for item in c{"trending_hashtags_slice", "items"}:
+    let tag = item{"hashtag"}.getStr
+    if tag.len > 0:
+      result.hashtags.add tag
+
 proc parseGraphList*(js: JsonNode): List =
   if js.isNull: return
 
@@ -228,6 +304,10 @@ proc parseLegacyMediaEntities(js: JsonNode; result: var Tweet) =
             result.attribution = some(parseUser(user))
           else:
             result.attribution = some(parseGraphUser(user))
+          # Set attribution link from expanded_url (strip /video/N suffix)
+          let expanded = m{"expanded_url"}.getStr
+          if expanded.len > 0:
+            result.attributionLink = expanded.parseUri.path.replace("/video/1", "")
       of "animated_gif":
         result.media.addMedia(Gif(
           url: m{"video_info", "variants"}[0]{"url"}.getImageStr,
@@ -236,11 +316,6 @@ proc parseLegacyMediaEntities(js: JsonNode; result: var Tweet) =
         ))
       else: discard
 
-      with url, m{"url"}:
-        if result.text.endsWith(url.getStr):
-          result.text.removeSuffix(url.getStr)
-          result.text = result.text.strip()
-
 proc parseMediaEntities(js: JsonNode; result: var Tweet) =
   with mediaEntities, js{"media_entities"}:
     var parsedMedia: MediaEntities
@@ -261,6 +336,18 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) =
             durationMs: mediaInfo{"duration_millis"}.getInt,
             variants: parseVideoVariants(mediaInfo{"variants"})
           ))
+
+          # Parse source user for video attribution
+          with sourceUser, mediaEntity{"source_user_results", "result"}:
+            if result.attribution.isNone:
+              let expanded = mediaEntity{"expanded_url"}.getStr
+              if expanded.len > 0:
+                result.attributionLink = expanded.parseUri.path.replace("/video/1", "")
+              result.attribution = some(User(
+                id: sourceUser{"rest_id"}.getStr,
+                fullname: sourceUser{"core", "name"}.getStr,
+                userPic: sourceUser{"avatar", "image_url"}.getImageStr.replace("_normal", "")
+              ))
         of "ApiGif":
           parsedMedia.addMedia(Gif(
             url: mediaInfo{"variants"}[0]{"url"}.getImageStr,
@@ -269,23 +356,9 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) =
           ))
         else: discard
 
-      if "expanded_url" in mediaEntity:
-        let expandedUrl = js.getExpandedUrl
-        if result.text.endsWith(expandedUrl):
-          result.text.removeSuffix(expandedUrl)
-          result.text = result.text.strip()
-
     if mediaEntities.len > 0 and parsedMedia.len == mediaEntities.len:
       result.media = parsedMedia
 
-  # Remove media URLs from text
-  with mediaList, js{"legacy", "entities", "media"}:
-    for url in mediaList:
-      let expandedUrl = url.getExpandedUrl
-      if result.text.endsWith(expandedUrl):
-        result.text.removeSuffix(expandedUrl)
-        result.text = result.text.strip()
-
 proc parsePromoVideo(js: JsonNode): Video =
   result = Video(
     thumb: js{"player_image_large"}.getImageVal,
@@ -362,7 +435,13 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
     result.url = vals{"player_url"}.getStrVal
     if "youtube.com" in result.url:
       result.url = result.url.replace("/embed/", "/watch?v=")
-  of audiospace, unknown:
+  of audiospace:
+    let spaceId = vals{"id"}.getStrVal
+    if spaceId.len > 0:
+      result.url = "/i/spaces/" & spaceId
+      result.title = "Twitter Space"
+      result.text = "Click to view Space"
+  of unknown:
     result.title = "This card type is not supported."
   else: discard
 
@@ -428,13 +507,13 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
   # graphql
   with rt, js{"retweeted_status_result", "result"}:
     # needed due to weird edgecase where the actual tweet data isn't included
-    if "legacy" in rt:
+    if "legacy" in rt or "rest_id" in rt:
       result.retweet = some parseGraphTweet(rt)
       return
 
   with reposts, js{"repostedStatusResults"}:
     with rt, reposts{"result"}:
-      if "legacy" in rt:
+      if "legacy" in rt or "rest_id" in rt:
         result.retweet = some parseGraphTweet(rt)
         return
 
@@ -449,7 +528,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
       result.poll = some parsePoll(jsCard)
     elif name == "amplify":
       result.media.addMedia(parsePromoVideo(jsCard{"binding_values"}))
-    else:
+    elif name.len > 0 and jsCard{"binding_values"}.notNull:
       result.card = some parseCard(jsCard, js{"entities", "urls"})
 
   result.expandTweetEntities(js)
@@ -469,7 +548,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
       result.text.removeSuffix(" Learn more.")
       result.available = false
 
-proc parseGraphTweet(js: JsonNode): Tweet =
+proc parseGraphTweet*(js: JsonNode): Tweet =
   if js.kind == JNull:
     return Tweet()
 
@@ -506,7 +585,7 @@ proc parseGraphTweet(js: JsonNode): Tweet =
           "binding_values": %bindingObj
         }
 
-  var replyId = 0
+  var replyId: int64 = 0
   with restId, js{"reply_to_results", "rest_id"}:
     replyId = restId.getId
 
@@ -537,14 +616,40 @@ proc parseGraphTweet(js: JsonNode): Tweet =
         result.poll = some parsePoll(jsCard)
       elif name == "amplify":
         result.media.addMedia(parsePromoVideo(jsCard{"binding_values"}))
-      else:
+      elif name.len > 0 and jsCard{"binding_values"}.notNull:
         result.card = some parseCard(jsCard, js{"url_entities"})
 
-    result.expandTweetEntitiesV2(js)
+    parseMediaEntities(js, result)
+    if result.attribution.isNone:
+      parseLegacyMediaEntities(js{"legacy"}, result)
+
+    let hasArticle = js{"article", "article_results", "result", "title"}.getStr.len > 0
+    result.expandTweetEntitiesV2(js, hasArticle)
+
+    # Strip video source URL from text (for videos from other tweets)
+    with mediaEntities, js{"media_entities"}:
+      for m in mediaEntities:
+        if "source_status_id_str" in m:
+          let mediaUrl = m{"url"}.getStr
+          if mediaUrl.len > 0:
+            let idx = result.text.rfind(mediaUrl)
+            if idx >= 0:
+              result.text = result.text[0 ..< idx].strip()
+            break
   else:
     result = parseTweet(js{"legacy"}, jsCard, replyId)
     result.id = js{"rest_id"}.getId
 
+  with artNode, js{"article", "article_results", "result"}:
+    let artTitle = artNode{"title"}.getStr
+    if artTitle.len > 0:
+      result.articlePreview = some ArticlePreview(
+        title: artTitle,
+        previewText: artNode{"preview_text"}.getStr,
+        coverImage: artNode{"cover_media_results", "result", "media_info", "original_img_url"}.getImageStr,
+        tweetId: result.id
+      )
+
   result.user = parseGraphUser(js{"core"})
 
   if result.reply.len == 0:
@@ -559,6 +664,20 @@ proc parseGraphTweet(js: JsonNode): Tweet =
 
   parseMediaEntities(js, result)
 
+  # Hide card if it's redundant with attribution (same video shown via embed)
+  if result.attribution.isSome and result.card.isSome:
+    let cardUri = get(result.card).url.parseUri
+    if cardUri.isTwitterUrl:
+      let cardPath = cardUri.path.replace("/video/1", "")
+      if cardPath.len > 0 and cardPath == result.attributionLink:
+        get(result.card).kind = hidden
+
+  # Handle retweets - check both legacy and top-level paths
+  with reposts, js{"legacy", "repostedStatusResults"}:
+    with rt, reposts{"result"}:
+      if "legacy" in rt or "rest_id" in rt:
+        result.retweet = some parseGraphTweet(rt)
+
   with quoted, js{"quoted_status_result", "result"}:
     result.quote = some(parseGraphTweet(quoted))
 
@@ -600,6 +719,16 @@ proc parseGraphTweetResult*(js: JsonNode): Tweet =
   with tweet, js{"data", "tweet_result", "result"}:
     result = parseGraphTweet(tweet)
 
+proc parseGraphTweetResults*(js: JsonNode): seq[Tweet] =
+  let results = js{"data", "tweetResult"}
+  if results.kind != JArray: return
+  for item in results:
+    let tweet = item{"result"}
+    if tweet.isNull: continue
+    let t = parseGraphTweet(tweet)
+    if t != nil:
+      result.add t
+
 proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
   result = Conversation(replies: Result[Chain](beginning: true))
 
@@ -807,3 +936,49 @@ proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] =
     elif typ == "TimelineReplaceEntry":
       if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"):
         result.bottom = instruction{"entry", "content", "value"}.getStr
+
+proc parseGraphCommunityTimeline*(js: JsonNode; after=""): Timeline =
+  result = Timeline(beginning: after.len == 0)
+
+  let communityResult = js{"data", "communityResults", "result"}
+  let instructions = ? select(
+    communityResult{"ranked_community_timeline", "timeline", "instructions"},
+    communityResult{"community_media_timeline", "timeline", "instructions"},
+    communityResult{"community_filtered_timeline", "timeline", "instructions"}
+  )
+  if instructions.len == 0:
+    return
+
+  for i in instructions:
+    if i{"entries"}.notNull:
+      for e in i{"entries"}:
+        let entryId = e.getEntryId
+        if entryId.startsWith("tweet") or entryId.startsWith("profile-grid") or
+           entryId.startsWith("communities-grid"):
+          for tweet in extractTweetsFromEntry(e):
+            result.content.add tweet
+        elif entryId.startsWith("cursor-bottom"):
+          result.bottom = e{"content", "value"}.getStr
+
+    if after.len == 0 and i.getTypeName == "TimelinePinEntry":
+      var tweets = extractTweetsFromEntry(i{"entry"})
+      for tweet in tweets.mitems:
+        tweet.pinned = true
+      if tweets.len > 0:
+        result.content.insert(tweets, 0)
+
+proc parseGraphCommunityMembers*(js: JsonNode; after=""): Result[User] =
+  result = Result[User](beginning: after.len == 0)
+
+  let r = js{"data", "communityResults", "result"}
+  let slice = if not r{"members_slice"}.isNull: r{"members_slice"}
+              else: r{"moderators_slice"}
+  for item in slice{"items_results"}:
+    let user = parseGraphUser(item{"result"})
+    if user.username.len > 0:
+      result.content.add user
+
+  let cursor = slice{"slice_info", "next_cursor"}.getStr
+  if cursor.len > 0:
+    result.bottom = cursor
+

src/parserutils.nim

@@ -185,12 +185,16 @@ proc extractSlice(js: JsonNode): Slice[int] =
   result = js["indices"][0].getInt ..< js["indices"][1].getInt
 
 proc extractUrls(result: var seq[ReplaceSlice]; js: JsonNode;
-                 textLen: int; hideTwitter = false) =
+                 textLen: int; hideTwitter = false;
+                 hideArticle = false) =
   let
     url = js.getExpandedUrl
     slice = js.extractSlice
 
-  if hideTwitter and slice.b.succ >= textLen and url.isTwitterUrl:
+  if hideArticle and url.isTwitterUrl and "/article/" in url:
+    if slice.a < textLen:
+      result.add ReplaceSlice(kind: rkRemove, slice: slice)
+  elif hideTwitter and slice.b.succ >= textLen and url.isTwitterUrl:
     if slice.a < textLen:
       result.add ReplaceSlice(kind: rkRemove, slice: slice)
   else:
@@ -202,19 +206,32 @@ proc extractHashtags(result: var seq[ReplaceSlice]; js: JsonNode) =
 
 proc replacedWith(runes: seq[Rune]; repls: openArray[ReplaceSlice];
                   textSlice: Slice[int]): string =
+  let
+    runeLen = runes.len
+    safeStart = max(0, textSlice.a)
+    safeEnd = min(runeLen, textSlice.b)
+
+  var validRepls: seq[ReplaceSlice]
+  for rep in repls:
+    if rep.slice.a >= 0 and rep.slice.b >= 0 and rep.slice.b < runeLen and rep.slice.a <= rep.slice.b:
+      validRepls.add rep
+
   template extractLowerBound(i: int; idx): int =
-    if i > 0: repls[idx].slice.b.succ else: textSlice.a
+    if i > 0: min(validRepls[idx].slice.b.succ, runeLen) else: safeStart
 
   result = newStringOfCap(runes.len)
 
-  for i, rep in repls:
-    result.add $runes[extractLowerBound(i, i - 1) ..< rep.slice.a]
+  for i, rep in validRepls:
+    let lower = extractLowerBound(i, i - 1)
+    if lower < rep.slice.a:
+      result.add $runes[lower ..< rep.slice.a]
     case rep.kind
     of rkHashtag:
-      let
-        name = $runes[rep.slice.a.succ .. rep.slice.b]
-        symbol = $runes[rep.slice.a]
-      result.add a(symbol & name, href = "/search?f=tweets&q=%23" & name)
+      if rep.slice.a.succ <= rep.slice.b:
+        let
+          name = $runes[rep.slice.a.succ .. rep.slice.b]
+          symbol = $runes[rep.slice.a]
+        result.add a(symbol & name, href = "/search?f=tweets&q=%23" & name)
     of rkMention:
       result.add a($runes[rep.slice], href = rep.url, title = rep.display)
     of rkUrl:
@@ -222,8 +239,8 @@ proc replacedWith(runes: seq[Rune]; repls: openArray[ReplaceSlice];
     of rkRemove:
       discard
 
-  let rest = extractLowerBound(repls.len, ^1) ..< textSlice.b
-  if rest.a <= rest.b:
+  let rest = extractLowerBound(validRepls.len, ^1) ..< safeEnd
+  if rest.a >= 0 and rest.a <= rest.b and rest.b <= runeLen:
     result.add $runes[rest]
 
 proc deduplicate(s: var seq[ReplaceSlice]) =
@@ -326,10 +343,11 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
       replyTo = reply.getStr
       tweet.reply.add replyTo
 
-  tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote or hasJobCard)
+  tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo,
+                           hasQuote or hasJobCard)
 
 proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: Slice[int];
-                          hasRedundantLink=false) =
+                          hasRedundantLink=false; hasArticle=false) =
   let hasCard = tweet.card.isSome
 
   var replacements = newSeq[ReplaceSlice]()
@@ -340,7 +358,8 @@ proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: S
       if urlStr.len == 0 or urlStr notin text:
         continue
 
-      replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink)
+      replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink,
+                               hideArticle = hasArticle)
 
       if hasCard and u{"url"}.getStr == get(tweet.card).url:
         get(tweet.card).url = u.getExpandedUrl
@@ -371,22 +390,26 @@ proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: S
 
   tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false)
 
-proc expandTweetEntitiesV2*(tweet: Tweet; js: JsonNode) =
+proc expandTweetEntitiesV2*(tweet: Tweet; js: JsonNode; hasArticle=false) =
   let
     textRange = js{"details", "display_text_range"}
     textSlice = textRange{0}.getInt .. textRange{1}.getInt
     hasQuote = "quoted_tweet_results" in js
     hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails
+    hasAttribution = tweet.attribution.isSome
 
-  tweet.expandTextEntitiesV2(js, tweet.text, textSlice, hasQuote or hasJobCard)
+  tweet.expandTextEntitiesV2(js, tweet.text, textSlice,
+                             hasQuote or hasJobCard or hasAttribution,
+                             hasArticle)
 
 proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) =
   let
     entities = ? js{"entity_set"}
     text = js{"text"}.getStr.multiReplace(("<", unicodeOpen), (">", unicodeClose))
     textSlice = 0..text.runeLen
+    hasAttribution = tweet.attribution.isSome
 
-  tweet.expandTextEntities(entities, text, textSlice)
+  tweet.expandTextEntities(entities, text, textSlice, hasRedundantLink=hasAttribution)
 
   tweet.text = tweet.text.multiReplace((unicodeOpen, xmlOpen), (unicodeClose, xmlClose))
 

src/redis_cache.nim

@@ -144,9 +144,10 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} =
   else:
     let user = await getGraphUserById(userId)
     result = user.username
-    await setEx(key, baseCacheTime, result)
-    if result.len > 0 and user.id.len > 0:
-      await all(cacheUserId(result, user.id), cache(user))
+    if result.len > 0:
+      await setEx(key, baseCacheTime, result)
+      if user.id.len > 0:
+        await all(cacheUserId(result, user.id), cache(user))
 
 # proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
 #   if id == 0: return
@@ -172,6 +173,21 @@ proc getCachedBroadcast*(id: string): Future[Broadcast] {.async.} =
     await cache(result)
   result.m3u8Url = await fetchBroadcastStream(result.mediaKey)
 
+proc cache*(data: AudioSpace) {.async.} =
+  if data.id.len == 0: return
+  let ttl = if data.state == "RUNNING": baseCacheTime div 6 else: baseCacheTime
+  await setEx("sp:" & data.id, ttl, compress(toFlatty(data)))
+
+proc getCachedAudioSpace*(id: string): Future[AudioSpace] {.async.} =
+  if id.len == 0: return
+  let cached = await get("sp:" & id)
+  if cached != redisNil:
+    cached.deserialize(AudioSpace)
+  else:
+    result = await getAudioSpace(id)
+    await cache(result)
+  result.m3u8Url = await fetchBroadcastStream(result.mediaKey)
+
 proc cache*(data: AccountInfo; name: string) {.async.} =
   await setEx("ai:" & toLower(name), baseCacheTime * 24, compress(toFlatty(data)))
 
@@ -194,6 +210,29 @@ proc getCachedPhotoRail*(id: string): Future[PhotoRail] {.async.} =
     result = await getPhotoRail(id)
     await cache(result, id)
 
+proc cache*(data: Community) {.async.} =
+  if data.id.len == 0: return
+  await setEx("cm:" & data.id, listCacheTime, compress(toFlatty(data)))
+
+proc getCachedCommunity*(id: string): Future[Community] {.async.} =
+  if id.len == 0: return
+  let cached = await get("cm:" & id)
+  if cached != redisNil:
+    cached.deserialize(Community)
+  else:
+    result = await getGraphCommunity(id)
+    await cache(result)
+
+proc getCachedCommunityModerators*(id: string): Future[seq[User]] {.async.} =
+  if id.len == 0: return
+  let cached = await get("cmm:" & id)
+  if cached != redisNil:
+    cached.deserialize(seq[User])
+  else:
+    let mods = await getGraphCommunityModerators(id)
+    result = mods.content
+    await setEx("cmm:" & id, listCacheTime, compress(toFlatty(result)))
+
 proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} =
   let list = if id.len == 0: redisNil
              else: await get("l:" & id)

src/routes/article.nim

@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import asyncdispatch, tables, strutils
+import jester, karax/vdom
+import ".."/[types, api]
+import ../views/[article, general]
+import router_utils
+
+export api, article, vdom, general, router_utils
+
+proc createArticleRouter*(cfg: Config) =
+  router articleRoute:
+    get "/i/article/@id":
+      cond @"id".allCharsInSet(Digits)
+
+      let article = await getGraphArticle(@"id")
+      if article == nil:
+        resp Http404, showError("Article not found", cfg)
+
+      var tweetIds: seq[string]
+      for e in article.entities.values:
+        if e.kind == "TWEET":
+          tweetIds.add e.tweetId
+
+      var tweets = initTable[int64, Tweet]()
+      if tweetIds.len > 0:
+        try:
+          for t in await getGraphTweetResults(tweetIds):
+            tweets[t.id] = t
+        except CatchableError:
+          discard
+
+      let
+        prefs = requestPrefs()
+        path = getPath()
+        html = renderArticle(article, tweets, path, prefs, @"id")
+        twitterUrl = "https://x.com/" & article.user.username & "/article/" & @"id"
+      resp renderMain(html, request, cfg, prefs, titleText=article.title,
+                      twitterLink=twitterUrl)
+
+    get "/@name/article/@id/?":
+      cond '.' notin @"name"
+      cond @"id".allCharsInSet(Digits)
+      redirect("/i/article/" & @"id")
+
+    get "/@name/status/@id/article":
+      cond '.' notin @"name"
+      cond @"id".allCharsInSet(Digits)
+      redirect("/i/article/" & @"id")

src/routes/community.nim

@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import strformat
+
+import jester
+
+import router_utils
+import ".."/[types, redis_cache, api]
+import ../views/[general, timeline, community]
+
+export community
+
+template respCommunity*(cmty: Community; title: string; nav, vnode: typed) =
+  if cmty.id.len == 0 or cmty.name.len == 0:
+    resp Http404, showError(&"""Community "{@"id"}" not found""", cfg)
+
+  let html = renderCommunity(vnode, nav, cmty)
+  resp renderMain(html, request, cfg, prefs, titleText=title, banner=cmty.banner)
+
+proc createCommunityRouter*(cfg: Config) =
+  router community:
+    get "/i/communities/@id/?":
+      cond '.' notin @"id"
+      let
+        prefs = requestPrefs()
+        cmty = await getCachedCommunity(@"id")
+        tl = await getGraphCommunityTweets(cmty.id, "Relevance", getCursor())
+      respCommunity(cmty, cmty.name,
+                     renderCommunityTabs(QueryKind.posts, cmty),
+                     renderTimelineTweets(tl, prefs, request.path))
+
+    get "/i/communities/@id/latest":
+      cond '.' notin @"id"
+      let
+        prefs = requestPrefs()
+        cmty = await getCachedCommunity(@"id")
+        tl = await getGraphCommunityTweets(cmty.id, "Recency", getCursor())
+      respCommunity(cmty, cmty.name & " - Latest",
+                     renderCommunityTabs(QueryKind.replies, cmty),
+                     renderTimelineTweets(tl, prefs, request.path))
+
+    get "/i/communities/@id/media":
+      cond '.' notin @"id"
+      let
+        prefs = requestPrefs()
+        cmty = await getCachedCommunity(@"id")
+        tl = await getGraphCommunityMedia(cmty.id, getCursor())
+      respCommunity(cmty, cmty.name & " - Media",
+                     renderCommunityTabs(QueryKind.media, cmty),
+                     renderTimelineTweets(tl, prefs, request.path))
+
+    get "/i/communities/@id/about":
+      cond '.' notin @"id"
+      let
+        prefs = requestPrefs()
+        cmty = await getCachedCommunity(@"id")
+        mods = await getCachedCommunityModerators(cmty.id)
+      respCommunity(cmty, cmty.name & " - About",
+                     renderCommunityTabs(QueryKind.userList, cmty),
+                     renderCommunityAbout(cmty, mods))
+
+    get "/i/communities/@id/members":
+      cond '.' notin @"id"
+      let
+        prefs = requestPrefs()
+        cmty = await getCachedCommunity(@"id")
+        members = await getGraphCommunityMembers(cmty.id, getCursor())
+      respCommunity(cmty, cmty.name & " - Members",
+                     renderMemberTabs(cmty, false),
+                     renderTimelineUsers(members, prefs, request.path))
+
+    get "/i/communities/@id/moderators":
+      cond '.' notin @"id"
+      let
+        prefs = requestPrefs()
+        cmty = await getCachedCommunity(@"id")
+        mods = await getCachedCommunityModerators(cmty.id)
+      respCommunity(cmty, cmty.name & " - Moderators",
+                     renderMemberTabs(cmty, true),
+                     renderTimelineUsers(Result[User](content: mods), prefs, request.path))
+
+    get "/i/communities/@id/hashtag/@tag":
+      cond '.' notin @"id"
+      let
+        prefs = requestPrefs()
+        cmty = await getCachedCommunity(@"id")
+        tl = await getGraphCommunityHashtags(cmty.id, @"tag", getCursor())
+      respCommunity(cmty, cmty.name & " - #" & @"tag",
+                     renderHashtagHeader(cmty, @"tag"),
+                     renderTimelineTweets(tl, prefs, request.path))

src/routes/media.nim

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import uri, strutils, httpclient, os, hashes, base64, re
 import asynchttpserver, asyncstreams, asyncfile, asyncnet
+import asyncdispatch
 
 import jester
 
@@ -15,7 +16,9 @@ const
   maxAge* = "max-age=604800"
 
 proc safeFetch*(url: string): Future[string] {.async.} =
-  let client = newAsyncHttpClient()
+  # maxRedirects=0: the caller already validated the host, so never follow a
+  # redirect off the allowlisted host (would re-open the #1411 SSRF).
+  let client = newAsyncHttpClient(maxRedirects = 0)
   try: result = await client.getContent(url)
   except: discard
   finally: client.close()
@@ -30,47 +33,79 @@ template respond*(req: asynchttpserver.Request; headers) =
 
 proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
   result = Http200
-  let
-    request = req.getNativeReq()
-    client = newAsyncHttpClient()
+  let request = req.getNativeReq()
+  var fetchUrl = url
 
-  try:
-    let res = await client.get(url)
-    if res.status != "200 OK":
-      if res.status != "404 Not Found":
-        echo "[media] Proxying failed, status: $1, url: $2" % [res.status, url]
-      return Http404
-
-    let hashed = $hash(url)
-    if request.headers.getOrDefault("If-None-Match") == hashed:
-      return Http304
-
-    let contentLength =
-      if res.headers.hasKey("content-length"):
-        res.headers["content-length", 0]
+  for attempt in 0 .. 2:
+    let client = newAsyncHttpClient(maxRedirects = 0)
+    var shouldRetry = false
+    try:
+      let resFut = client.get(fetchUrl)
+      let completed = await withTimeout(resFut, 5000)
+      if not completed:
+        if attempt < 2:
+          echo "[media] Retry $1/2, timeout after 5s, url: $2" % [$(attempt + 1), fetchUrl]
+          shouldRetry = true
+        else:
+          echo "[media] Proxying timeout after 5s, url: $1" % [fetchUrl]
+          return Http504
       else:
-        ""
+        let res = resFut.read()
+        if res.status != "200 OK":
+          if res.status == "404 Not Found":
+            return Http404
+          if res.status.startsWith("30") and res.headers.hasKey("location"):
+            let location = res.headers["location", 0]
+            if isTwitterUrl(location):
+              fetchUrl = location
+              shouldRetry = true
+              continue
+            else:
+              return Http403
+          if attempt < 2:
+            echo "[media] Retry $1/2, status: $2, url: $3" % [$(attempt + 1), res.status, fetchUrl]
+            shouldRetry = true
+          else:
+            echo "[media] Proxying failed, status: $1, url: $2" % [res.status, fetchUrl]
+            return Http404
+        else:
+          let hashed = $hash(url)
+          if request.headers.getOrDefault("If-None-Match") == hashed:
+            return Http304
 
-    let headers = newHttpHeaders({
-      "content-type": res.headers["content-type", 0],
-      "content-length": contentLength,
-      "cache-control": maxAge,
-      "etag": hashed
-    })
+          let contentLength =
+            if res.headers.hasKey("content-length"):
+              res.headers["content-length", 0]
+            else:
+              ""
 
-    respond(request, headers)
+          let headers = newHttpHeaders({
+            "content-type": res.headers["content-type", 0],
+            "content-length": contentLength,
+            "cache-control": maxAge,
+            "etag": hashed
+          })
 
-    var (hasValue, data) = (true, "")
-    while hasValue:
-      (hasValue, data) = await res.bodyStream.read()
-      if hasValue:
-        await request.client.send(data)
-    data.setLen 0
-  except HttpRequestError, ProtocolError, OSError:
-    echo "[media] Proxying exception, error: $1, url: $2" % [getCurrentExceptionMsg(), url]
-    result = Http404
-  finally:
-    client.close()
+          respond(request, headers)
+
+          var (hasValue, data) = (true, "")
+          while hasValue:
+            (hasValue, data) = await res.bodyStream.read()
+            if hasValue:
+              await request.client.send(data)
+          data.setLen 0
+          return Http200
+    except CatchableError:
+      if attempt < 2:
+        echo "[media] Retry $1/2, error: $2, url: $3" % [$(attempt + 1), getCurrentExceptionMsg(), fetchUrl]
+        shouldRetry = true
+      else:
+        echo "[media] Proxying exception, error: $1, url: $2" % [getCurrentExceptionMsg(), fetchUrl]
+        result = Http404
+    finally:
+      client.close()
+    if not shouldRetry:
+      break
 
 template check*(code): untyped =
   if code != Http200:
@@ -122,12 +157,12 @@ proc createMediaRouter*(cfg: Config) =
 
     get re"^\/video\/(enc)?\/?(.+)\/(.+)$":
       let url = decoded(request, 2)
-      cond "http" in url
+      cond isTwitterUrl(url)
 
       if getHmac(url) != request.matches[1]:
         resp Http403, showError("Failed to verify signature", cfg)
 
-      if ".mp4" in url or ".ts" in url or ".m4s" in url:
+      if ".mp4" in url or ".ts" in url or ".m4s" in url or ".aac" in url:
         let code = await proxyMedia(request, url)
         check code
 

src/routes/preferences.nim

@@ -40,3 +40,6 @@ proc createPrefRouter*(cfg: Config) =
       savePref("hlsPlayback", "on", request)
       redirect(refPath())
 
+    post "/enablemp4":
+      savePref("mp4Playback", "on", request)
+      redirect(refPath())

src/routes/router_utils.nim

@@ -8,8 +8,9 @@ export utils, prefs, types, uri
 
 template savePref*(pref, value: string; req: Request; expire=false) =
   if not expire or pref in cookies(req):
+    let sameSite = if cfg.useHttps: None else: Lax
     setCookie(pref, value, daysForward(when expire: -10 else: 360),
-              httpOnly=true, secure=cfg.useHttps, sameSite=None, path="/")
+              httpOnly=true, secure=cfg.useHttps, sameSite=sameSite, path="/")
 
 template requestPrefs*(): untyped {.dirty.} =
   getPrefs(cookies(request), params(request))

src/routes/search.nim

@@ -46,6 +46,7 @@ proc createSearchRouter*(cfg: Config) =
       redirect("/search?f=tweets&q=" & encodeUrl("#" & @"hash"))
 
     get "/opensearch":
-      let url = getUrlPrefix(cfg) & "/search?f=tweets&q="
-      resp Http200, {"Content-Type": "application/opensearchdescription+xml"},
-                     generateOpenSearchXML(cfg.title, cfg.hostname, url)
+      let
+        url = getUrlPrefix(cfg) & "/search?f=tweets&q="
+        headers = {"Content-Type": "application/opensearchdescription+xml"}
+      resp Http200, headers, generateOpenSearchXML(cfg.title, cfg.hostname, url)

src/routes/space.nim

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import asyncdispatch, strutils
+import jester
+
+import router_utils
+import ".."/[types, formatters, redis_cache]
+import ../views/[general, space]
+import media
+
+export space
+
+proc createSpaceRouter*(cfg: Config) =
+  router spaceRoute:
+    get "/i/spaces/@id":
+      cond @"id".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9'})
+      let sp = await getCachedAudioSpace(@"id")
+
+      if sp.id.len == 0:
+        resp Http404, showError("Space not found", cfg)
+
+      let prefs = requestPrefs()
+      resp renderMain(renderSpace(sp, prefs, request.path), request, cfg, prefs,
+                      sp.title, ogTitle=sp.title)
+
+    get "/i/spaces/@id/stream":
+      cond @"id".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9'})
+      let sp = await getCachedAudioSpace(@"id")
+
+      if sp.m3u8Url.len == 0:
+        resp Http404
+
+      let manifest = await safeFetch(sp.m3u8Url)
+      if manifest.len == 0:
+        resp Http502
+
+      resp proxifyVideo(manifest, requestPrefs().proxyVideos, sp.m3u8Url), m3u8Mime

src/routes/timeline.nim

@@ -135,6 +135,39 @@ proc createTimelineRouter*(cfg: Config) =
       resp renderMain(aboutHtml, request, cfg, prefs,
                       "About @" & info.username)
 
+    get "/@name/@kind/?":
+      cond @"kind" in ["followers", "following"]
+      cond '.' notin @"name"
+      cond @"name".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9', '_'})
+      let
+        prefs = requestPrefs()
+        name = @"name"
+        userId = await getUserId(name)
+
+      if userId.len == 0:
+        resp Http404, showError("User \"" & name & "\" not found", cfg)
+      if userId == "suspended":
+        resp showError(getSuspended(name), cfg)
+
+      let
+        cursor = getCursor()
+        isFollowers = @"kind" == "followers"
+        user = await getCachedUser(name)
+        results = if isFollowers: await getGraphFollowers(userId, cursor)
+                  else: await getGraphFollowing(userId, cursor)
+
+      if user.protected:
+        resp renderMain(renderProtected(user.username), request, cfg, prefs,
+                        "Protected account @" & name)
+
+      let
+        tab = if isFollowers: "Followers" else: "Following"
+        title = (if isFollowers: "People following @" else: "People followed by @") & name
+        html = renderUserList(user, results, prefs, request.path, tab)
+
+      resp renderMain(html, request, cfg, prefs, title,
+                      images = @[user.getUserPic("_400x400")])
+
     get "/@name/?@tab?/?":
       cond '.' notin @"name"
       cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]

src/sass/_article.scss

@@ -0,0 +1,278 @@
+.article-page {
+  max-width: 700px;
+  margin: 0 auto 20px;
+  background-color: var(--bg_panel);
+
+  > .top-ref {
+    padding-top: 20px;
+  }
+
+  .article-cover {
+    width: 100%;
+    display: block;
+  }
+
+  .article-body {
+    padding: 20px;
+
+    > :last-child {
+      margin-bottom: 0;
+    }
+
+    .article-title {
+      display: block;
+      font-size: 2rem;
+      line-height: 1.3;
+      margin: 0 0 10px;
+      color: var(--fg_color);
+    }
+
+    .article-author {
+      margin-bottom: 12px;
+      padding-bottom: 10px;
+      border-bottom: 1px solid var(--border_grey);
+      font-size: 14px;
+
+      .article-author-row {
+        display: flex;
+        align-items: center;
+        gap: 8px;
+      }
+
+      .article-avatar {
+        display: flex;
+      }
+
+      .avatar {
+        width: 40px;
+        height: 40px;
+      }
+
+      .article-author-name {
+        display: flex;
+        align-items: center;
+        margin-bottom: 2px;
+
+        .fullname {
+          font-size: 15px;
+        }
+
+        .verified-icon {
+          margin-left: 2px;
+        }
+      }
+
+      .article-author-meta {
+        display: flex;
+        align-items: center;
+      }
+
+      .fullname {
+        font-weight: 700;
+        color: var(--fg_color);
+        max-width: unset;
+        text-overflow: unset;
+        overflow: visible;
+        white-space: normal;
+      }
+
+      .username,
+      .article-date-sep,
+      .article-date {
+        color: var(--fg_dark);
+      }
+
+      .username {
+        margin-left: 0;
+      }
+
+      .article-date-sep {
+        margin: 0 4px;
+      }
+
+      .article-date {
+        text-decoration: none;
+
+        &:hover {
+          text-decoration: underline;
+        }
+      }
+
+      .tweet-stats {
+        margin-top: 6px;
+
+        .tweet-stat {
+          padding-top: 0;
+        }
+      }
+    }
+
+    > h1 {
+      display: block;
+      font-size: 1.8rem;
+      margin: 25px 0 15px;
+    }
+
+    > h2 {
+      font-size: 1.4rem;
+      font-weight: bold;
+      margin: 20px 0 12px;
+    }
+
+    > h3 {
+      font-size: 1.2rem;
+      font-weight: bold;
+      margin: 18px 0 10px;
+    }
+
+    > p {
+      font-size: 16px;
+      line-height: 1.7;
+      margin: 16px 0;
+      word-wrap: break-word;
+    }
+
+    .blockquote-attribution {
+      display: block;
+      margin-top: 0.5em;
+    }
+
+    > blockquote {
+      border-left: 3px solid var(--accent);
+      padding-left: 16px;
+      margin: 16px 0;
+      color: var(--fg_faded);
+      font-size: 16px;
+      line-height: 1.7;
+    }
+
+    > pre {
+      background-color: var(--bg_elements);
+      padding: 12px 16px;
+      border-radius: 6px;
+      overflow-x: auto;
+      margin: 16px 0;
+
+      code {
+        font-family: monospace;
+        font-size: 14px;
+        color: var(--fg_color);
+      }
+    }
+
+    code {
+      background-color: var(--bg_elements);
+      padding: 2px 5px;
+      border-radius: 3px;
+      font-family: monospace;
+      font-size: 0.9em;
+    }
+
+    > ul,
+    > ol {
+      margin: 16px 0;
+      padding-left: 2em;
+
+      li {
+        font-size: 16px;
+        line-height: 1.7;
+        margin: 6px 0;
+      }
+    }
+
+    .article-media {
+      text-align: center;
+      margin: 20px 0;
+
+      img,
+      video {
+        max-width: 100%;
+        border-radius: 12px;
+      }
+
+      .article-media-caption {
+        color: var(--fg_faded);
+        font-size: 0.875rem;
+        margin-top: 6px;
+      }
+    }
+
+    > a,
+    > p a,
+    > h1 a,
+    > h2 a,
+    > h3 a,
+    > blockquote a,
+    > ul a,
+    > ol a {
+      color: var(--accent);
+      text-decoration: none;
+
+      &:hover {
+        text-decoration: underline;
+      }
+    }
+
+    .article-divider {
+      border: none;
+      border-top: 1px solid var(--border_grey);
+      margin: 30px 0;
+    }
+
+    .timeline-item {
+      margin: 20px 0;
+      border: 1px solid var(--border_grey);
+      border-radius: 12px;
+      overflow: hidden;
+    }
+  }
+}
+
+.conversation .article-page {
+  max-width: 100%;
+  margin-bottom: 0;
+}
+
+.article-card {
+  .card-image-container {
+    position: relative;
+  }
+
+  .card-image img {
+    height: auto;
+  }
+
+  .article-card-badge {
+    position: absolute;
+    bottom: 8px;
+    left: 8px;
+    background: rgba(0, 0, 0, 0.75);
+    color: #fff;
+    font-size: 13px;
+    font-weight: 700;
+    padding: 2px 8px;
+    border-radius: 4px;
+  }
+}
+
+.quote .article-card {
+  margin: 0;
+
+  .card-container {
+    border: none;
+    border-radius: 0;
+    border-top: solid 1px var(--dark_grey);
+  }
+}
+
+@media (max-width: 700px) {
+  .article-page {
+    .article-body {
+      padding: 12px 15px 25px;
+
+      .article-title {
+        font-size: 1.6rem;
+      }
+    }
+  }
+}

src/sass/_space.scss

@@ -0,0 +1,149 @@
+.space-page {
+  max-width: 800px;
+  width: 100%;
+  margin: 20px auto 0;
+}
+
+.space-panel {
+  background-color: var(--bg_panel);
+  border: 1px solid var(--border_grey);
+  border-radius: 8px;
+  overflow: hidden;
+}
+
+.space-player {
+  position: relative;
+  background: linear-gradient(135deg, #7b2a8c 0%, #9b3ab1 100%);
+  min-height: 140px;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+
+  audio {
+    width: 100%;
+    padding: 15px;
+    box-sizing: border-box;
+
+    &:not([controls]) {
+      display: none;
+    }
+  }
+
+  .video-overlay {
+    background-color: transparent;
+  }
+}
+
+.space-live {
+  background: #e0245e;
+  color: white;
+  padding: 3px 8px;
+  border-radius: 4px;
+  font-weight: bold;
+  font-size: 12px;
+  text-transform: uppercase;
+  position: absolute;
+  top: 8px;
+  right: 8px;
+}
+
+.space-info {
+  padding: 16px;
+}
+
+.space-header {
+  display: flex;
+  justify-content: space-between;
+  align-items: flex-start;
+  gap: 12px;
+  margin-bottom: 16px;
+}
+
+.space-title {
+  font-size: 18px;
+  font-weight: bold;
+  margin: 0;
+  line-height: 1.3;
+  flex: 1;
+}
+
+.space-meta {
+  display: flex;
+  flex-direction: column;
+  align-items: flex-end;
+  gap: 4px;
+  flex-shrink: 0;
+  font-size: 14px;
+  color: var(--fg_faded);
+}
+
+.listener-count {
+  color: var(--fg_color);
+}
+
+.space-state {
+  color: var(--fg_dark);
+}
+
+.space-participants {
+  border-top: 1px solid var(--border_grey);
+  padding-top: 12px;
+}
+
+.space-participant {
+  margin-bottom: 10px;
+
+  a {
+    display: flex;
+    align-items: center;
+    gap: 10px;
+    color: var(--fg_color);
+    padding: 6px 0;
+  }
+
+  img {
+    width: 40px;
+    height: 40px;
+    border-radius: 50%;
+    flex-shrink: 0;
+  }
+}
+
+.participant-info {
+  min-width: 0;
+}
+
+.participant-name {
+  display: flex;
+  align-items: center;
+  gap: 4px;
+
+  strong {
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
+
+  .verified-icon {
+    margin-bottom: 0;
+    position: relative;
+    top: -2px;
+  }
+}
+
+.host-badge {
+  background: var(--accent);
+  color: white;
+  padding: 2px 7px;
+  border-radius: 3px;
+  font-size: 11px;
+  font-weight: 600;
+  line-height: 1;
+  position: relative;
+  top: 1px;
+}
+
+.participant-username {
+  color: var(--fg_dark);
+  font-size: 13px;
+}

src/sass/index.scss

@@ -8,6 +8,8 @@
 @import "timeline";
 @import "search";
 @import "broadcast";
+@import "space";
+@import "_article";
 
 body {
   // colors
@@ -159,6 +161,7 @@ body.fixed-nav .container {
 
 .verified-icon {
   display: inline-block;
+  position: relative;
   width: 14px;
   height: 14px;
   margin-bottom: 2px;

src/sass/profile/_base.scss

@@ -4,6 +4,7 @@
 @import "card";
 @import "about-account";
 @import "photo-rail";
+@import "community";
 
 .profile-tabs {
   @include panel(auto, 900px);

src/sass/profile/_community.scss

@@ -0,0 +1,203 @@
+.community-header {
+  padding: 12px 15px;
+  border-bottom: 1px solid var(--border_grey);
+  background-color: var(--bg_panel);
+
+  .community-name {
+    font-size: 22px;
+    margin-bottom: 6px;
+
+    a {
+      color: inherit;
+    }
+  }
+
+  .community-category {
+    display: inline-block;
+    background-color: var(--bg_elements);
+    border: 1px solid var(--border_grey);
+    border-radius: 16px;
+    padding: 2px 12px;
+    font-size: 13px;
+    color: var(--fg_faded);
+    margin-bottom: 8px;
+  }
+
+  .community-description {
+    color: var(--fg_faded);
+    margin-bottom: 8px;
+    line-height: 1.4;
+  }
+
+  .community-member-count {
+    font-weight: bold;
+    color: inherit;
+  }
+
+  .community-stats {
+    color: var(--grey);
+    font-size: 14px;
+  }
+}
+
+.community-about {
+  padding: 16px 15px 15px;
+  background-color: var(--bg_panel);
+
+  h2 {
+    font-size: 18px;
+    margin: 0 0 12px;
+  }
+
+  .community-info {
+    border-bottom: 1px solid var(--border_grey);
+    padding-bottom: 12px;
+  }
+
+  .community-info-item {
+    display: flex;
+    gap: 10px;
+    padding: 8px 0;
+    align-items: center;
+
+    .verified-icon {
+      margin-left: 2px;
+    }
+
+    > .icon-container {
+      display: flex;
+      align-items: center;
+      justify-content: center;
+      font-size: 16px;
+      color: var(--grey);
+      flex-shrink: 0;
+      width: 26px;
+      height: 26px;
+    }
+
+    strong {
+      color: var(--fg_color);
+    }
+
+    a {
+      color: var(--accent);
+    }
+  }
+
+  .community-rules {
+    border-bottom: 1px solid var(--border_grey);
+    padding: 16px 0 12px;
+  }
+
+  .community-rules-intro {
+    color: var(--fg_faded);
+    font-size: 14px;
+    margin: 0 0 12px;
+  }
+
+  .community-rule {
+    display: flex;
+    gap: 10px;
+    padding: 10px 0;
+    align-items: flex-start;
+
+    .community-rule-number {
+      display: flex;
+      align-items: center;
+      justify-content: center;
+      min-width: 26px;
+      height: 26px;
+      border-radius: 50%;
+      background-color: var(--accent);
+      color: var(--fg_color);
+      font-weight: bold;
+      font-size: 13px;
+      flex-shrink: 0;
+    }
+
+    .community-rule-content p {
+      margin: 4px 0 0;
+      color: var(--fg_faded);
+      font-size: 14px;
+    }
+  }
+
+  .community-moderators {
+    padding-top: 16px;
+
+    h2 {
+      display: flex;
+      align-items: center;
+      justify-content: space-between;
+    }
+
+    .community-mods-link {
+      font-size: 14px;
+      font-weight: normal;
+      color: var(--accent);
+    }
+  }
+
+  .community-moderator {
+    display: flex;
+    gap: 10px;
+    padding: 8px 0;
+    align-items: center;
+
+    .community-mod-avatar {
+      width: 40px;
+      height: 40px;
+      border-radius: 50%;
+    }
+
+    .community-mod-info {
+      display: flex;
+      flex-direction: column;
+    }
+
+    .community-mod-name {
+      display: flex;
+      align-items: center;
+      font-weight: bold;
+      color: var(--fg_color);
+
+      .verified-icon {
+        margin-left: 2px;
+      }
+    }
+
+    .community-mod-username {
+      color: var(--fg_faded);
+      font-size: 14px;
+    }
+  }
+}
+
+.community-tags {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px;
+  padding: 10px 15px;
+  border-bottom: 1px solid var(--border_grey);
+
+  .community-tag {
+    display: inline-block;
+    background-color: var(--bg_elements);
+    border: 1px solid var(--border_grey);
+    border-radius: 16px;
+    padding: 4px 12px;
+    font-size: 13px;
+    color: var(--accent);
+  }
+}
+
+.community-hashtag-header {
+  padding: 12px 15px;
+  border-bottom: 1px solid var(--border_grey);
+
+  .community-hashtag-title {
+    font-size: 20px;
+    color: var(--accent);
+    margin: 0;
+  }
+}

src/sass/tweet/_base.scss

@@ -305,7 +305,7 @@
   margin-top: 4px;
   margin-bottom: -2px;
 
-  .icon-attention {
+  .icon-attention-circled {
     margin-right: -3px;
   }
 }

src/sass/tweet/media.scss

@@ -90,7 +90,10 @@
   max-height: unset;
 }
 
-.media-gif video {
+.media-gif video,
+.media-gif img {
+  width: 100%;
+  height: 100%;
   max-height: 530px;
   background-color: #101010;
 }

src/sass/tweet/quote.scss

@@ -100,7 +100,8 @@
     justify-content: center;
     background-color: var(--bg_color);
 
-    video {
+    video,
+    img {
       height: unset;
       width: unset;
       max-height: 100%;

src/sass/tweet/video.scss

@@ -28,6 +28,40 @@ video {
   }
 }
 
+.video-download {
+  position: absolute;
+  top: 8px;
+  right: 8px;
+  z-index: 2;
+  color: var(--accent);
+  text-decoration: none;
+  opacity: 0;
+  transition: opacity 0.2s;
+  filter: drop-shadow(0 1px 2px rgba(0, 0, 0, 0.7));
+
+  .icon-container {
+    margin: 0;
+  }
+
+  .icon-download-alt {
+    font-size: 18px;
+  }
+}
+
+.attachment:hover .video-download {
+  opacity: 1;
+
+  &:hover {
+    color: var(--fg_color);
+  }
+}
+
+@media (hover: none) {
+  .video-download {
+    opacity: 1;
+  }
+}
+
 .video-overlay {
   @include play-button;
   background-color: $shadow;

src/types.nim

@@ -16,6 +16,7 @@ type
   ApiUrl* = object
     endpoint*: string
     params*: seq[(string, string)]
+    skipTid*: bool
 
   ApiReq* = object
     oauth*: ApiUrl
@@ -60,6 +61,7 @@ type
     rateLimited = 88
     expiredToken = 89
     listIdOrSlug = 112
+    timelineUnavailable = 131
     tweetNotFound = 144
     tweetNotAuthorized = 179
     forbidden = 200
@@ -127,6 +129,28 @@ type
     availableForReplay*: bool
     user*: User
 
+  SpaceParticipant* = object
+    userId*: string
+    username*: string
+    displayName*: string
+    avatarUrl*: string
+    isVerified*: bool
+
+  AudioSpace* = object
+    id*: string
+    title*: string
+    state*: string
+    mediaKey*: string
+    m3u8Url*: string
+    totalLiveListeners*: int
+    totalReplayWatched*: int
+    startTime*: DateTime
+    endTime*: DateTime
+    availableForReplay*: bool
+    creator*: User
+    admins*: seq[SpaceParticipant]
+    speakers*: seq[SpaceParticipant]
+
   VideoType* = enum
     m3u8 = "application/x-mpegURL"
     mp4 = "video/mp4"
@@ -150,7 +174,7 @@ type
     variants*: seq[VideoVariant]
 
   QueryKind* = enum
-    posts, replies, media, users, tweets, userList
+    posts, replies, media, users, tweets, userList, followers, following
 
   Query* = object
     kind*: QueryKind
@@ -197,6 +221,44 @@ type
 
   PhotoRail* = seq[GalleryPhoto]
 
+  Article* = ref object
+    title*: string
+    coverImage*: string
+    user*: User
+    time*: DateTime
+    stats*: TweetStats
+    paragraphs*: seq[ArticleParagraph]
+    entities*: Table[int, ArticleEntity]
+    media*: Table[string, ArticleMedia]
+
+  ArticleParagraph* = object
+    text*: string
+    kind*: string
+    inlineStyles*: seq[ArticleStyle]
+    entityRanges*: seq[ArticleEntityRange]
+
+  ArticleStyle* = object
+    offset*: int
+    length*: int
+    style*: string
+
+  ArticleEntityRange* = object
+    offset*: int
+    length*: int
+    key*: int
+
+  ArticleEntity* = object
+    kind*: string
+    url*: string
+    mediaIds*: seq[string]
+    tweetId*: string
+    markdown*: string
+    caption*: string
+
+  ArticleMedia* = object
+    kind*: string
+    url*: string
+
   Poll* = object
     options*: seq[string]
     values*: seq[int]
@@ -246,6 +308,12 @@ type
     likes*: int
     views*: int
 
+  ArticlePreview* = object
+    title*: string
+    previewText*: string
+    coverImage*: string
+    tweetId*: int64
+
   Tweet* = ref object
     id*: int64
     threadId*: int64
@@ -264,6 +332,7 @@ type
     stats*: TweetStats
     retweet*: Option[Tweet]
     attribution*: Option[User]
+    attributionLink*: string
     mediaTags*: seq[User]
     quote*: Option[Tweet]
     card*: Option[Card]
@@ -273,6 +342,7 @@ type
     note*: string
     isAd*: bool
     isAI*: bool
+    articlePreview*: Option[ArticlePreview]
 
   Tweets* = seq[Tweet]
 
@@ -315,6 +385,23 @@ type
     members*: int
     banner*: string
 
+  CommunityRule* = object
+    name*: string
+    description*: string
+
+  Community* = object
+    id*: string
+    name*: string
+    description*: string
+    memberCount*: int
+    banner*: string
+    creator*: User
+    category*: string
+    joinPolicy*: string
+    createdAt*: DateTime
+    rules*: seq[CommunityRule]
+    hashtags*: seq[string]
+
   GlobalObjects* = ref object
     tweets*: Table[string, Tweet]
     users*: Table[string, User]

src/utils.nim

@@ -40,12 +40,14 @@ proc getVidUrl*(link: string): string =
     &"/video/{sig}/{encodeUrl(link)}"
 
 proc getPicUrl*(link: string): string =
+  if link.len == 0: return
   if base64Media:
     &"/pic/enc/{encode(link, safe=true)}"
   else:
     &"/pic/{encodeUrl(link)}"
 
 proc getOrigPicUrl*(link: string): string =
+  if link.len == 0: return
   if base64Media:
     &"/pic/orig/enc/{encode(link, safe=true)}"
   else:
@@ -57,8 +59,8 @@ proc filterParams*(params: Table): seq[(string, string)] =
       result.add p
 
 proc isTwitterUrl*(uri: Uri): bool =
-  uri.hostname in twitterDomains or
-    uri.hostname.endsWith(".video.pscp.tv")
+  uri.scheme in ["http", "https"] and
+    (uri.hostname in twitterDomains or uri.hostname.endsWith(".video.pscp.tv"))
 
 proc isTwitterUrl*(url: string): bool =
   isTwitterUrl(parseUri(url))

src/views/article.nim

@@ -0,0 +1,248 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import strutils, strformat, tables, unicode, bitops, uri
+import karax/[karaxdsl, vdom]
+
+import renderutils, tweet, timeline
+import ".."/[types, utils, formatters]
+
+proc renderAtomicParagraph(paragraph: ArticleParagraph; article: Article;
+                           tweets: Table[int64, Tweet]; path: string;
+                           prefs: Prefs): VNode =
+  if paragraph.entityRanges.len == 0:
+    return text ""
+
+  let er = paragraph.entityRanges[0]
+  if er.key notin article.entities:
+    return text ""
+
+  let entity = article.entities[er.key]
+
+  case entity.kind
+  of "MEDIA":
+    buildHtml(tdiv(class="article-media")):
+      for id in entity.mediaIds:
+        let media = article.media.getOrDefault(id)
+        if media.url.len == 0:
+          continue
+        case media.kind
+        of "ApiGif":
+          video(src=getVidUrl(media.url), controls="", autoplay="", loop="",
+                muted="")
+        of "ApiVideo":
+          video(src=getVidUrl(media.url), controls="")
+        else:
+          a(href=getOrigPicUrl(media.url), target="_blank"):
+            img(src=getSmallPic(media.url), alt=entity.caption, loading="lazy")
+      if entity.caption.len > 0:
+        p(class="article-media-caption"): text entity.caption
+  of "TWEET":
+    let tweet = tweets.getOrDefault(
+      try: parseBiggestInt(entity.tweetId)
+      except ValueError: 0, nil)
+    if tweet != nil:
+      renderTweet(tweet, prefs, path)
+    else:
+      text ""
+  of "MARKDOWN":
+    var content = entity.markdown
+    if content.startsWith("```"):
+      let firstNl = content.find('\n')
+      if firstNl >= 0: content = content[firstNl + 1 .. ^1]
+      if content.endsWith("```"): content = content[0 .. ^4]
+      content = content.strip
+    buildHtml(pre()):
+      code(): text content
+  of "DIVIDER":
+    buildHtml(hr(class="article-divider"))
+  else:
+    text ""
+
+proc wrapStyle(node: VNode; style: int): VNode =
+  result = node
+  if style.testBit(4): result = buildHtml(code()): result
+  if style.testBit(0): result = buildHtml(strong()): result
+  if style.testBit(1): result = buildHtml(em()): result
+  if style.testBit(2): result = buildHtml(del()): result
+  if style.testBit(3): result = buildHtml(underlined()): result
+
+proc addContent(target: VNode; content: string; style = 0) =
+  var first = true
+  for line in content.split('\n'):
+    if not first:
+      target.add VNode(kind: VNodeKind.br)
+    first = false
+    var pos = 0
+    while pos < line.len:
+      let atPos = line.find('@', pos)
+      if atPos == -1:
+        target.add wrapStyle(text line[pos .. ^1], style)
+        break
+      if atPos > 0 and line[atPos - 1] in Letters + Digits + {'_'}:
+        target.add wrapStyle(text line[pos .. atPos], style)
+        pos = atPos + 1
+        continue
+      var j = atPos + 1
+      while j < line.len and j - atPos - 1 < 15 and
+            line[j] in Letters + Digits + {'_'}:
+        inc j
+      if j == atPos + 1:
+        target.add wrapStyle(text line[pos .. atPos], style)
+        pos = atPos + 1
+        continue
+      if atPos > pos:
+        target.add wrapStyle(text line[pos ..< atPos], style)
+      let username = line[atPos + 1 ..< j]
+      let link = a.newVNode()
+      link.setAttr("href", "/" & username)
+      link.add wrapStyle(text ("@" & username), style)
+      target.add link
+      pos = j
+
+proc applyInlineStyles(target: VNode; runes: seq[Rune]; start, length: int;
+                       styles: seq[ArticleStyle]) =
+  if styles.len == 0:
+    target.addContent($runes[start ..< start + length])
+    return
+
+  var
+    lastStyle = 0
+    lastStart = start
+  let endPos = start + length
+
+  for i in start ..< endPos:
+    var style = 0
+    for sr in styles:
+      let
+        sStart = sr.offset
+        sEnd = sStart + sr.length
+      if sStart <= i and sEnd > i:
+        case sr.style
+        of "Bold": style.setBit(0)
+        of "Italic": style.setBit(1)
+        of "Strikethrough": style.setBit(2)
+        of "Underline": style.setBit(3)
+        of "Code": style.setBit(4)
+        else: discard
+
+    if style != lastStyle:
+      if i > lastStart:
+        addContent(target, $runes[lastStart ..< i], lastStyle)
+      lastStyle = style
+      lastStart = i
+
+  if lastStart < endPos:
+    addContent(target, $runes[lastStart ..< endPos], lastStyle)
+
+proc renderTextParagraph(paragraph: ArticleParagraph; article: Article): VNode =
+  let text = paragraph.text
+
+  result = case paragraph.kind
+    of "header-one": h1.newVNode()
+    of "header-two": h2.newVNode()
+    of "header-three": h3.newVNode()
+    of "ordered-list-item", "unordered-list-item": li.newVNode()
+    of "blockquote": VNode(kind: VNodeKind.blockquote)
+    of "code-block":
+      let pre = pre.newVNode()
+      let code = code.newVNode()
+      code.add text text
+      pre.add code
+      return pre
+    else: p.newVNode()
+
+  let
+    runes = text.toRunes
+    textLen = runes.len
+  var last = 0
+  for er in paragraph.entityRanges:
+    if er.offset > last:
+      applyInlineStyles(result, runes, last, er.offset - last,
+                        paragraph.inlineStyles)
+
+    last = er.offset + er.length
+
+    var target = result
+    if er.key in article.entities:
+      let entity = article.entities[er.key]
+      if entity.kind == "LINK":
+        let parsed = parseUri(entity.url)
+        if parsed.scheme in ["http", "https"]:
+          target = a.newVNode()
+          if parsed.isTwitterUrl:
+            target.setAttr("href", parsed.path)
+          else:
+            target.setAttr("href", entity.url)
+
+    applyInlineStyles(target, runes, er.offset, er.length,
+                      paragraph.inlineStyles)
+    if target != result:
+      result.add target
+
+  if last < textLen:
+    applyInlineStyles(result, runes, last, textLen - last,
+                      paragraph.inlineStyles)
+
+  if paragraph.kind == "blockquote" and result.len > 0:
+    let lastChild = result[result.len - 1]
+    if lastChild.kind == VNodeKind.strong and lastChild.len > 0 and
+       lastChild[0].kind == VNodeKind.text:
+      lastChild.setAttr("class", "blockquote-attribution")
+
+proc renderArticle*(article: Article; tweets: Table[int64, Tweet];
+                    path: string; prefs: Prefs; tweetId=""): VNode =
+  let author = article.user
+
+  let main = buildHtml(article(class="article-body")):
+    h1(class="article-title"): text article.title
+
+    tdiv(class="article-author"):
+      tdiv(class="article-author-row"):
+        a(class="article-avatar", href=("/" & author.username)):
+          genImg(author.getUserPic("_bigger"), class=prefs.getAvatarClass)
+        tdiv(class="article-author-info"):
+          tdiv(class="article-author-name"):
+            linkUser(author, class="fullname")
+            verifiedIcon(author)
+          tdiv(class="article-author-meta"):
+            linkUser(author, class="username")
+            span(class="article-date-sep"): text " · "
+            a(class="article-date",
+              href=("/" & author.username & "/status/" & tweetId)):
+              text article.time.getShortTime
+      if not prefs.hideTweetStats:
+        renderStats(article.stats)
+
+  var listKind = ""
+  var list: VNode = nil
+
+  for paragraph in article.paragraphs:
+    let isListItem = paragraph.kind in [
+      "ordered-list-item", "unordered-list-item"]
+
+    if not isListItem and list != nil:
+      main.add list
+      list = nil
+      listKind = ""
+
+    if paragraph.kind == "atomic":
+      main.add renderAtomicParagraph(paragraph, article, tweets, path, prefs)
+    elif isListItem:
+      if paragraph.kind != listKind:
+        if list != nil:
+          main.add list
+        list = if paragraph.kind == "ordered-list-item": ol.newVNode()
+               else: ul.newVNode()
+        listKind = paragraph.kind
+      list.add renderTextParagraph(paragraph, article)
+    else:
+      main.add renderTextParagraph(paragraph, article)
+
+  if list != nil:
+    main.add list
+
+  buildHtml(tdiv(class="article-page")):
+    if article.coverImage.len > 0:
+      a(href=getOrigPicUrl(article.coverImage), target="_blank"):
+        img(class="article-cover", src=getSmallPic(article.coverImage), alt="")
+    main
+    renderToTop()

src/views/community.nim

@@ -0,0 +1,128 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import strutils, strformat, times
+import karax/[karaxdsl, vdom]
+
+import renderutils
+import ".."/[types, utils, formatters]
+
+proc renderCommunityTabs*(kind: QueryKind; community: Community): VNode =
+  let
+    path = &"/i/communities/{community.id}"
+    q = Query(kind: kind)
+  buildHtml(tdiv):
+    ul(class="tab"):
+      li(class=q.getTabClass(posts)):
+        a(href=path): text "Top"
+      li(class=q.getTabClass(replies)):
+        a(href=(path & "/latest")): text "Latest"
+      li(class=q.getTabClass(media)):
+        a(href=(path & "/media")): text "Media"
+      li(class=q.getTabClass(userList)):
+        a(href=(path & "/about")): text "About"
+    if community.hashtags.len > 0:
+      tdiv(class="community-tags"):
+        for tag in community.hashtags:
+          let bare = tag.strip(chars={'#'})
+          a(class="community-tag",
+            href=(&"/i/communities/{community.id}/hashtag/{bare}")):
+            text tag
+
+proc renderMemberTabs*(community: Community; isModerators: bool): VNode =
+  let path = &"/i/communities/{community.id}"
+  buildHtml(ul(class="tab")):
+    li(class=(if not isModerators: "tab-item active" else: "tab-item")):
+      a(href=(path & "/members")): text "All"
+    li(class=(if isModerators: "tab-item active" else: "tab-item")):
+      a(href=(path & "/moderators")): text "Moderators"
+
+proc renderHashtagHeader*(community: Community; tag: string): VNode =
+  buildHtml(tdiv(class="community-hashtag-header")):
+    h2(class="community-hashtag-title"): text "#" & tag
+
+proc renderCommunityAbout*(community: Community; moderators: seq[User]): VNode =
+  buildHtml(tdiv(class="community-about")):
+    tdiv(class="community-info"):
+      h2: text "Community Info"
+      tdiv(class="community-info-item"):
+        icon "group"
+        if community.joinPolicy == "Open":
+          text "Anyone can join this Community."
+        else:
+          text "Membership is by approval only."
+
+      tdiv(class="community-info-item"):
+        icon "info"
+        text "All Communities are publicly visible."
+
+      tdiv(class="community-info-item"):
+        icon "calendar"
+        let
+          date = community.createdAt.format("MMMM d, yyyy")
+          creator = community.creator.username
+        span:
+          text &"Created {date} by "
+          a(href=(&"/{creator}")): text &"@{creator}"
+          if community.creator.verifiedType != none:
+            verifiedIcon(community.creator)
+
+    if community.rules.len > 0:
+      tdiv(class="community-rules"):
+        h2: text "Rules"
+        p(class="community-rules-intro"):
+          text "These are set and enforced by Community admins and are in addition to "
+          a(href="https://help.x.com/rules-and-policies/x-rules"): text "X's rules"
+          text "."
+
+        for i, rule in community.rules:
+          tdiv(class="community-rule"):
+            span(class="community-rule-number"): text $(i + 1)
+            tdiv(class="community-rule-content"):
+              strong: text rule.name
+              if rule.description.len > 0:
+                p: text rule.description
+
+    if moderators.len > 0:
+      tdiv(class="community-moderators"):
+        h2:
+          text "Moderators"
+          a(class="community-mods-link",
+            href=(&"/i/communities/{community.id}/moderators")):
+            text "See all"
+        for user in moderators:
+          tdiv(class="community-moderator"):
+            a(href=(&"/{user.username}")):
+              genImg(user.getUserPic("_bigger"), class="community-mod-avatar")
+            tdiv(class="community-mod-info"):
+              a(href=(&"/{user.username}"), class="community-mod-name"):
+                text user.fullname
+                if user.verifiedType != none:
+                  verifiedIcon(user)
+              a(href=(&"/{user.username}"), class="community-mod-username"):
+                text &"@{user.username}"
+
+proc renderCommunity*(body, nav: VNode; community: Community): VNode =
+  buildHtml(tdiv(class="timeline-container")):
+    if community.banner.len > 0:
+      tdiv(class="timeline-banner"):
+        a(href=getPicUrl(community.banner), target="_blank"):
+          genImg(community.banner)
+
+    tdiv(class="community-header"):
+      h1(class="community-name"):
+        a(href=(&"/i/communities/{community.id}")): text community.name
+
+      if community.category.len > 0:
+        span(class="community-category"): text community.category
+
+      if community.description.len > 0:
+        tdiv(class="community-description"):
+          text community.description
+
+      tdiv(class="community-stats"):
+        a(class="community-member-count",
+          href=(&"/i/communities/{community.id}/members")):
+          text insertSep($community.memberCount, ',')
+          text " Members"
+
+    nav
+    body

src/views/general.nim

@@ -50,8 +50,8 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
   let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
 
   buildHtml(head):
-    link(rel="stylesheet", type="text/css", href="/css/style.css?v=35")
-    link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=5")
+    link(rel="stylesheet", type="text/css", href="/css/style.css?v=45")
+    link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=7")
 
     if theme.len > 0:
       link(rel="stylesheet", type="text/css", href=(&"/css/themes/{theme}.css"))
@@ -72,7 +72,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
 
     if prefs.hlsPlayback:
       script(src="/js/hls.min.js", `defer`="")
-      script(src="/js/hlsPlayback.js", `defer`="")
+      script(src="/js/hlsPlayback.js?v=1", `defer`="")
 
     if prefs.infiniteScroll:
       script(src="/js/infiniteScroll.js", `defer`="")
@@ -84,6 +84,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
         text cfg.title
 
     meta(name="viewport", content="width=device-width, initial-scale=1.0")
+    meta(name="referrer", content="same-origin")
     meta(name="theme-color", content="#1F1F1F")
     meta(property="og:type", content=ogType)
     meta(property="og:title", content=(if ogTitle.len > 0: ogTitle else: titleText))
@@ -96,6 +97,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
       link(rel="preload", type="image/png", href=bannerUrl, `as`="image")
 
     for url in images:
+      if url.len == 0: continue
       let preloadUrl = if "400x400" in url: getPicUrl(url)
                        else: getSmallPic(url)
       link(rel="preload", type="image/png", href=preloadUrl, `as`="image")
@@ -117,13 +119,16 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
     # this is last so images are also preloaded
     # if this is done earlier, Chrome only preloads one image for some reason
     link(rel="preload", type="font/woff2", `as`="font",
-         href="/fonts/fontello.woff2?61663884", crossorigin="anonymous")
+         href="/fonts/fontello.woff2?59696369", crossorigin="anonymous")
 
 proc renderMain*(body: VNode; req: Request; cfg: Config; prefs=defaultPrefs;
                  titleText=""; desc=""; ogTitle=""; rss=""; video="";
-                 images: seq[string] = @[]; banner=""): string =
+                 images: seq[string] = @[]; banner="";
+                 twitterLink=""): string =
 
-  let twitterLink = getTwitterLink(req.path, req.params)
+  let twitterLink =
+    if twitterLink.len > 0: twitterLink
+    else: getTwitterLink(req.path, req.params)
 
   let node = buildHtml(html(lang="en")):
     renderHead(prefs, cfg, req, titleText, desc, video, images, banner, ogTitle,

src/views/profile.nim

@@ -2,7 +2,7 @@
 import strutils, strformat
 import karax/[karaxdsl, vdom, vstyles]
 
-import renderutils, search
+import renderutils, search, timeline
 import ".."/[types, utils, formatters]
 
 proc renderStat(num: int; class: string; text=""): VNode =
@@ -12,6 +12,13 @@ proc renderStat(num: int; class: string; text=""): VNode =
     span(class="profile-stat-num"):
       text insertSep($num, ',')
 
+proc renderStatLink(num: int; class, href: string): VNode =
+  buildHtml(li(class=class)):
+    a(href=href):
+      span(class="profile-stat-header"): text capitalizeAscii(class)
+      span(class="profile-stat-num"):
+        text insertSep($num, ',')
+
 proc renderUserCard*(user: User; prefs: Prefs; info: AccountInfo): VNode =
   buildHtml(tdiv(class="profile-card")):
     tdiv(class="profile-card-info"):
@@ -65,8 +72,8 @@ proc renderUserCard*(user: User; prefs: Prefs; info: AccountInfo): VNode =
       tdiv(class="profile-card-extra-links"):
         ul(class="profile-statlist"):
           renderStat(user.tweets, "posts", text="Tweets")
-          renderStat(user.following, "following")
-          renderStat(user.followers, "followers")
+          renderStatLink(user.following, "following", &"/{user.username}/following")
+          renderStatLink(user.followers, "followers", &"/{user.username}/followers")
           renderStat(user.likes, "likes")
 
 proc renderPhotoRail(profile: Profile): VNode =
@@ -99,7 +106,7 @@ proc renderBanner(banner: string): VNode =
     else:
       a(href=getPicUrl(banner), target="_blank"): genImg(banner)
 
-proc renderProtected(username: string): VNode =
+proc renderProtected*(username: string): VNode =
   buildHtml(tdiv(class="timeline-container")):
     tdiv(class="timeline-header timeline-protected"):
       h2: text "This account's tweets are protected."
@@ -128,3 +135,31 @@ proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
       renderProtected(profile.user.username)
     else:
       renderTweetSearch(profile.tweets, prefs, path, profile.pinned)
+
+proc renderFollowTabs(user: User; activeTab: string): VNode =
+  buildHtml(ul(class="tab")):
+    for tab in ["Following", "Followers"]:
+      li(class=(if activeTab == tab: "tab-item active" else: "tab-item")):
+        a(href=(&"/{user.username}/{tab.toLowerAscii()}")): text tab
+
+proc renderUserList*(user: User; results: Result[User]; prefs: Prefs;
+                     path, activeTab: string): VNode =
+  # Check if we've loaded all results (for page 1)
+  var displayResults = results
+  if results.beginning:
+    let expectedCount = if activeTab == "Followers": user.followers else: user.following
+    if results.content.len >= expectedCount:
+      displayResults.bottom = ""
+
+  buildHtml(tdiv(class="profile-tabs")):
+    if not prefs.hideBanner:
+      tdiv(class="profile-banner"):
+        renderBanner(user.banner)
+
+    let sticky = if prefs.stickyProfile: " sticky" else: ""
+    tdiv(class=("profile-tab" & sticky)):
+      renderUserCard(user, prefs, AccountInfo())
+
+    tdiv(class="timeline-container"):
+      renderFollowTabs(user, activeTab)
+      renderTimelineUsers(displayResults, prefs, path)

src/views/renderutils.nim

@@ -7,18 +7,20 @@ const smallWebp* = "?name=small&format=webp"
 const mediumWebp* = "?name=medium&format=webp"
 
 proc getSmallPic*(url: string): string =
+  if url.len == 0: return
   result = url
   if "?" notin url and not url.endsWith("placeholder.png"):
     result &= smallWebp
   result = getPicUrl(result)
 
 proc getMediumPic*(url: string): string =
+  if url.len == 0: return
   result = url
   if "?" notin url and not url.endsWith("placeholder.png"):
     result &= mediumWebp
   result = getPicUrl(result)
 
-proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
+proc icon*(icon: string; label=""; title=""; class=""; href=""): VNode =
   var c = "icon-" & icon
   if class.len > 0: c = &"{c} {class}"
   buildHtml(tdiv(class="icon-container")):
@@ -27,8 +29,8 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
     else:
       span(class=c, title=title)
 
-    if text.len > 0:
-      text " " & text
+    if label.len > 0:
+      text " " & label
 
 template verifiedIcon*(user: User): untyped {.dirty.} =
   if user.verifiedType != VerifiedType.none:

src/views/rss.nimf

@@ -12,7 +12,7 @@
 #elif retweet.len > 0: prefix = &"RT by @{retweet}: "
 #elif tweet.reply.len > 0: prefix = &"R to @{tweet.reply[0]}: "
 #end if
-#var text = stripHtml(tweet.text)
+#var text = strutils.splitWhitespace(stripHtml(tweet.text)).join(" ")
 ##if unicode.runeLen(text) > 32:
 ##  text = unicode.runeSubStr(text, 0, 32) & "..."
 ##end if
@@ -34,6 +34,18 @@
 #    end case
 #  end if
 #end if
+#if result.len == 0 and tweet.articlePreview.isSome:
+#  let art = tweet.articlePreview.get()
+#  if art.title.len > 0:
+#    result = prefix & xmltree.escape(art.title)
+#  end if
+#end if
+#if result.len == 0 and tweet.card.isSome:
+#  let card = tweet.card.get()
+#  if card.kind notin {hidden, unknown} and card.title.len > 0:
+#    result = prefix & xmltree.escape(card.title)
+#  end if
+#end if
 #end proc
 #
 #proc getDescription(desc: string; cfg: Config): string =
@@ -60,6 +72,72 @@ Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)}
 #end case
 #end proc
 #
+#proc renderRssCard(card: Card; prefs: Prefs; urlPrefix: string): string =
+#let cardLink = if card.url.startsWith("/"): urlPrefix & card.url else: replaceUrls(card.url, prefs)
+#let title = xmltree.escape(card.title)
+<hr/>
+<b>Link</b><br>
+#if cardLink.len > 0:
+<a href="${xmltree.escape(cardLink)}">
+#end if
+#if card.image.len > 0:
+<img src="${urlPrefix}${getPicUrl(card.image)}" style="max-width:250px;" />
+#end if
+#if title.len > 0:
+#  if card.image.len > 0:
+<br>
+#  end if
+<b>${title}</b>
+#end if
+#if cardLink.len > 0:
+</a>
+#end if
+#if card.text.len > 0:
+<p>${xmltree.escape(card.text)}</p>
+#end if
+#if cardLink.len > 0:
+#  let destText = if card.dest.len > 0: xmltree.escape(card.dest) else: xmltree.escape(cardLink)
+<small><a href="${xmltree.escape(cardLink)}">${destText}</a></small>
+#elif card.dest.len > 0:
+<small>${xmltree.escape(card.dest)}</small>
+#end if
+#end proc
+#
+#proc renderRssArticle(article: ArticlePreview; urlPrefix: string): string =
+#let link = urlPrefix & "/i/article/" & $article.tweetId
+<hr/>
+<b>Article</b><br>
+<a href="${link}">
+#if article.coverImage.len > 0:
+<img src="${urlPrefix}${getPicUrl(article.coverImage)}" style="max-width:250px;" />
+#end if
+#if article.title.len > 0:
+#  if article.coverImage.len > 0:
+<br>
+#  end if
+<b>${xmltree.escape(article.title)}</b>
+#end if
+</a>
+#if article.previewText.len > 0:
+<p>${xmltree.escape(article.previewText)}</p>
+#end if
+#end proc
+#
+#proc renderRssPoll(poll: Poll): string =
+<hr/>
+<b>Poll</b>
+<p>
+#for i in 0 ..< poll.options.len:
+#  let perc = if poll.votes > 0: poll.values[i] / poll.votes * 100 else: 0.0
+#  let pct = (&"{perc:.0f}").strip(chars={'.'})
+#  let line = pct & "% — " & xmltree.escape(poll.options[i])
+${line}<br>
+#end for
+#let votesStr = insertSep($poll.votes, ',')
+<i>${votesStr} votes • ${xmltree.escape(poll.status)}</i>
+</p>
+#end proc
+#
 #proc getTweetsWithPinned(profile: Profile): seq[Tweets] =
 #result = profile.tweets.content
 #if profile.pinned.isSome and result.len > 0:
@@ -82,16 +160,21 @@ Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)}
 #let tweet = tweet.retweet.get(tweet)
 #let urlPrefix = getUrlPrefix(cfg)
 #let text = replaceUrls(tweet.text, prefs, absolute=urlPrefix)
+#if text.len > 0:
 <p>${text.replace("\n", "<br>\n")}</p>
+#end if
 #if tweet.media.len > 0:
 #  for media in tweet.media:
 ${renderRssMedia(media, tweet, urlPrefix)}
 #  end for
-#elif tweet.card.isSome:
-#  let card = tweet.card.get()
-#  if card.image.len > 0:
-<img src="${urlPrefix}${getPicUrl(card.image)}" style="max-width:250px;" />
-#  end if
+#elif tweet.card.isSome and tweet.card.get().kind notin {hidden, unknown}:
+${renderRssCard(tweet.card.get(), prefs, urlPrefix)}
+#end if
+#if tweet.articlePreview.isSome:
+${renderRssArticle(tweet.articlePreview.get(), urlPrefix)}
+#end if
+#if tweet.poll.isSome:
+${renderRssPoll(tweet.poll.get())}
 #end if
 #if tweet.note.len > 0 and not prefs.hideCommunityNotes:
 <p><b>Community note:</b> ${replaceUrls(tweet.note, prefs, absolute=urlPrefix)}</p>

src/views/space.nim

@@ -0,0 +1,86 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import strutils, times
+import karax/[karaxdsl, vdom]
+
+import renderutils
+import ".."/[types, utils, formatters]
+
+proc renderParticipant(p: SpaceParticipant; role: string): VNode =
+  buildHtml(tdiv(class="space-participant")):
+    a(href=("/" & p.username)):
+      genImg(p.avatarUrl.replace("_normal", "_bigger"))
+      tdiv(class="participant-info"):
+        tdiv(class="participant-name"):
+          strong: text p.displayName
+          if p.isVerified:
+            tdiv(class="verified-icon blue"):
+              icon "circle", class="verified-icon-circle", title="Verified account"
+              icon "ok", class="verified-icon-check", title="Verified account"
+          if role.len > 0:
+            span(class="host-badge"): text role
+        span(class="participant-username"): text "@" & p.username
+
+proc renderSpace*(sp: AudioSpace; prefs: Prefs; path: string): VNode =
+  let
+    isLive = sp.state == "RUNNING"
+    source = if prefs.proxyVideos and sp.m3u8Url.startsWith("http"):
+               getVidUrl(sp.m3u8Url) else: sp.m3u8Url
+    stateText =
+      if isLive: "LIVE"
+      elif sp.endTime.year > 1: "Ended " & sp.endTime.format("MMM d, YYYY")
+      elif sp.state.len > 0: sp.state
+      else: "Ended"
+    durationMs =
+      if sp.startTime.year > 1 and sp.endTime.year > 1:
+        int((sp.endTime - sp.startTime).inMilliseconds)
+      else: 0
+    duration = if durationMs > 0: getDuration(durationMs) else: ""
+    totalListeners =
+      if sp.totalReplayWatched > 0: sp.totalReplayWatched
+      else: sp.totalLiveListeners
+
+  buildHtml(tdiv(class="space-page")):
+    tdiv(class="space-panel"):
+      tdiv(class="space-player"):
+        if sp.m3u8Url.len > 0 and prefs.hlsPlayback:
+          audio(data-url=source, data-autoload="false")
+          verbatim "<div class=\"video-overlay\" onclick=\"playAudio(this)\">"
+          tdiv(class="overlay-circle"): span(class="overlay-triangle")
+          if isLive:
+            tdiv(class="space-live"): text "LIVE"
+          elif duration.len > 0:
+            tdiv(class="overlay-duration"): text duration
+          verbatim "</div>"
+        elif sp.m3u8Url.len > 0:
+          tdiv(class="video-overlay"):
+            buttonReferer "/enablehls", "Enable hls playback", path
+            if isLive:
+              tdiv(class="space-live"): text "LIVE"
+            elif duration.len > 0:
+              tdiv(class="overlay-duration"): text duration
+        elif sp.availableForReplay:
+          tdiv(class="video-overlay"):
+            p: text "Audio stream unavailable"
+        else:
+          tdiv(class="video-overlay"):
+            p: text "Replay is not available"
+
+      tdiv(class="space-info"):
+        tdiv(class="space-header"):
+          h2(class="space-title"): text sp.title
+          tdiv(class="space-meta"):
+            if totalListeners > 0:
+              span(class="listener-count"): text insertSep($totalListeners, ',') & " listeners"
+            if isLive:
+              span(class="space-live"): text stateText
+            else:
+              span(class="space-state"): text stateText
+
+        if sp.admins.len > 0 or sp.speakers.len > 0:
+          tdiv(class="space-participants"):
+            for admin in sp.admins:
+              let role = if admin.username == sp.creator.username: "Host"
+                         else: "Co-host"
+              renderParticipant(admin, role)
+            for speaker in sp.speakers:
+              renderParticipant(speaker, "")

src/views/timeline.nim

@@ -154,7 +154,8 @@ proc renderTimelineTweets*(results: Timeline; prefs: Prefs; path: string;
             else: renderThread(thread, prefs, path, bigThumb)
       else:
         for thread in filtered:
-          if thread.len == 1: renderTweet(thread[0], prefs, path)
+          if thread.len == 1:
+            renderTweet(thread[0], prefs, path)
           else: renderThread(thread, prefs, path)
 
       var cursor = getSearchMaxId(results, path)

src/views/tweet.nim

@@ -9,17 +9,36 @@ import general
 
 const doctype = "<!DOCTYPE html>\n"
 
-proc renderMiniAvatar(user: User; prefs: Prefs): VNode =
+proc renderMiniAvatar*(user: User; prefs: Prefs): VNode =
   genImg(user.getUserPic("_mini"), class=(prefs.getAvatarClass & " mini"))
 
-proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs): VNode =
+proc renderArticleCard(preview: ArticlePreview; prefs: Prefs): VNode =
+  let url = "/i/article/" & $preview.tweetId
+  buildHtml(tdiv(class="article-card card large")):
+    a(class="card-container", href=url):
+      if preview.coverImage.len > 0:
+        tdiv(class="card-image-container"):
+          tdiv(class="card-image"):
+            genImg(preview.coverImage)
+          span(class="article-card-badge"): text "Article"
+      tdiv(class="card-content-container"):
+        tdiv(class="card-content"):
+          h2(class="card-title"): text preview.title
+          if preview.previewText.len > 0:
+            p(class="card-description"): text preview.previewText
+
+proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs;
+                   path = ""): VNode =
   buildHtml(tdiv):
     if pinned:
+      let pinnedLabel =
+        if "/i/communities/" in path: "Pinned by Community mods"
+        else: "Pinned Tweet"
       tdiv(class="pinned"):
-        span: icon "pin", "Pinned Tweet"
+        span: icon("pin", pinnedLabel)
     elif retweet.len > 0:
       tdiv(class="retweet-header"):
-        span: icon "retweet", retweet & " retweeted"
+        span: icon("retweet", retweet & " retweeted")
 
     tdiv(class="tweet-header"):
       a(class="tweet-avatar", href=("/" & tweet.user.username)):
@@ -66,7 +85,7 @@ proc renderVideoDisabled(playbackType: VideoType; path=""): VNode =
   buildHtml(tdiv(class="video-overlay")):
     case playbackType
     of mp4:
-      p: text "mp4 playback disabled in preferences"
+      buttonReferer "/enablemp4", "Enable mp4 playback", path
     of m3u8, vmap:
       buttonReferer "/enablehls", "Enable hls playback", path
 
@@ -78,6 +97,12 @@ proc renderVideoUnavailable(video: Video): VNode =
     else:
       p: text "This media is unavailable"
 
+proc getVideoDownloadUrl(videoData: Video): string =
+  let mp4Vars = videoData.variants.filterIt(it.contentType == mp4)
+  if mp4Vars.len == 0: return ""
+  let best = mp4Vars.sortedByIt(it.bitrate)[^1].url
+  if best.startsWith("http"): getVidUrl(best) else: best
+
 proc renderVideoAttachment(videoData: Video; prefs: Prefs; path=""; bigThumb=false): VNode =
   let
     playbackType = if not prefs.proxyVideos and videoData.hasMp4Url: mp4
@@ -108,6 +133,11 @@ proc renderVideoAttachment(videoData: Video; prefs: Prefs; path=""; bigThumb=fal
         if videoData.durationMs > 0:
           tdiv(class="overlay-duration"): text getDuration(videoData)
         verbatim "</div>"
+    if videoData.available:
+      let dlUrl = getVideoDownloadUrl(videoData)
+      if dlUrl.len > 0:
+        a(class="video-download", href=dlUrl, download="",
+          title="Download video"): icon "download-alt"
 
 proc renderVideo*(video: Video; prefs: Prefs; path: string; bigThumb=false): VNode =
   let hasCardContent = video.description.len > 0 or video.title.len > 0
@@ -121,13 +151,13 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string; bigThumb=false): VNo
           if video.description.len > 0:
             p(class="card-description"): text video.description
 
-proc renderGifAttachment(gif: Gif; prefs: Prefs): VNode =
+proc renderGifAttachment(gif: Gif; prefs: Prefs; path=""): VNode =
   let thumb = getSmallPic(gif.thumb)
 
   buildHtml(tdiv(class="attachment")):
     if not prefs.mp4Playback:
       img(src=thumb, loading="lazy")
-      renderVideoDisabled(mp4)
+      renderVideoDisabled(mp4, path)
     elif prefs.autoplayGifs:
       video(class="gif", poster=thumb, autoplay="", muted="", loop=""):
         source(src=getPicUrl(gif.url), `type`="video/mp4")
@@ -137,9 +167,9 @@ proc renderGifAttachment(gif: Gif; prefs: Prefs): VNode =
     if gif.altText.len > 0:
       renderAltText(gif.altText)
 
-proc renderGif(gif: Gif; prefs: Prefs): VNode =
+proc renderGif(gif: Gif; prefs: Prefs; path=""): VNode =
   buildHtml(tdiv(class="attachments media-gif")):
-    renderGifAttachment(gif, prefs)
+    renderGifAttachment(gif, prefs, path)
 
 proc renderMedia(media: seq[Media]; prefs: Prefs; path: string; bigThumb=false): VNode =
   if media.len == 0:
@@ -150,7 +180,7 @@ proc renderMedia(media: seq[Media]; prefs: Prefs; path: string; bigThumb=false):
     if item.kind == videoMedia:
       return renderVideo(item.video, prefs, path, bigThumb)
     if item.kind == gifMedia:
-      return renderGif(item.gif, prefs)
+      return renderGif(item.gif, prefs, path)
 
   let
     groups = if media.len < 3: @[media]
@@ -169,7 +199,7 @@ proc renderMedia(media: seq[Media]; prefs: Prefs; path: string; bigThumb=false):
           of videoMedia:
             renderVideoAttachment(mediaItem.video, prefs, path, bigThumb)
           of gifMedia:
-            renderGifAttachment(mediaItem.gif, prefs)
+            renderGifAttachment(mediaItem.gif, prefs, path)
 
 proc renderPoll(poll: Poll): VNode =
   buildHtml(tdiv(class="poll")):
@@ -225,7 +255,7 @@ func formatStat(stat: int): string =
   if stat > 0: insertSep($stat, ',')
   else: ""
 
-proc renderStats(stats: TweetStats): VNode =
+proc renderStats*(stats: TweetStats): VNode =
   buildHtml(tdiv(class="tweet-stats")):
     span(class="tweet-stat"): icon "comment", formatStat(stats.replies)
     span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets)
@@ -239,8 +269,9 @@ proc renderReply(tweet: Tweet): VNode =
       if i > 0: text " "
       a(href=("/" & u)): text "@" & u
 
-proc renderAttribution(user: User; prefs: Prefs): VNode =
-  buildHtml(a(class="attribution", href=("/" & user.username))):
+proc renderAttribution(user: User; prefs: Prefs; link = ""): VNode =
+  let href = if link.len > 0: link else: "/" & user.username
+  buildHtml(a(class="attribution", href=href)):
     renderMiniAvatar(user, prefs)
     strong: text user.fullname
     verifiedIcon(user)
@@ -307,6 +338,9 @@ proc renderQuote(quote: Tweet; prefs: Prefs; path: string): VNode =
     if quote.media.len > 0:
       renderQuoteMedia(quote, prefs, path)
 
+    if quote.articlePreview.isSome:
+      renderArticleCard(quote.articlePreview.get(), prefs)
+
     if quote.note.len > 0 and not prefs.hideCommunityNotes:
       renderCommunityNote(quote.note, prefs)
 
@@ -322,10 +356,10 @@ proc renderDisclosures*(tweet: Tweet): VNode =
   buildHtml(tdiv(class="disclosures")):
     if tweet.isAI:
       span(data-disclosure="ai"):
-        icon "attention", "Made with AI"
+        icon "attention-circled", "Made with AI"
     if tweet.isAd:
       span(data-disclosure="ad"):
-        icon "attention", "Paid partnership (ad)"
+        icon "attention-circled", "Paid partnership (ad)"
 
 proc renderLocation*(tweet: Tweet): string =
   let (place, url) = tweet.getLocation()
@@ -339,7 +373,8 @@ proc renderLocation*(tweet: Tweet): string =
   return $node
 
 proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
-                  last=false; mainTweet=false; afterTweet=false; bigThumb=false): VNode =
+                  last=false; mainTweet=false; afterTweet=false;
+                  bigThumb=false): VNode =
   var divClass = class
   if index == -1 or last:
     divClass = "thread-last " & class
@@ -372,7 +407,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       a(class="tweet-link", href=getLink(tweet))
 
     tdiv(class="tweet-body"):
-      renderHeader(tweet, retweet, pinned, prefs)
+      renderHeader(tweet, retweet, pinned, prefs, path)
 
       if not afterTweet and index == 0 and tweet.reply.len > 0 and
          (tweet.reply.len > 1 or tweet.reply[0] != tweet.user.username or pinned):
@@ -386,11 +421,14 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         verbatim replaceUrls(tweet.text, prefs) & renderLocation(tweet)
 
       if tweet.attribution.isSome:
-        renderAttribution(tweet.attribution.get(), prefs)
+        renderAttribution(tweet.attribution.get(), prefs, tweet.attributionLink)
 
       if tweet.card.isSome and tweet.card.get().kind != hidden:
         renderCard(tweet.card.get(), prefs, path)
 
+      if tweet.articlePreview.isSome:
+        renderArticleCard(tweet.articlePreview.get(), prefs)
+
       if tweet.media.len > 0:
         renderMedia(tweet.media, prefs, path, bigThumb)