diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 7a46257..20a15a0 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -7,55 +7,105 @@ on: branches: - master +concurrency: + group: docker-publish-${{ github.ref }} + cancel-in-progress: true + +env: + IMAGE: zedeus/nitter + jobs: tests: uses: ./.github/workflows/run-tests.yml secrets: inherit - build-docker-amd64: + build: needs: [tests] - runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + platform: linux/amd64 + - runner: ubuntu-24.04-arm + platform: linux/arm64 + runs-on: ${{ matrix.runner }} steps: + - name: Prepare platform name + run: echo "PLATFORM_PAIR=${platform//\//-}" >> "$GITHUB_ENV" + env: + platform: ${{ matrix.platform }} + - uses: actions/checkout@v6 + - name: Set up Docker Buildx - id: buildx uses: docker/setup-buildx-action@v3 with: version: latest + - name: Login to DockerHub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and push AMD64 Docker image - uses: docker/build-push-action@v3 + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v6 with: context: . file: ./Dockerfile - platforms: linux/amd64 - push: true - tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }} + platforms: ${{ matrix.platform }} + outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true + provenance: false + sbom: false - build-docker-arm64: - needs: [tests] - runs-on: ubuntu-24.04-arm + - name: Export digest + run: | + mkdir -p "${{ runner.temp }}/digests" + digest="${{ steps.build.outputs.digest }}" + touch "${{ runner.temp }}/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ env.PLATFORM_PAIR }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 + + # Combine the per-arch digests into one multi-arch manifest so that + # `docker pull zedeus/nitter:latest` serves the right image on any CPU. + merge: + needs: [build] + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: ${{ runner.temp }}/digests + pattern: digests-* + merge-multiple: true + - name: Set up Docker Buildx - id: buildx uses: docker/setup-buildx-action@v3 with: version: latest + - name: Login to DockerHub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and push ARM64 Docker image - uses: docker/build-push-action@v3 - with: - context: . - file: ./Dockerfile.arm64 - platforms: linux/arm64 - push: true - tags: zedeus/nitter:latest-arm64,zedeus/nitter:${{ github.sha }}-arm64 + + - name: Create manifest list and push + working-directory: ${{ runner.temp }}/digests + run: | + docker buildx imagetools create \ + -t ${{ env.IMAGE }}:latest \ + -t ${{ env.IMAGE }}:latest-arm64 \ + -t ${{ env.IMAGE }}:${{ github.sha }} \ + $(printf '${{ env.IMAGE }}@sha256:%s ' *) + + - name: Inspect image + run: docker buildx imagetools inspect ${{ env.IMAGE }}:${{ github.sha }} diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index c86087a..dcfff52 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -32,10 +32,12 @@ jobs: id: cache-nimble uses: actions/cache@v5 with: - path: ~/.nimble - key: ${{ matrix.nim }}-nimble-v2-${{ hashFiles('*.nimble') }} + path: | + ~/.nimble/pkgcache + ~/.nimble/packages_official.json + key: ${{ matrix.nim }}-nimble-v6-${{ hashFiles('*.nimble') }} restore-keys: | - ${{ matrix.nim }}-nimble-v2- + ${{ matrix.nim }}-nimble-v6- - name: Setup Nim uses: jiro4989/setup-nim-action@v2 @@ -103,10 +105,12 @@ jobs: - name: Cache Nimble Dependencies uses: actions/cache@v5 with: - path: ~/.nimble - key: 2.2.x-nimble-v2-${{ hashFiles('*.nimble') }} + path: | + ~/.nimble/pkgcache + ~/.nimble/packages_official.json + key: 2.2.x-nimble-v6-${{ hashFiles('*.nimble') }} restore-keys: | - 2.2.x-nimble-v2- + 2.2.x-nimble-v6- - name: Setup Nim uses: jiro4989/setup-nim-action@v2 @@ -115,6 +119,9 @@ jobs: use-nightlies: true repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install Nimble dependencies + run: nimble install -y --depsOnly + - name: Download 2.2.x build artifact uses: actions/download-artifact@v4 with: @@ -130,10 +137,8 @@ jobs: sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf sed -i 's/maxRetries = 1/maxRetries = 10/g' nitter.conf - # Run both Nimble tasks concurrently - nim r tools/rendermd.nim & - nim r tools/gencss.nim & - wait + nim r tools/rendermd.nim + nim r tools/gencss.nim echo '${{ secrets.SESSIONS }}' | head -n1 echo '${{ secrets.SESSIONS }}' > ./sessions.jsonl diff --git a/.gitignore b/.gitignore index 09bdaa4..2e52163 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ sessions.json* dump.rdb *.bak /tools/*.json* +nimbledeps/ +nimble.paths +nimble.develop diff --git a/Dockerfile b/Dockerfile index ab442ba..251b63a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM nimlang/nim:2.2.0-alpine-regular as nim +FROM nimlang/nim:2.2.6-alpine-regular as nim LABEL maintainer="setenforce@protonmail.com" RUN apk --no-cache add libsass-dev pcre @@ -15,7 +15,7 @@ RUN nimble build -d:danger -d:lto -d:strip --mm:refc \ FROM alpine:latest WORKDIR /src/ -RUN apk --no-cache add pcre ca-certificates +RUN apk --no-cache add pcre ca-certificates openssl COPY --from=nim /src/nitter/nitter ./ COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf COPY --from=nim /src/nitter/public ./public diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 deleted file mode 100644 index 46352c7..0000000 --- a/Dockerfile.arm64 +++ /dev/null @@ -1,25 +0,0 @@ -FROM alpine:3.20.6 as nim -LABEL maintainer="setenforce@protonmail.com" - -RUN apk --no-cache add libsass-dev pcre gcc git libc-dev nim nimble - -WORKDIR /src/nitter - -COPY nitter.nimble . -RUN nimble install -y --depsOnly - -COPY . . -RUN nimble build -d:danger -d:lto -d:strip --mm:refc \ - && nimble scss \ - && nimble md - -FROM alpine:3.20.6 -WORKDIR /src/ -RUN apk --no-cache add pcre ca-certificates openssl -COPY --from=nim /src/nitter/nitter ./ -COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf -COPY --from=nim /src/nitter/public ./public -EXPOSE 8080 -RUN adduser -h /src/ -D -s /bin/sh nitter -USER nitter -CMD ./nitter diff --git a/README.md b/README.md index 05c2be4..0b0a14f 100644 --- a/README.md +++ b/README.md @@ -104,9 +104,9 @@ along with the scss and md files. # su nitter $ git clone https://github.com/zedeus/nitter $ cd nitter -$ nimble build -d:danger --mm:refc -$ nimble scss -$ nimble md +$ nimble -l build -d:danger --mm:refc +$ nimble -l scss +$ nimble -l md $ cp nitter.example.conf nitter.conf ``` @@ -123,12 +123,23 @@ performance reasons. Page for the Docker image: https://hub.docker.com/r/zedeus/nitter -#### NOTE: For ARM64 support, please use the separate ARM64 docker image: [`zedeus/nitter:latest-arm64`](https://hub.docker.com/r/zedeus/nitter/tags). +#### NOTE: The published image is multi-arch — `zedeus/nitter:latest` runs natively on both `amd64` and `arm64`. To run Nitter with Docker, you'll need to install and run Redis separately before you can run the container. See below for how to also run Redis using Docker. +First create your config file. The Docker commands mount it into the container, +so it has to exist on the host beforehand. If you've cloned the repo: + +```bash +cp nitter.example.conf nitter.conf +``` + +If you're using the prebuilt image without a local clone, download +[`nitter.example.conf`](https://raw.githubusercontent.com/zedeus/nitter/master/nitter.example.conf) +and save it as `nitter.conf` instead. + To build and run Nitter in Docker: ```bash @@ -136,8 +147,6 @@ docker build -t nitter:latest . docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest ``` -Note: For ARM64, use this Dockerfile: [`Dockerfile.arm64`](https://github.com/zedeus/nitter/blob/master/Dockerfile.arm64). - A prebuilt Docker image is provided as well: ```bash @@ -151,8 +160,11 @@ Change `redisHost` from `localhost` to `nitter-redis` in `nitter.conf`, then run docker-compose up -d ``` -Note the Docker commands expect a `nitter.conf` file in the directory you run -them. +Note the Docker commands mount `nitter.conf` (and `sessions.jsonl` for +docker-compose) from the directory you run them in. If a mounted file doesn't +exist, Docker silently creates a directory in its place and the container fails +with `not a directory: Are you trying to mount a directory onto a file`. Remove +that directory and create the file as shown above. ### systemd diff --git a/docker-compose.yml b/compose.yml similarity index 98% rename from docker-compose.yml rename to compose.yml index 3d75751..72d5a96 100644 --- a/docker-compose.yml +++ b/compose.yml @@ -1,5 +1,3 @@ -version: "3" - services: nitter: diff --git a/config.nims b/config.nims index 4a7af27..3ee4842 100644 --- a/config.nims +++ b/config.nims @@ -11,3 +11,7 @@ warning("HoleEnumConv", off) hint("XDeclaredButNotUsed", off) hint("XCannotRaiseY", off) hint("User", off) +# begin Nimble config (version 2) +when withDir(thisDir(), system.fileExists("nimble.paths")): + include "nimble.paths" +# end Nimble config diff --git a/nitter.example.conf b/nitter.example.conf index a58c63c..32fb618 100644 --- a/nitter.example.conf +++ b/nitter.example.conf @@ -20,7 +20,7 @@ redisMaxConnections = 30 # you receive tons of requests per second [Config] -hmacKey = "secretkey" # random key for cryptographic signing of video urls +hmacKey = "secretkey" # CHANGE THIS to a unique random value (e.g. `openssl rand -hex 32`); signs media urls base64Media = false # use base64 encoding for proxied media urls enableRSS = true # master switch, set to false to disable all RSS feeds enableRSSUserTweets = true # /@user/rss diff --git a/nitter.nimble b/nitter.nimble index c206105..b36f498 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -11,19 +11,18 @@ bin = @["nitter"] # Dependencies requires "nim >= 2.0.0" -requires "jester#baca3f" -requires "karax#5cf360c" -requires "sass#7dfdd03" -requires "nimcrypto#a079df9" -requires "markdown#158efe3" +requires "jester == 0.6.0" +requires "karax == 1.5.0" +requires "sass == 0.2.0" +requires "nimcrypto == 0.7.3" +requires "markdown == 0.8.8" requires "packedjson#9e6fbb6" -requires "supersnappy#6c94198" -requires "redpool#8b7c1db" -requires "https://github.com/zedeus/redis#d0a0e6f" -requires "zippy#ca5989a" -requires "flatty#e668085" -requires "jsony#1de1f08" -requires "oauth#b8c163b" +requires "supersnappy == 2.1.4" +requires "redpool == 0.2.2" +requires "zippy == 0.10.19" +requires "flatty == 0.4.0" +requires "jsony == 1.1.6" +requires "oauth == 0.11" # Tasks diff --git a/public/css/fontello.css b/public/css/fontello.css index eb41de7..8f9abad 100644 --- a/public/css/fontello.css +++ b/public/css/fontello.css @@ -1,12 +1,12 @@ @font-face { font-family: "fontello"; - src: url("/fonts/fontello.eot?49059696"); + src: url("/fonts/fontello.eot?59696369"); src: - url("/fonts/fontello.eot?49059696#iefix") format("embedded-opentype"), - url("/fonts/fontello.woff2?49059696") format("woff2"), - url("/fonts/fontello.woff?49059696") format("woff"), - url("/fonts/fontello.ttf?49059696") format("truetype"), - url("/fonts/fontello.svg?49059696#fontello") format("svg"); + url("/fonts/fontello.eot?59696369#iefix") format("embedded-opentype"), + url("/fonts/fontello.woff2?59696369") format("woff2"), + url("/fonts/fontello.woff?59696369") format("woff"), + url("/fonts/fontello.ttf?59696369") format("truetype"), + url("/fonts/fontello.svg?59696369#fontello") format("svg"); font-weight: normal; font-style: normal; } @@ -36,113 +36,118 @@ -moz-osx-font-smoothing: grayscale; } +.icon-group:before { + content: "\e0c3"; +} + +/* '' */ .icon-views:before { content: "\e800"; } -/* '' */ +/* '' */ .icon-heart:before { content: "\e801"; } -/* '' */ +/* '' */ .icon-quote:before { content: "\e802"; } -/* '' */ +/* '' */ .icon-comment:before { content: "\e803"; } -/* '' */ -.icon-group:before { - content: "\e804"; -} - -/* '' */ +/* '' */ .icon-play:before { content: "\e805"; } -/* '' */ +/* '' */ .icon-link:before { content: "\e806"; } -/* '' */ +/* '' */ .icon-calendar:before { content: "\e807"; } -/* '' */ +/* '' */ .icon-location:before { content: "\e808"; } -/* '' */ +/* '' */ .icon-picture:before { content: "\e809"; } -/* '' */ +/* '' */ .icon-lock:before { content: "\e80a"; } -/* '' */ +/* '' */ .icon-down:before { content: "\e80b"; } -/* '' */ +/* '' */ .icon-retweet:before { content: "\e80c"; } -/* '' */ +/* '' */ .icon-search:before { content: "\e80d"; } -/* '' */ +/* '' */ .icon-pin:before { content: "\e80e"; } -/* '' */ +/* '' */ .icon-cog:before { content: "\e80f"; } -/* '' */ +/* '' */ .icon-rss:before { content: "\e810"; } -/* '' */ +/* '' */ .icon-ok:before { content: "\e811"; } -/* '' */ -.icon-attention:before { +/* '' */ +.icon-attention-circled:before { content: "\e812"; } -/* '' */ +/* '' */ +.icon-download-alt:before { + content: "\e813"; +} + +/* '' */ .icon-circle:before { content: "\f111"; } -/* '' */ +/* '' */ .icon-info:before { content: "\f128"; } -/* '' */ +/* '' */ .icon-bird:before { content: "\f309"; } -/* '' */ +/* '' */ diff --git a/public/fonts/fontello.eot b/public/fonts/fontello.eot index ed1dba8..a5b8b1c 100644 Binary files a/public/fonts/fontello.eot and b/public/fonts/fontello.eot differ diff --git a/public/fonts/fontello.svg b/public/fonts/fontello.svg index 19db3bd..5dc65a8 100644 --- a/public/fonts/fontello.svg +++ b/public/fonts/fontello.svg @@ -6,6 +6,8 @@ + + @@ -14,8 +16,6 @@ - - @@ -44,6 +44,8 @@ + + diff --git a/public/fonts/fontello.ttf b/public/fonts/fontello.ttf index be34c4e..a2b972e 100644 Binary files a/public/fonts/fontello.ttf and b/public/fonts/fontello.ttf differ diff --git a/public/fonts/fontello.woff b/public/fonts/fontello.woff index 699b2ba..65508ca 100644 Binary files a/public/fonts/fontello.woff and b/public/fonts/fontello.woff differ diff --git a/public/fonts/fontello.woff2 b/public/fonts/fontello.woff2 index cfe8dfb..a8d96da 100644 Binary files a/public/fonts/fontello.woff2 and b/public/fonts/fontello.woff2 differ diff --git a/public/js/hlsPlayback.js b/public/js/hlsPlayback.js index 9919fec..5970011 100644 --- a/public/js/hlsPlayback.js +++ b/public/js/hlsPlayback.js @@ -1,27 +1,30 @@ // @license http://www.gnu.org/licenses/agpl-3.0.html AGPL-3.0 // SPDX-License-Identifier: AGPL-3.0-only -function playVideo(overlay) { - const video = overlay.parentElement.querySelector('video'); - const url = video.getAttribute("data-url"); - const startTime = parseFloat(video.getAttribute("data-start") || "0"); - video.setAttribute("controls", ""); +function playMedia(overlay, tagName) { + const media = overlay.parentElement.querySelector(tagName); + const url = media.getAttribute("data-url"); + const startTime = parseFloat(media.getAttribute("data-start") || "0"); + media.setAttribute("controls", ""); overlay.style.display = "none"; if (Hls.isSupported()) { var hls = new Hls({autoStartLoad: false}); hls.loadSource(url); - hls.attachMedia(video); + hls.attachMedia(media); hls.on(Hls.Events.MANIFEST_PARSED, function () { hls.loadLevel = hls.levels.length - 1; hls.startLoad(startTime); - video.play(); + media.play(); }); - } else if (video.canPlayType('application/vnd.apple.mpegurl')) { - video.src = url; - video.addEventListener('canplay', function() { - if (startTime > 0) video.currentTime = startTime; - video.play(); + } else if (media.canPlayType('application/vnd.apple.mpegurl')) { + media.src = url; + media.addEventListener('canplay', function() { + if (startTime > 0) media.currentTime = startTime; + media.play(); }); } } + +function playVideo(overlay) { playMedia(overlay, 'video'); } +function playAudio(overlay) { playMedia(overlay, 'audio'); } // @license-end diff --git a/src/api.nim b/src/api.nim index f3c6c12..f73ef10 100644 --- a/src/api.nim +++ b/src/api.nim @@ -2,7 +2,7 @@ import asyncdispatch, httpclient, strutils, sequtils, sugar import packedjson import types, query, formatters, consts, apiutils, parser, utils -import experimental/parser as newParser +import experimental/parser # Helper to generate params object for GraphQL requests proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] = @@ -11,13 +11,18 @@ proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] = if fieldToggles.len > 0: result.add ("fieldToggles", fieldToggles) -proc apiUrl(endpoint, variables: string; fieldToggles = ""): ApiUrl = - return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles)) +proc apiUrl(endpoint, variables: string; fieldToggles = ""; skipTid = false): ApiUrl = + return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles), skipTid: skipTid) -proc apiReq(endpoint, variables: string; fieldToggles = ""): ApiReq = - let url = apiUrl(endpoint, variables, fieldToggles) +proc apiReq(endpoint, variables: string; fieldToggles = ""; skipTid = false): ApiReq = + let url = apiUrl(endpoint, variables, fieldToggles, skipTid) return ApiReq(cookie: url, oauth: url) +proc cursorParam(after: string): string = + ## JSON-escape the user-supplied cursor so it cannot break out of the GraphQL + ## variables object (same input-validation class as the #1411 media SSRF). + if after.len > 0: "\"cursor\":" & $(%after) & "," else: "" + proc mediaUrl(id, cursor: string; count=20): ApiReq = result = ApiReq( cookie: apiUrl(graphUserMedia, userMediaVars % [id, cursor, $count]), @@ -25,33 +30,24 @@ proc mediaUrl(id, cursor: string; count=20): ApiReq = ) proc userTweetsUrl(id: string; cursor: string): ApiReq = - result = ApiReq( - # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles), - oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"]) - ) - # might change this in the future pending testing - result.cookie = result.oauth + return apiReq(graphUserTweetsV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles) proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq = - let cookieVars = userTweetsAndRepliesVars % [id, cursor] - result = ApiReq( - cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles), - oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"]) - ) + return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles, skipTid=true) proc tweetDetailUrl(id: string; cursor: string): ApiReq = - let cookieVars = tweetDetailVars % [id, cursor] - result = ApiReq( - # cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles), - cookie: apiUrl(graphTweet, tweetVars % [id, cursor]), - oauth: apiUrl(graphTweet, tweetVars % [id, cursor]) - ) + return apiReq(graphTweet, tweetVars % [id, cursor]) + # let cookieVars = tweetDetailVars % [id, cursor] + # result = ApiReq( + # cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles), + # oauth: apiUrl(graphTweet, tweetVars % [id, cursor]) + # ) proc userUrl(username: string): ApiReq = - let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username + let cookieVars = $(%*{"screen_name": username, "withGrokTranslatedBio": false}) result = ApiReq( cookie: apiUrl(graphUser, cookieVars, tweetDetailFieldToggles), - oauth: apiUrl(graphUserV2, """{"screen_name": "$1"}""" % username) + oauth: apiUrl(graphUserV2, $(%*{"screen_name": username})) ) proc getGraphUser*(username: string): Future[User] {.async.} = @@ -69,7 +65,7 @@ proc getGraphUserById*(id: string): Future[User] {.async.} = proc getAboutAccount*(username: string): Future[AccountInfo] {.async.} = if username.len == 0: return let - url = apiReq(graphAboutAccount, """{"screenName":"$1"}""" % username) + url = apiReq(graphAboutAccount, $(%*{"screenName": username})) js = await fetch(url) result = parseAboutAccount(js) @@ -80,7 +76,7 @@ proc restReq(endpoint: string; params: seq[(string, string)] = @[]): ApiReq = proc getBroadcastInfo*(id: string): Future[Broadcast] {.async.} = if id.len == 0: return let - req = apiReq(graphBroadcast, """{"id":"$1"}""" % id) + req = apiReq(graphBroadcast, $(%*{"id": id})) js = await fetch(req) result = parseBroadcastInfo(js) @@ -92,10 +88,23 @@ proc fetchBroadcastStream*(mediaKey: string): Future[string] {.async.} = result = streamJs{"source", "noRedirectPlaybackUrl"}.getStr( streamJs{"source", "location"}.getStr) +proc getAudioSpace*(id: string): Future[AudioSpace] {.async.} = + if id.len == 0: return + let + variables = %*{ + "id": id, + "isMetatagsQuery": false, + "withReplays": true, + "withListeners": true + } + req = apiReq(graphAudioSpace, $variables) + js = await fetch(req) + result = parseAudioSpace(js) + proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} = if id.len == 0: return let - cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + cursor = cursorParam(after) url = case kind of TimelineKind.tweets: userTweetsUrl(id, cursor) of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor) @@ -103,10 +112,61 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi js = await fetch(url) result = parseGraphTimeline(js, after) +proc getGraphCommunity*(id: string): Future[Community] {.async.} = + if id.len == 0: return + let + url = apiReq(graphCommunity, $(%*{"communityId": id})) + js = await fetch(url) + result = parseGraphCommunity(js) + +proc getGraphCommunityTweets*(id: string; rankingMode: string; after=""): Future[Timeline] {.async.} = + if id.len == 0: return + let + cursor = cursorParam(after) + url = apiReq(graphCommunityTweets, communityTweetsVars % [id, cursor, rankingMode]) + js = await fetch(url) + result = parseGraphCommunityTimeline(js, after) + +proc getGraphCommunityMedia*(id: string; after=""): Future[Timeline] {.async.} = + if id.len == 0: return + let + cursor = cursorParam(after) + url = apiReq(graphCommunityMedia, communityMediaVars % [id, cursor]) + js = await fetch(url) + result = parseGraphCommunityTimeline(js, after) + +proc communitySliceReq(endpoint, variables: string): ApiReq = + let url = ApiUrl(endpoint: endpoint, params: @[("variables", variables)]) + ApiReq(cookie: url, oauth: url) + +proc getGraphCommunityMembers*(id: string; after=""): Future[Result[User]] {.async.} = + if id.len == 0: return + let + cursor = if after.len > 0: $(%after) else: "null" + url = communitySliceReq(graphCommunityMembers, communityMembersVars % [id, cursor]) + js = await fetch(url) + result = parseGraphCommunityMembers(js, after) + +proc getGraphCommunityModerators*(id: string): Future[Result[User]] {.async.} = + if id.len == 0: return + let + url = communitySliceReq(graphCommunityModerators, communityMembersVars % [id, "null"]) + js = await fetch(url) + result = parseGraphCommunityMembers(js) + +proc getGraphCommunityHashtags*(id, hashtag: string; after=""): Future[Timeline] {.async.} = + if id.len == 0 or hashtag.len == 0: return + let + safeTag = multiReplace(hashtag, ("\"", ""), ("\\", "")) + cursor = cursorParam(after) + url = apiReq(graphCommunityHashtags, communityHashtagsVars % [id, cursor, safeTag]) + js = await fetch(url) + result = parseGraphCommunityTimeline(js, after) + proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return let - cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + cursor = cursorParam(after) url = apiReq(graphListTweets, restIdVars % [id, cursor, "20"]) js = await fetch(url) result = parseGraphTimeline(js, after).tweets @@ -120,7 +180,7 @@ proc getGraphListBySlug*(name, list: string): Future[List] {.async.} = proc getGraphList*(id: string): Future[List] {.async.} = let - url = apiReq(graphListById, """{"listId": "$1"}""" % id) + url = apiReq(graphListById, $(%*{"listId": id})) js = await fetch(url) result = parseGraphList(js) @@ -141,17 +201,39 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} js = await fetchRaw(url) result = parseGraphListMembers(js, after) +proc getGraphUserConnections(userId: string; endpoint: string; kind: QueryKind; + after=""): Future[Result[User]] {.async.} = + if userId.len == 0: return + var variables = %*{ + "userId": userId, + "count": 20, + "includePromotedContent": false, + "withGrokTranslatedBio": true + } + if after.len > 0: + variables["cursor"] = %after + let + url = apiReq(endpoint, $variables) + js = await fetchRaw(url) + result = parseGraphFollowers(js, after, kind) + +proc getGraphFollowers*(userId: string; after=""): Future[Result[User]] {.async.} = + result = await getGraphUserConnections(userId, graphFollowers, followers, after) + +proc getGraphFollowing*(userId: string; after=""): Future[Result[User]] {.async.} = + result = await getGraphUserConnections(userId, graphFollowing, following, after) + proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} = if id.len == 0: return let - url = apiReq(graphTweetResult, """{"rest_id": "$1"}""" % id) + url = apiReq(graphTweetResult, $(%*{"rest_id": id})) js = await fetch(url) result = parseGraphTweetResult(js) proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} = if id.len == 0: return let - cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + cursor = cursorParam(after) js = await fetch(tweetDetailUrl(id, cursor)) result = parseGraphConversation(js, id) @@ -184,13 +266,13 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} = var variables = %*{ "rawQuery": q, - "query_source": "typedQuery", "count": 20, + "querySource": "typed_query", "product": "Latest", - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false + "withGrokTranslatedBio":true, + "withQuickPromoteEligibilityTweetFields":false } + if after.len > 0 and maxId.len == 0: variables["cursor"] = % after let @@ -212,12 +294,11 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} var variables = %*{ "rawQuery": query.text, - "query_source": "typedQuery", "count": 20, + "querySource": "typed_query", "product": "People", - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false + "withGrokTranslatedBio":true, + "withQuickPromoteEligibilityTweetFields":false } if after.len > 0: variables["cursor"] = % after @@ -234,6 +315,21 @@ proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} = let js = await fetch(mediaUrl(id, "", 30)) result = parseGraphPhotoRail(js) +proc getGraphArticle*(id: string): Future[Article] {.async.} = + if id.len == 0: return + let + url = apiReq(graphTweetResultByRestId, articleVars % id, articleFieldToggles) + json = await fetchRaw(url) + result = parseGraphArticle(json) + +proc getGraphTweetResults*(ids: seq[string]): Future[seq[Tweet]] {.async.} = + if ids.len == 0: return + let + idsJson = "[" & ids.mapIt("\"" & it & "\"").join(",") & "]" + url = apiReq(graphTweetResultsByRestIds, articleBatchVars % idsJson, articleFieldToggles) + js = await fetch(url) + result = parseGraphTweetResults(js) + proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} = let client = newAsyncHttpClient(maxRedirects=0) try: diff --git a/src/apiutils.nim b/src/apiutils.nim index d6952e8..b22de41 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -1,6 +1,6 @@ # SPDX-License-Identifier: AGPL-3.0-only import httpclient, asyncdispatch, options, strutils, uri, times, math, tables -import jsony, packedjson, zippy, oauth1 +import jsony, packedjson, zippy, oauth/oauth1 import types, auth, consts, parserutils, http_pool, tid import experimental/types/common @@ -8,6 +8,7 @@ const rlRemaining = "x-rate-limit-remaining" rlReset = "x-rate-limit-reset" rlLimit = "x-rate-limit-limit" + npCache = "x-np-cache" errorsToSkip = {null, doesntExist, tweetNotFound, timeout, unauthorized, badRequest} var @@ -63,7 +64,7 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string = proc getCookieHeader(authToken, ct0: string): string = "auth_token=" & authToken & "; ct0=" & ct0 -proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} = +proc genHeaders*(session: Session, url: Uri, skipTid: bool): Future[HttpHeaders] {.async.} = result = newHttpHeaders({ "accept": "*/*", "accept-encoding": "gzip", @@ -84,13 +85,14 @@ proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} = result["x-twitter-auth-type"] = "OAuth2Session" result["x-csrf-token"] = session.ct0 result["cookie"] = getCookieHeader(session.authToken, session.ct0) + result["referer"] = "https://x.com/" result["sec-ch-ua"] = """"Google Chrome";v="142", "Chromium";v="142", "Not A(Brand";v="24"""" result["sec-ch-ua-mobile"] = "?0" result["sec-ch-ua-platform"] = "Windows" result["sec-fetch-dest"] = "empty" result["sec-fetch-mode"] = "cors" - result["sec-fetch-site"] = "same-site" - if disableTid or "/1.1/" in url.path: + result["sec-fetch-site"] = "same-origin" + if disableTid or skipTid or "/1.1/" in url.path: result["authorization"] = bearerToken2 else: result["authorization"] = bearerToken @@ -114,7 +116,12 @@ template fetchImpl(result, fetchBody) {.dirty.} = try: var resp: AsyncResponse - pool.use(await genHeaders(session, url)): + let skipTid = case session.kind + of oauth: req.oauth.skipTid + of cookie: req.cookie.skipTid + let headers = await genHeaders(session, url, skipTid) + + pool.use(headers): template getContent = # TODO: this is a temporary simple implementation if apiProxy.len > 0 and "/1.1/" notin url.path: @@ -130,10 +137,11 @@ template fetchImpl(result, fetchBody) {.dirty.} = raise newException(BadClientError, "Bad client") if resp.status == $Http404 and result.len == 0: - echo "[sessions] transient 404 (empty body), retrying: ", url.path + echo "[sessions] transient 404 (empty body), retrying: ", url.path, ", session: ", session.pretty raise rateLimitError() - if resp.headers.hasKey(rlRemaining): + let cacheStatus = resp.headers.getOrDefault(npCache) + if cacheStatus notin ["HIT", "STALE"] and resp.headers.hasKey(rlRemaining): let remaining = parseInt(resp.headers[rlRemaining]) reset = parseInt(resp.headers[rlReset]) @@ -147,7 +155,7 @@ template fetchImpl(result, fetchBody) {.dirty.} = if result.startsWith("{\"errors"): let errors = result.fromJson(Errors) if errors notin errorsToSkip: - echo "Fetch error, API: ", url.path, ", errors: ", errors + echo "Fetch error, API: ", url.path, ", errors: ", errors, ", session: ", session.pretty if errors in {expiredToken, badToken, locked}: invalidate(session) raise rateLimitError() @@ -162,7 +170,7 @@ template fetchImpl(result, fetchBody) {.dirty.} = fetchBody if resp.status == $Http400: - echo "ERROR 400, ", url.path, ": ", result + echo "ERROR 400, ", url.path, ": ", result, ", session: ", session.pretty raise newException(InternalError, $url) except InternalError as e: raise e @@ -177,22 +185,34 @@ template fetchImpl(result, fetchBody) {.dirty.} = finally: release(session) -template retry(bod) = +template retry(bod) {.dirty.} = + var session: Session + var retrySuccess = false for i in 0 ..< maxRetries: try: + session = nil bod + retrySuccess = true break except RateLimitError: - echo "[sessions] Rate limited, retrying ", req.cookie.endpoint, - " request (", i, "/", maxRetries, ")..." + let api = if session.isNil: req.cookie.endpoint + else: req.endpoint(session) + if session.isNil: + echo "[sessions] Rate limited, retrying ", api, + " request (", i, "/", maxRetries, ")..." + else: + echo "[sessions] Rate limited, retrying ", api, + " request (", i, "/", maxRetries, ")..., session: ", session.pretty + session = nil if retryDelayMs > 0: await sleepAsync(retryDelayMs) + if not retrySuccess: + raise rateLimitError() proc fetch*(req: ApiReq): Future[JsonNode] {.async.} = retry: - var - body: string - session = await getAndValidateSession(req) + var body: string + session = await getAndValidateSession(req) let url = req.toUrl(session.kind) @@ -200,22 +220,22 @@ proc fetch*(req: ApiReq): Future[JsonNode] {.async.} = if body.startsWith('{') or body.startsWith('['): result = parseJson(body) else: - echo resp.status, ": ", body, " --- url: ", url + echo resp.status, ": ", body, " --- url: ", url, ", session: ", session.pretty result = newJNull() let error = result.getError if error != null and error notin errorsToSkip: - echo "Fetch error, API: ", url.path, ", error: ", error + echo "Fetch error, API: ", url.path, ", error: ", error, ", session: ", session.pretty if error in {expiredToken, badToken, locked}: invalidate(session) raise rateLimitError() proc fetchRaw*(req: ApiReq): Future[string] {.async.} = retry: - var session = await getAndValidateSession(req) + session = await getAndValidateSession(req) let url = req.toUrl(session.kind) fetchImpl result: if not (result.startsWith('{') or result.startsWith('[')): - echo resp.status, ": ", result, " --- url: ", url + echo resp.status, ": ", result, " --- url: ", url, ", session: ", session.pretty result.setLen(0) diff --git a/src/auth.nim b/src/auth.nim index d801489..259c360 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -18,7 +18,7 @@ proc setMaxConcurrentReqs*(reqs: int) = template log(str: varargs[string, `$`]) = echo "[sessions] ", str.join("") -proc endpoint(req: ApiReq; session: Session): string = +proc endpoint*(req: ApiReq; session: Session): string = case session.kind of oauth: req.oauth.endpoint of cookie: req.cookie.endpoint @@ -50,6 +50,8 @@ proc getSessionPoolHealth*(): JsonNode = oldest = now.int64 newest = 0'i64 average = 0'i64 + oauthTotal, cookieTotal = 0 + oauthLimited, cookieLimited = 0 for session in sessionPool: let created = snowflakeToEpoch(session.id) @@ -59,8 +61,15 @@ proc getSessionPoolHealth*(): JsonNode = oldest = created average += created + case session.kind + of oauth: inc oauthTotal + of cookie: inc cookieTotal + if session.limited: limited.incl session.id + case session.kind + of oauth: inc oauthLimited + of cookie: inc cookieLimited for api in session.apis.keys: let @@ -84,6 +93,8 @@ proc getSessionPoolHealth*(): JsonNode = "sessions": %*{ "total": sessionPool.len, "limited": limited.card, + "oauth": %*{"total": oauthTotal, "limited": oauthLimited}, + "cookie": %*{"total": cookieTotal, "limited": cookieLimited}, "oldest": $fromUnix(oldest), "newest": $fromUnix(newest), "average": $fromUnix(average) @@ -100,6 +111,7 @@ proc getSessionPoolDebug*(): JsonNode = for session in sessionPool: let sessionJson = %*{ + "kind": $session.kind, "apis": newJObject(), "pending": session.pending, } @@ -173,7 +185,10 @@ proc getSession*(req: ApiReq): Future[Session] {.async.} = if not result.isNil and result.isReady(req): inc result.pending else: - log "no sessions available for API: ", req.cookie.endpoint + if result.isNil: + log "no sessions available for API: ", req.cookie.endpoint + else: + log "no sessions available for API: ", req.endpoint(result), ", last tried: ", result.pretty raise noSessionsError() proc setLimited*(session: Session; req: ApiReq) = diff --git a/src/consts.nim b/src/consts.nim index 29a582b..899a06c 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -7,109 +7,86 @@ const bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" bearerToken2* = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F" - graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName" - graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery" - graphUserById* = "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery" - graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2" - graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2" - graphUserTweets* = "oRJs8SLCRNRbQzuZG93_oA/UserTweets" - graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies" - graphUserMedia* = "36oKqyQ7E_9CmtONGjJRsA/UserMedia" - graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2" - graphTweet* = "b4pV7sWOe97RncwHcGESUA/ConversationTimeline" - graphTweetDetail* = "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail" - graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery" - graphTweetEditHistory* = "upS9teTSG45aljmP9oTuXA/TweetEditHistory" - graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline" - graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId" - graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug" - graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers" - graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline" - graphAboutAccount* = "zs_jFPFT78rBpXv9Z3U2YQ/AboutAccountQuery" + graphUser* = "IGgvgiOx4QZndDHuD3x9TQ/UserByScreenName" + graphUserV2* = "-ZzAG_Bckx16LMbEvHC3lg/UserResultByScreenNameQuery" + graphUserById* = "-DAaa9jPxPswYeI2fZ9rug/UserResultByIdQuery" + graphUserTweetsV2* = "LE3eTyeqhBh2g-fX85O2eQ/UserWithProfileTweetsQueryV2" + graphUserTweetsAndRepliesV2* = "AcYHjc_YAx-9_rKWdMsKvA/UserWithProfileTweetsAndRepliesQueryV2" + graphUserTweets* = "PNd0vlufvrcIwrAnBYKE9g/UserTweets" + graphUserTweetsAndReplies* = "EqtpEwt0CoQXmDfq5DKH0A/UserTweetsAndReplies" + graphUserMedia* = "g_rGPF0fLON-M9cyVjXuzA/UserMedia" + graphUserMediaV2* = "WK111rbR0vM0ZX4lyZCYjw/MediaTimelineV2" + graphTweet* = "OZMbEnEa96AN8Pq6HyTWdw/ConversationTimeline" + graphTweetDetail* = "6uCvnic3m5reVuehkvHa3w/TweetDetail" + graphTweetResult* = "xYOrBQoTlfKJJPsX76MZEw/TweetResultByIdQuery" + graphTweetEditHistory* = "MGElmrYILE8wUfI8GorUYA/TweetEditHistory" + graphSearchTimeline* = "-TFXKoMnMTKdEXcCn-eahw/SearchTimeline" - graphBroadcast* = "0nMmbMh-_JwwRRFNXkyH3Q/BroadcastQuery" + graphListById* = "t9AbdyHaJVfjL9jsODwgpQ/ListByRestId" + graphListBySlug* = "LDQpQ89B5ipR8izCKrWU0g/ListBySlug" + graphListMembers* = "EM7YRaM3gCnzDESmchA7RA/ListMembers" + graphListTweets* = "0QJtcuMzVywHGAWD6Dtjlw/ListTimeline" + graphAboutAccount* = "zUnx-DLN9dkwOkNhTLySjg/AboutAccountQuery" + + graphCommunity* = "-ElI1vg3dYbttVMhBhGdLw/CommunityQuery" + graphCommunityTweets* = "Mvs5UOOEkpXVMDZtUcxR-Q/CommunityTweetsTimeline" + graphCommunityMedia* = "Bt9XYnY7D3OcmZE5lhdx-A/CommunityMediaTimeline" + graphCommunityMembers* = "WSbJGJjZaVasSj9bnqSZSA/membersSliceTimeline_Query" + graphCommunityModerators* = "GBMT3GOWy5dYsYC4XJfvow/moderatorsSliceTimeline_Query" + graphCommunityHashtags* = "40DyrMxfCknGuZwE-keW_Q/CommunityHashtagsTimeline" + + graphTweetResultByRestId* = "qtXMy1p5Y62uCskc_NUPJw/TweetResultByRestId" + graphTweetResultsByRestIds* = "Sc9EUQTZNEH-wzegn-nHvQ/TweetResultsByRestIds" + + graphBroadcast* = "FJLCzpXCLPM1jUZqmM7oEA/BroadcastQuery" + graphAudioSpace* = "rWRLsOhNJ2xjpI1tREYurQ/AudioSpaceById" restLiveStream* = "1.1/live_video_stream/status/" + graphFollowers* = "9jsVJ9l2uXUIKslHvJqIhw/Followers" + graphFollowing* = "OLm4oHZBfqWx8jbcEhWoFw/Following" + gqlFeatures* = """{ - "android_ad_formats_media_component_render_overlay_enabled": false, - "android_graphql_skip_api_media_color_palette": false, - "android_professional_link_spotlight_display_enabled": false, - "articles_api_enabled": false, - "articles_preview_enabled": true, - "blue_business_profile_image_shape_enabled": false, - "c9s_tweet_anatomy_moderator_badge_enabled": true, - "commerce_android_shop_module_enabled": false, - "communities_web_enable_tweet_community_results_fetch": true, - "creator_subscriptions_quote_tweet_preview_enabled": false, - "creator_subscriptions_subscription_count_enabled": false, - "creator_subscriptions_tweet_preview_api_enabled": true, - "freedom_of_speech_not_reach_fetch_enabled": true, - "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, - "grok_android_analyze_trend_fetch_enabled": false, - "grok_translations_community_note_auto_translation_is_enabled": false, - "grok_translations_community_note_translation_is_enabled": false, - "grok_translations_post_auto_translation_is_enabled": false, - "grok_translations_timeline_user_bio_auto_translation_is_enabled": false, - "hidden_profile_likes_enabled": false, - "highlights_tweets_tab_ui_enabled": false, - "immersive_video_status_linkable_timestamps": false, - "interactive_text_enabled": false, - "longform_notetweets_consumption_enabled": true, - "longform_notetweets_inline_media_enabled": true, - "longform_notetweets_richtext_consumption_enabled": true, - "longform_notetweets_rich_text_read_enabled": true, - "mobile_app_spotlight_module_enabled": false, - "payments_enabled": false, - "post_ctas_fetch_enabled": true, - "premium_content_api_read_enabled": false, + "rweb_video_screen_enabled": false, + "rweb_cashtags_enabled": true, "profile_label_improvements_pcf_label_in_post_enabled": true, - "profile_label_improvements_pcf_label_in_profile_enabled": false, - "responsive_web_edit_tweet_api_enabled": true, - "responsive_web_enhance_cards_enabled": false, - "responsive_web_graphql_exclude_directive_enabled": true, - "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, + "responsive_web_profile_redirect_enabled": false, + "rweb_tipjar_consumption_enabled": false, + "verified_phone_label_enabled": false, + "creator_subscriptions_tweet_preview_api_enabled": true, "responsive_web_graphql_timeline_navigation_enabled": true, - "responsive_web_grok_analysis_button_from_backend": true, + "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, + "premium_content_api_read_enabled": false, + "communities_web_enable_tweet_community_results_fetch": true, + "c9s_tweet_anatomy_moderator_badge_enabled": true, + "c9s_list_members_action_api_enabled": false, + "c9s_superc9s_indication_enabled": false, "responsive_web_grok_analyze_button_fetch_trends_enabled": false, "responsive_web_grok_analyze_post_followups_enabled": true, + "rweb_cashtags_composer_attachment_enabled": true, + "responsive_web_jetfuel_frame": true, + "responsive_web_grok_share_attachment_enabled": true, "responsive_web_grok_annotations_enabled": true, - "responsive_web_grok_community_note_auto_translation_is_enabled": false, + "articles_preview_enabled": true, + "responsive_web_edit_tweet_api_enabled": true, + "rweb_conversational_replies_downvote_enabled": false, + "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, + "view_counts_everywhere_api_enabled": true, + "longform_notetweets_consumption_enabled": true, + "responsive_web_twitter_article_tweet_consumption_enabled": true, + "content_disclosure_indicator_enabled": true, + "content_disclosure_ai_generated_indicator_enabled": true, + "responsive_web_grok_show_grok_translated_post": true, + "responsive_web_grok_analysis_button_from_backend": true, + "post_ctas_fetch_enabled": true, + "freedom_of_speech_not_reach_fetch_enabled": true, + "standardized_nudges_misinfo": true, + "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, + "longform_notetweets_rich_text_read_enabled": true, + "longform_notetweets_inline_media_enabled": false, "responsive_web_grok_image_annotation_enabled": true, "responsive_web_grok_imagine_annotation_enabled": true, - "responsive_web_grok_share_attachment_enabled": true, - "responsive_web_grok_show_grok_translated_post": false, - "responsive_web_jetfuel_frame": true, - "responsive_web_media_download_video_enabled": false, - "responsive_web_profile_redirect_enabled": false, - "responsive_web_text_conversations_enabled": false, - "responsive_web_twitter_article_notes_tab_enabled": false, - "responsive_web_twitter_article_tweet_consumption_enabled": true, - "responsive_web_twitter_blue_verified_badge_is_enabled": true, - "rweb_lists_timeline_redesign_enabled": true, - "rweb_tipjar_consumption_enabled": true, - "rweb_video_screen_enabled": false, - "rweb_video_timestamps_enabled": false, - "spaces_2022_h2_clipping": true, - "spaces_2022_h2_spaces_communities": true, - "standardized_nudges_misinfo": true, - "subscriptions_feature_can_gift_premium": false, - "subscriptions_verification_info_enabled": true, - "subscriptions_verification_info_is_identity_verified_enabled": false, - "subscriptions_verification_info_reason_enabled": true, - "subscriptions_verification_info_verified_since_enabled": true, - "super_follow_badge_privacy_enabled": false, - "super_follow_exclusive_tweet_notifications_enabled": false, - "super_follow_tweet_api_enabled": false, - "super_follow_user_api_enabled": false, - "tweet_awards_web_tipping_enabled": false, - "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, - "tweetypie_unmention_optimization_enabled": false, - "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false, - "unified_cards_destination_url_params_enabled": false, - "verified_phone_label_enabled": false, - "vibe_api_enabled": false, - "view_counts_everywhere_api_enabled": true, - "hidden_profile_subscriptions_enabled": false + "responsive_web_grok_community_note_auto_translation_is_enabled": true, + "responsive_web_enhance_cards_enabled": false }""".replace(" ", "").replace("\n", "") tweetVars* = """{ @@ -143,7 +120,7 @@ const restIdVars* = """{ "rest_id": "$1", $2 "count": $3 -}""" +}""".replace(" ", "").replace("\n", "") userMediaVars* = """{ "userId": "$1", $2 @@ -170,6 +147,50 @@ const "withVoice": true }""".replace(" ", "").replace("\n", "") + articleVars* = """{ + "tweetId": "$1", + "includePromotedContent": false, + "withBirdwatchNotes": true, + "withVoice": true, + "withCommunity": true +}""".replace(" ", "").replace("\n", "") + + articleBatchVars* = """{ + "tweetIds": $1, + "includePromotedContent": false, + "withBirdwatchNotes": true, + "withVoice": true, + "withCommunity": true +}""".replace(" ", "").replace("\n", "") + + articleFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withArticleSummaryText":true,"withArticleVoiceOver":true}""" + + communityTweetsVars* = """{ + "communityId": "$1", $2 + "count": 20, + "displayLocation": "Community", + "rankingMode": "$3", + "withCommunity": true +}""".replace(" ", "").replace("\n", "") + + communityMediaVars* = """{ + "communityId": "$1", $2 + "count": 20, + "withCommunity": true +}""".replace(" ", "").replace("\n", "") + + communityMembersVars* = """{ + "communityId": "$1", + "cursor": $2 +}""".replace(" ", "").replace("\n", "") + + communityHashtagsVars* = """{ + "communityId": "$1", $2 + "count": 20, + "hashtags": ["$3"], + "withCommunity": true +}""".replace(" ", "").replace("\n", "") + userFieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}""" - userTweetsFieldToggles* = """{"withArticlePlainText":false}""" + userTweetsFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false}""" tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}""" diff --git a/src/experimental/parser.nim b/src/experimental/parser.nim index 40986f5..e22a51f 100644 --- a/src/experimental/parser.nim +++ b/src/experimental/parser.nim @@ -1,2 +1,2 @@ -import parser/[user, graphql] -export user, graphql +import parser/[user, graphql, article] +export user, graphql, article diff --git a/src/experimental/parser/article.nim b/src/experimental/parser/article.nim new file mode 100644 index 0000000..ae5de7c --- /dev/null +++ b/src/experimental/parser/article.nim @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import std/[strutils, tables, times, options] +import jsony +import utils, graphql, ../types/article +from ../../types import Article, ArticleParagraph, ArticleEntity, ArticleMedia, + User, TweetStats + +proc parseGraphArticle*(json: string): Article = + if json.len == 0 or json[0] != '{': + return + + var raw: GraphArticle + try: + raw = json.fromJson(GraphArticle) + except CatchableError: + return + + let + tweet = raw.data.tweetResult.result + article = tweet.article.articleResults.result + + if article.title.len == 0: + return + + let publishedAt = article.metadata.firstPublishedAtSecs + var articleTime: DateTime + if publishedAt > 0: + articleTime = publishedAt.int64.fromUnix.utc + elif tweet.legacy.createdAt.len > 0: + articleTime = parseTwitterDate(tweet.legacy.createdAt) + + result = Article( + title: article.title, + coverImage: getImageUrl(article.coverMedia.mediaInfo.originalImgUrl), + time: articleTime, + user: parseUserResult(tweet.core.userResults.result), + ) + + result.stats = TweetStats( + replies: tweet.legacy.replyCount, + retweets: tweet.legacy.retweetCount, + likes: tweet.legacy.favoriteCount, + ) + if tweet.views.count.len > 0: + try: result.stats.views = parseInt(tweet.views.count) + except ValueError: discard + + for blk in article.contentState.blocks: + result.paragraphs.add ArticleParagraph( + text: blk.text, + kind: blk.blockKind, + inlineStyles: blk.inlineStyleRanges, + entityRanges: blk.entityRanges, + ) + + for entry in article.contentState.entityMap: + let key = try: parseInt(entry.key) except ValueError: continue + var entity = ArticleEntity(kind: entry.value.entityKind) + case entity.kind + of "LINK": entity.url = entry.value.data.url + of "MEDIA": + for mi in entry.value.data.mediaItems: + entity.mediaIds.add mi.mediaId + entity.caption = entry.value.data.caption + of "TWEET": entity.tweetId = entry.value.data.tweetId + of "MARKDOWN": entity.markdown = entry.value.data.markdown + else: discard + result.entities[key] = entity + + for me in article.mediaEntities: + let typeName = me.mediaInfo.typeName + var media = ArticleMedia(kind: typeName) + if me.mediaInfo.videoInfo.isSome: + let variants = me.mediaInfo.videoInfo.get.variants + case typeName + of "ApiGif": + if variants.len > 0: + media.url = variants[0].url + of "ApiVideo": + var bestBitrate = -1 + for v in variants: + if v.bitrate > bestBitrate: + bestBitrate = v.bitrate + media.url = v.url + else: discard + elif typeName == "ApiImage": + media.url = getImageUrl(me.mediaInfo.originalImgUrl) + result.media[me.mediaId] = media diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim index 65ebc3d..0a7484c 100644 --- a/src/experimental/parser/graphql.nim +++ b/src/experimental/parser/graphql.nim @@ -1,6 +1,6 @@ import options, strutils import jsony -import user, utils, ../types/[graphuser, graphlistmembers] +import user, utils, ../types/[graphuser, graphlistmembers, graphfollowers] from ../../types import User, VerifiedType, Result, Query, QueryKind proc parseUserResult*(userResult: UserResult): User = @@ -67,3 +67,25 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] = of TimelineTimelineCursor: if entry.content.cursorType == "Bottom": result.bottom = entry.content.value + +proc parseGraphFollowers*(json, cursor: string; kind: QueryKind): Result[User] = + result = Result[User]( + beginning: cursor.len == 0, + query: Query(kind: kind) + ) + + if json.len == 0 or json[0] != '{': + return + + let raw = json.fromJson(GraphFollowers) + for instruction in raw.data.user.result.timeline.timeline.instructions: + if instruction.kind == "TimelineAddEntries": + for entry in instruction.entries: + case entry.content.entryType + of TimelineTimelineItem: + let userResult = entry.content.itemContent.userResults.result + if userResult.restId.len > 0: + result.content.add parseUserResult(userResult) + of TimelineTimelineCursor: + if entry.content.cursorType == "Bottom": + result.bottom = entry.content.value diff --git a/src/experimental/types/article.nim b/src/experimental/types/article.nim new file mode 100644 index 0000000..946f8c3 --- /dev/null +++ b/src/experimental/types/article.nim @@ -0,0 +1,79 @@ +import std/options +import graphuser +from ../../types import ArticleStyle, ArticleEntityRange + +type + GraphArticle* = object + data*: tuple[tweetResult: tuple[result: TweetResultNode]] + + TweetResultNode* = object + article*: tuple[articleResults: tuple[result: ArticleResultNode]] + legacy*: TweetLegacy + core*: tuple[userResults: UserData] + views*: tuple[count: string] + + TweetLegacy* = object + createdAt*: string + replyCount*: int + retweetCount*: int + favoriteCount*: int + + ArticleResultNode* = object + title*: string + coverMedia*: tuple[mediaInfo: MediaInfoNode] + contentState*: ContentState + metadata*: tuple[firstPublishedAtSecs: int] + mediaEntities*: seq[RawMediaEntity] + + ContentState* = object + blocks*: seq[ContentBlock] + entityMap*: seq[EntityMapEntry] + + ContentBlock* = object + text*: string + blockKind*: string + inlineStyleRanges*: seq[ArticleStyle] + entityRanges*: seq[ArticleEntityRange] + + EntityMapEntry* = object + key*: string + value*: EntityMapValue + + EntityMapValue* = object + entityKind*: string + data*: EntityDataNode + + EntityDataNode* = object + url*: string + mediaItems*: seq[tuple[mediaId: string]] + tweetId*: string + markdown*: string + caption*: string + + RawMediaEntity* = object + mediaId*: string + mediaInfo*: MediaInfoNode + + MediaInfoNode* = object + typeName*: string + originalImgUrl*: string + videoInfo*: Option[VideoInfoNode] + + VideoInfoNode* = object + variants*: seq[VideoVariant] + + VideoVariant* = object + url*: string + bitrate*: int + +proc renameHook*(v: var ContentBlock; fieldName: var string) = + if fieldName == "type": + fieldName = "blockKind" + +proc renameHook*(v: var EntityMapValue; fieldName: var string) = + if fieldName == "type": + fieldName = "entityKind" + +proc renameHook*(v: var MediaInfoNode; fieldName: var string) = + if fieldName == "__typename": + fieldName = "typeName" diff --git a/src/experimental/types/graphfollowers.nim b/src/experimental/types/graphfollowers.nim new file mode 100644 index 0000000..ba9210b --- /dev/null +++ b/src/experimental/types/graphfollowers.nim @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import graphlistmembers + +type + GraphFollowers* = object + data*: tuple[user: UserWrapper] + + UserWrapper = object + result*: UserResultWrapper + + UserResultWrapper = object + timeline*: tuple[timeline: graphlistmembers.Timeline] + +# Hook to normalize snake_case field from API to camelCase used by shared types +proc renameHook*(v: var Content; fieldName: var string) = + if fieldName == "user_results": + fieldName = "userResults" diff --git a/src/experimental/types/graphlistmembers.nim b/src/experimental/types/graphlistmembers.nim index 4cb3757..9e520d8 100644 --- a/src/experimental/types/graphlistmembers.nim +++ b/src/experimental/types/graphlistmembers.nim @@ -7,10 +7,10 @@ type List = object membersTimeline*: tuple[timeline: Timeline] - Timeline = object + Timeline* = object instructions*: seq[Instruction] - Instruction = object + Instruction* = object kind*: string entries*: seq[tuple[content: Content]] @@ -18,7 +18,7 @@ type TimelineTimelineItem TimelineTimelineCursor - Content = object + Content* = object case entryType*: ContentEntryType of TimelineTimelineItem: itemContent*: tuple[userResults: UserData] diff --git a/src/formatters.nim b/src/formatters.nim index aef1c12..958e518 100644 --- a/src/formatters.nim +++ b/src/formatters.nim @@ -140,25 +140,30 @@ proc pageDesc*(user: User): string = "The latest tweets from " & user.fullname proc getJoinDate*(user: User): string = + if user.joinDate.year == 0: return "" user.joinDate.format("'Joined' MMMM YYYY") proc getJoinDateFull*(user: User): string = + if user.joinDate.year == 0: return "" user.joinDate.format("h:mm tt - d MMM YYYY") proc getTime*(tweet: Tweet): string = + if tweet.time.year == 0: return "" tweet.time.format("MMM d', 'YYYY' · 'h:mm tt' UTC'") proc getRfc822Time*(tweet: Tweet): string = + if tweet.time.year == 0: return "" tweet.time.format("ddd', 'dd MMM yyyy HH:mm:ss 'GMT'") -proc getShortTime*(tweet: Tweet): string = +proc getShortTime*(time: DateTime): string = + if time.year == 0: return "" let now = now() - let since = now - tweet.time + let since = now - time - if now.year != tweet.time.year: - result = tweet.time.format("d MMM yyyy") + if now.year != time.year: + result = time.format("d MMM yyyy") elif since.inDays >= 1: - result = tweet.time.format("MMM d") + result = time.format("MMM d") elif since.inHours >= 1: result = $since.inHours & "h" elif since.inMinutes >= 1: @@ -168,6 +173,9 @@ proc getShortTime*(tweet: Tweet): string = else: result = "now" +proc getShortTime*(tweet: Tweet): string = + getShortTime(tweet.time) + proc getDuration*(ms: int): string = let sec = int(round(ms / 1000)) diff --git a/src/nitter.nim b/src/nitter.nim index d1c0ef1..dc38161 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -2,15 +2,15 @@ import asyncdispatch, strformat, logging from net import Port from htmlgen import a -from os import getEnv +from os import getEnv, normalizedPath import jester import types, config, prefs, formatters, redis_cache, http_pool, auth, apiutils import views/[general, about] import routes/[ - preferences, timeline, status, media, search, rss, list, debug, - unsupported, embed, resolver, broadcast, router_utils] + preferences, timeline, status, media, search, rss, list, community, debug, + unsupported, embed, resolver, broadcast, space, article, router_utils] const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances" const issuesUrl = "https://github.com/zedeus/nitter/issues" @@ -34,6 +34,10 @@ stdout.flushFile updateDefaultPrefs(fullCfg) setCacheTimes(cfg) setHmacKey(cfg.hmacKey) +if cfg.hmacKey.len == 0 or cfg.hmacKey == "secretkey": + stderr.write "WARNING: insecure default 'hmacKey' in nitter.conf; " & + "set a unique random value to stop media URL signatures being forgeable.\n" + stderr.flushFile setProxyEncoding(cfg.base64Media) setMaxHttpConns(cfg.httpMaxConns) setHttpProxy(cfg.proxy, cfg.proxyAuth) @@ -48,27 +52,35 @@ waitFor initRedisPool(cfg) stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n" stdout.flushFile +createArticleRouter(cfg) createUnsupportedRouter(cfg) createResolverRouter(cfg) createPrefRouter(cfg) createTimelineRouter(cfg) createListRouter(cfg) +createCommunityRouter(cfg) createStatusRouter(cfg) createSearchRouter(cfg) createMediaRouter(cfg) createEmbedRouter(cfg) createRssRouter(cfg) createBroadcastRouter(cfg) +createSpaceRouter(cfg) createDebugRouter(cfg) settings: port = Port(cfg.port) - staticDir = cfg.staticDir + staticDir = normalizedPath(cfg.staticDir) bindAddr = cfg.address reusePort = true + maxBody = 64 * 1024 routes: before: + # Reject malformed paths + if request.path.len == 0 or request.path[0] != '/': + halt Http400 + # skip all file URLs cond "." notin request.path applyUrlPrefs() @@ -113,15 +125,18 @@ routes: resp Http429, showError( &"Instance has no auth tokens, or is fully rate limited.
Use {link} or try again later.", cfg) + extend articleRoute, "" extend rss, "" extend status, "" extend search, "" extend timeline, "" extend media, "" extend list, "" + extend community, "" extend preferences, "" extend resolver, "" extend embed, "" extend broadcastRoute, "" + extend spaceRoute, "" extend debug, "" extend unsupported, "" diff --git a/src/parser.nim b/src/parser.nim index d55c546..04a7d33 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -1,10 +1,10 @@ # SPDX-License-Identifier: AGPL-3.0-only -import strutils, options, times, math, tables +import strutils, options, times, math, tables, uri import packedjson, packedjson/deserialiser import types, parserutils, utils import experimental/parser/unifiedcard -proc parseGraphTweet(js: JsonNode): Tweet +proc parseGraphTweet*(js: JsonNode): Tweet proc parseVerifiedType(s: string; current: VerifiedType): VerifiedType = try: parseEnum[VerifiedType](s) @@ -46,10 +46,10 @@ proc parseUser(js: JsonNode; id=""): User = proc parseGraphUser(js: JsonNode): User = var user = js{"user_result", "result"} if user.isNull: - user = ? js{"user_results", "result"} + user = js{"user_results", "result"} if user.isNull: - if js{"core"}.notNull and js{"legacy"}.notNull: + if js{"core"}.notNull: user = js else: return @@ -61,6 +61,7 @@ proc parseGraphUser(js: JsonNode): User = # fallback to support UserMedia/recent GraphQL updates if result.username.len == 0: + result.id = user{"rest_id"}.getStr result.username = user{"core", "screen_name"}.getStr result.fullname = user{"core", "name"}.getStr result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "") @@ -130,6 +131,81 @@ proc parseBroadcastInfo*(js: JsonNode): Broadcast = user: parseGraphUser(bc) ) +proc parseSpaceParticipant(js: JsonNode): SpaceParticipant = + result = SpaceParticipant( + userId: js{"user_results", "rest_id"}.getStr, + username: js{"twitter_screen_name"}.getStr, + displayName: js{"display_name"}.getStr, + avatarUrl: js{"avatar_url"}.getStr, + isVerified: js{"is_verified"}.getBool or + js{"user_results", "result", "is_blue_verified"}.getBool + ) + +proc parseAudioSpace*(js: JsonNode): AudioSpace = + let space = ? js{"data", "audioSpace"} + let meta = space{"metadata"} + + result = AudioSpace( + id: meta{"rest_id"}.getStr, + title: meta{"title"}.getStr, + state: meta{"state"}.getStr.toUpperAscii, + mediaKey: meta{"media_key"}.getStr, + totalLiveListeners: meta{"total_live_listeners"}.getInt, + totalReplayWatched: meta{"total_replay_watched"}.getInt, + availableForReplay: meta{"is_space_available_for_replay"}.getBool + ) + + let startedAt = meta{"started_at"}.getInt(0) + if startedAt > 0: + result.startTime = fromUnix(startedAt div 1000).utc() + + let endedAtStr = meta{"ended_at"}.getStr + if endedAtStr.len > 0: + try: + let endedAt = parseBiggestInt(endedAtStr) + if endedAt > 0: + result.endTime = fromUnix(endedAt div 1000).utc() + except ValueError: + discard + + result.creator = parseGraphUser(meta{"creator_results", "result"}) + + for admin in space{"participants", "admins"}: + result.admins.add parseSpaceParticipant(admin) + + for speaker in space{"participants", "speakers"}: + result.speakers.add parseSpaceParticipant(speaker) + +proc parseGraphCommunity*(js: JsonNode): Community = + if js.isNull: return + let c = ? js{"data", "communityResults", "result"} + + result = Community( + id: c{"rest_id"}.getStr(c{"id_str"}.getStr), + name: c{"name"}.getStr, + description: c{"description"}.getStr, + memberCount: c{"member_count"}.getInt, + joinPolicy: c{"join_policy"}.getStr, + category: c{"primary_community_topic", "topic_name"}.getStr, + banner: c{"custom_banner_media", "media_info", "original_img_url"}.getImageStr, + creator: parseGraphUser(c{"creator_results", "result"}), + ) + + let createdMs = c{"created_at"}.getInt(0) + if createdMs > 0: + result.createdAt = fromUnix(createdMs div 1000).utc() + + for rule in c{"rules"}: + result.rules.add CommunityRule( + name: rule{"name"}.getStr, + description: rule{"description"}.getStr + ) + + for item in c{"trending_hashtags_slice", "items"}: + let tag = item{"hashtag"}.getStr + if tag.len > 0: + result.hashtags.add tag + proc parseGraphList*(js: JsonNode): List = if js.isNull: return @@ -228,6 +304,10 @@ proc parseLegacyMediaEntities(js: JsonNode; result: var Tweet) = result.attribution = some(parseUser(user)) else: result.attribution = some(parseGraphUser(user)) + # Set attribution link from expanded_url (strip /video/N suffix) + let expanded = m{"expanded_url"}.getStr + if expanded.len > 0: + result.attributionLink = expanded.parseUri.path.replace("/video/1", "") of "animated_gif": result.media.addMedia(Gif( url: m{"video_info", "variants"}[0]{"url"}.getImageStr, @@ -236,11 +316,6 @@ proc parseLegacyMediaEntities(js: JsonNode; result: var Tweet) = )) else: discard - with url, m{"url"}: - if result.text.endsWith(url.getStr): - result.text.removeSuffix(url.getStr) - result.text = result.text.strip() - proc parseMediaEntities(js: JsonNode; result: var Tweet) = with mediaEntities, js{"media_entities"}: var parsedMedia: MediaEntities @@ -261,6 +336,18 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) = durationMs: mediaInfo{"duration_millis"}.getInt, variants: parseVideoVariants(mediaInfo{"variants"}) )) + + # Parse source user for video attribution + with sourceUser, mediaEntity{"source_user_results", "result"}: + if result.attribution.isNone: + let expanded = mediaEntity{"expanded_url"}.getStr + if expanded.len > 0: + result.attributionLink = expanded.parseUri.path.replace("/video/1", "") + result.attribution = some(User( + id: sourceUser{"rest_id"}.getStr, + fullname: sourceUser{"core", "name"}.getStr, + userPic: sourceUser{"avatar", "image_url"}.getImageStr.replace("_normal", "") + )) of "ApiGif": parsedMedia.addMedia(Gif( url: mediaInfo{"variants"}[0]{"url"}.getImageStr, @@ -269,23 +356,9 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) = )) else: discard - if "expanded_url" in mediaEntity: - let expandedUrl = js.getExpandedUrl - if result.text.endsWith(expandedUrl): - result.text.removeSuffix(expandedUrl) - result.text = result.text.strip() - if mediaEntities.len > 0 and parsedMedia.len == mediaEntities.len: result.media = parsedMedia - # Remove media URLs from text - with mediaList, js{"legacy", "entities", "media"}: - for url in mediaList: - let expandedUrl = url.getExpandedUrl - if result.text.endsWith(expandedUrl): - result.text.removeSuffix(expandedUrl) - result.text = result.text.strip() - proc parsePromoVideo(js: JsonNode): Video = result = Video( thumb: js{"player_image_large"}.getImageVal, @@ -362,7 +435,13 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card = result.url = vals{"player_url"}.getStrVal if "youtube.com" in result.url: result.url = result.url.replace("/embed/", "/watch?v=") - of audiospace, unknown: + of audiospace: + let spaceId = vals{"id"}.getStrVal + if spaceId.len > 0: + result.url = "/i/spaces/" & spaceId + result.title = "Twitter Space" + result.text = "Click to view Space" + of unknown: result.title = "This card type is not supported." else: discard @@ -428,13 +507,13 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull(); # graphql with rt, js{"retweeted_status_result", "result"}: # needed due to weird edgecase where the actual tweet data isn't included - if "legacy" in rt: + if "legacy" in rt or "rest_id" in rt: result.retweet = some parseGraphTweet(rt) return with reposts, js{"repostedStatusResults"}: with rt, reposts{"result"}: - if "legacy" in rt: + if "legacy" in rt or "rest_id" in rt: result.retweet = some parseGraphTweet(rt) return @@ -449,7 +528,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull(); result.poll = some parsePoll(jsCard) elif name == "amplify": result.media.addMedia(parsePromoVideo(jsCard{"binding_values"})) - else: + elif name.len > 0 and jsCard{"binding_values"}.notNull: result.card = some parseCard(jsCard, js{"entities", "urls"}) result.expandTweetEntities(js) @@ -469,7 +548,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull(); result.text.removeSuffix(" Learn more.") result.available = false -proc parseGraphTweet(js: JsonNode): Tweet = +proc parseGraphTweet*(js: JsonNode): Tweet = if js.kind == JNull: return Tweet() @@ -506,7 +585,7 @@ proc parseGraphTweet(js: JsonNode): Tweet = "binding_values": %bindingObj } - var replyId = 0 + var replyId: int64 = 0 with restId, js{"reply_to_results", "rest_id"}: replyId = restId.getId @@ -537,14 +616,40 @@ proc parseGraphTweet(js: JsonNode): Tweet = result.poll = some parsePoll(jsCard) elif name == "amplify": result.media.addMedia(parsePromoVideo(jsCard{"binding_values"})) - else: + elif name.len > 0 and jsCard{"binding_values"}.notNull: result.card = some parseCard(jsCard, js{"url_entities"}) - result.expandTweetEntitiesV2(js) + parseMediaEntities(js, result) + if result.attribution.isNone: + parseLegacyMediaEntities(js{"legacy"}, result) + + let hasArticle = js{"article", "article_results", "result", "title"}.getStr.len > 0 + result.expandTweetEntitiesV2(js, hasArticle) + + # Strip video source URL from text (for videos from other tweets) + with mediaEntities, js{"media_entities"}: + for m in mediaEntities: + if "source_status_id_str" in m: + let mediaUrl = m{"url"}.getStr + if mediaUrl.len > 0: + let idx = result.text.rfind(mediaUrl) + if idx >= 0: + result.text = result.text[0 ..< idx].strip() + break else: result = parseTweet(js{"legacy"}, jsCard, replyId) result.id = js{"rest_id"}.getId + with artNode, js{"article", "article_results", "result"}: + let artTitle = artNode{"title"}.getStr + if artTitle.len > 0: + result.articlePreview = some ArticlePreview( + title: artTitle, + previewText: artNode{"preview_text"}.getStr, + coverImage: artNode{"cover_media_results", "result", "media_info", "original_img_url"}.getImageStr, + tweetId: result.id + ) + result.user = parseGraphUser(js{"core"}) if result.reply.len == 0: @@ -559,6 +664,20 @@ proc parseGraphTweet(js: JsonNode): Tweet = parseMediaEntities(js, result) + # Hide card if it's redundant with attribution (same video shown via embed) + if result.attribution.isSome and result.card.isSome: + let cardUri = get(result.card).url.parseUri + if cardUri.isTwitterUrl: + let cardPath = cardUri.path.replace("/video/1", "") + if cardPath.len > 0 and cardPath == result.attributionLink: + get(result.card).kind = hidden + + # Handle retweets - check both legacy and top-level paths + with reposts, js{"legacy", "repostedStatusResults"}: + with rt, reposts{"result"}: + if "legacy" in rt or "rest_id" in rt: + result.retweet = some parseGraphTweet(rt) + with quoted, js{"quoted_status_result", "result"}: result.quote = some(parseGraphTweet(quoted)) @@ -600,6 +719,16 @@ proc parseGraphTweetResult*(js: JsonNode): Tweet = with tweet, js{"data", "tweet_result", "result"}: result = parseGraphTweet(tweet) +proc parseGraphTweetResults*(js: JsonNode): seq[Tweet] = + let results = js{"data", "tweetResult"} + if results.kind != JArray: return + for item in results: + let tweet = item{"result"} + if tweet.isNull: continue + let t = parseGraphTweet(tweet) + if t != nil: + result.add t + proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation = result = Conversation(replies: Result[Chain](beginning: true)) @@ -807,3 +936,49 @@ proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] = elif typ == "TimelineReplaceEntry": if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"): result.bottom = instruction{"entry", "content", "value"}.getStr + +proc parseGraphCommunityTimeline*(js: JsonNode; after=""): Timeline = + result = Timeline(beginning: after.len == 0) + + let communityResult = js{"data", "communityResults", "result"} + let instructions = ? select( + communityResult{"ranked_community_timeline", "timeline", "instructions"}, + communityResult{"community_media_timeline", "timeline", "instructions"}, + communityResult{"community_filtered_timeline", "timeline", "instructions"} + ) + if instructions.len == 0: + return + + for i in instructions: + if i{"entries"}.notNull: + for e in i{"entries"}: + let entryId = e.getEntryId + if entryId.startsWith("tweet") or entryId.startsWith("profile-grid") or + entryId.startsWith("communities-grid"): + for tweet in extractTweetsFromEntry(e): + result.content.add tweet + elif entryId.startsWith("cursor-bottom"): + result.bottom = e{"content", "value"}.getStr + + if after.len == 0 and i.getTypeName == "TimelinePinEntry": + var tweets = extractTweetsFromEntry(i{"entry"}) + for tweet in tweets.mitems: + tweet.pinned = true + if tweets.len > 0: + result.content.insert(tweets, 0) + +proc parseGraphCommunityMembers*(js: JsonNode; after=""): Result[User] = + result = Result[User](beginning: after.len == 0) + + let r = js{"data", "communityResults", "result"} + let slice = if not r{"members_slice"}.isNull: r{"members_slice"} + else: r{"moderators_slice"} + for item in slice{"items_results"}: + let user = parseGraphUser(item{"result"}) + if user.username.len > 0: + result.content.add user + + let cursor = slice{"slice_info", "next_cursor"}.getStr + if cursor.len > 0: + result.bottom = cursor + diff --git a/src/parserutils.nim b/src/parserutils.nim index 8d6ea2e..07aa088 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -185,12 +185,16 @@ proc extractSlice(js: JsonNode): Slice[int] = result = js["indices"][0].getInt ..< js["indices"][1].getInt proc extractUrls(result: var seq[ReplaceSlice]; js: JsonNode; - textLen: int; hideTwitter = false) = + textLen: int; hideTwitter = false; + hideArticle = false) = let url = js.getExpandedUrl slice = js.extractSlice - if hideTwitter and slice.b.succ >= textLen and url.isTwitterUrl: + if hideArticle and url.isTwitterUrl and "/article/" in url: + if slice.a < textLen: + result.add ReplaceSlice(kind: rkRemove, slice: slice) + elif hideTwitter and slice.b.succ >= textLen and url.isTwitterUrl: if slice.a < textLen: result.add ReplaceSlice(kind: rkRemove, slice: slice) else: @@ -202,19 +206,32 @@ proc extractHashtags(result: var seq[ReplaceSlice]; js: JsonNode) = proc replacedWith(runes: seq[Rune]; repls: openArray[ReplaceSlice]; textSlice: Slice[int]): string = + let + runeLen = runes.len + safeStart = max(0, textSlice.a) + safeEnd = min(runeLen, textSlice.b) + + var validRepls: seq[ReplaceSlice] + for rep in repls: + if rep.slice.a >= 0 and rep.slice.b >= 0 and rep.slice.b < runeLen and rep.slice.a <= rep.slice.b: + validRepls.add rep + template extractLowerBound(i: int; idx): int = - if i > 0: repls[idx].slice.b.succ else: textSlice.a + if i > 0: min(validRepls[idx].slice.b.succ, runeLen) else: safeStart result = newStringOfCap(runes.len) - for i, rep in repls: - result.add $runes[extractLowerBound(i, i - 1) ..< rep.slice.a] + for i, rep in validRepls: + let lower = extractLowerBound(i, i - 1) + if lower < rep.slice.a: + result.add $runes[lower ..< rep.slice.a] case rep.kind of rkHashtag: - let - name = $runes[rep.slice.a.succ .. rep.slice.b] - symbol = $runes[rep.slice.a] - result.add a(symbol & name, href = "/search?f=tweets&q=%23" & name) + if rep.slice.a.succ <= rep.slice.b: + let + name = $runes[rep.slice.a.succ .. rep.slice.b] + symbol = $runes[rep.slice.a] + result.add a(symbol & name, href = "/search?f=tweets&q=%23" & name) of rkMention: result.add a($runes[rep.slice], href = rep.url, title = rep.display) of rkUrl: @@ -222,8 +239,8 @@ proc replacedWith(runes: seq[Rune]; repls: openArray[ReplaceSlice]; of rkRemove: discard - let rest = extractLowerBound(repls.len, ^1) ..< textSlice.b - if rest.a <= rest.b: + let rest = extractLowerBound(validRepls.len, ^1) ..< safeEnd + if rest.a >= 0 and rest.a <= rest.b and rest.b <= runeLen: result.add $runes[rest] proc deduplicate(s: var seq[ReplaceSlice]) = @@ -326,10 +343,11 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = replyTo = reply.getStr tweet.reply.add replyTo - tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote or hasJobCard) + tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, + hasQuote or hasJobCard) proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: Slice[int]; - hasRedundantLink=false) = + hasRedundantLink=false; hasArticle=false) = let hasCard = tweet.card.isSome var replacements = newSeq[ReplaceSlice]() @@ -340,7 +358,8 @@ proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: S if urlStr.len == 0 or urlStr notin text: continue - replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink) + replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink, + hideArticle = hasArticle) if hasCard and u{"url"}.getStr == get(tweet.card).url: get(tweet.card).url = u.getExpandedUrl @@ -371,22 +390,26 @@ proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: S tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false) -proc expandTweetEntitiesV2*(tweet: Tweet; js: JsonNode) = +proc expandTweetEntitiesV2*(tweet: Tweet; js: JsonNode; hasArticle=false) = let textRange = js{"details", "display_text_range"} textSlice = textRange{0}.getInt .. textRange{1}.getInt hasQuote = "quoted_tweet_results" in js hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails + hasAttribution = tweet.attribution.isSome - tweet.expandTextEntitiesV2(js, tweet.text, textSlice, hasQuote or hasJobCard) + tweet.expandTextEntitiesV2(js, tweet.text, textSlice, + hasQuote or hasJobCard or hasAttribution, + hasArticle) proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) = let entities = ? js{"entity_set"} text = js{"text"}.getStr.multiReplace(("<", unicodeOpen), (">", unicodeClose)) textSlice = 0..text.runeLen + hasAttribution = tweet.attribution.isSome - tweet.expandTextEntities(entities, text, textSlice) + tweet.expandTextEntities(entities, text, textSlice, hasRedundantLink=hasAttribution) tweet.text = tweet.text.multiReplace((unicodeOpen, xmlOpen), (unicodeClose, xmlClose)) diff --git a/src/redis_cache.nim b/src/redis_cache.nim index bfd271f..b9ddbcc 100644 --- a/src/redis_cache.nim +++ b/src/redis_cache.nim @@ -144,9 +144,10 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} = else: let user = await getGraphUserById(userId) result = user.username - await setEx(key, baseCacheTime, result) - if result.len > 0 and user.id.len > 0: - await all(cacheUserId(result, user.id), cache(user)) + if result.len > 0: + await setEx(key, baseCacheTime, result) + if user.id.len > 0: + await all(cacheUserId(result, user.id), cache(user)) # proc getCachedTweet*(id: int64): Future[Tweet] {.async.} = # if id == 0: return @@ -172,6 +173,21 @@ proc getCachedBroadcast*(id: string): Future[Broadcast] {.async.} = await cache(result) result.m3u8Url = await fetchBroadcastStream(result.mediaKey) +proc cache*(data: AudioSpace) {.async.} = + if data.id.len == 0: return + let ttl = if data.state == "RUNNING": baseCacheTime div 6 else: baseCacheTime + await setEx("sp:" & data.id, ttl, compress(toFlatty(data))) + +proc getCachedAudioSpace*(id: string): Future[AudioSpace] {.async.} = + if id.len == 0: return + let cached = await get("sp:" & id) + if cached != redisNil: + cached.deserialize(AudioSpace) + else: + result = await getAudioSpace(id) + await cache(result) + result.m3u8Url = await fetchBroadcastStream(result.mediaKey) + proc cache*(data: AccountInfo; name: string) {.async.} = await setEx("ai:" & toLower(name), baseCacheTime * 24, compress(toFlatty(data))) @@ -194,6 +210,29 @@ proc getCachedPhotoRail*(id: string): Future[PhotoRail] {.async.} = result = await getPhotoRail(id) await cache(result, id) +proc cache*(data: Community) {.async.} = + if data.id.len == 0: return + await setEx("cm:" & data.id, listCacheTime, compress(toFlatty(data))) + +proc getCachedCommunity*(id: string): Future[Community] {.async.} = + if id.len == 0: return + let cached = await get("cm:" & id) + if cached != redisNil: + cached.deserialize(Community) + else: + result = await getGraphCommunity(id) + await cache(result) + +proc getCachedCommunityModerators*(id: string): Future[seq[User]] {.async.} = + if id.len == 0: return + let cached = await get("cmm:" & id) + if cached != redisNil: + cached.deserialize(seq[User]) + else: + let mods = await getGraphCommunityModerators(id) + result = mods.content + await setEx("cmm:" & id, listCacheTime, compress(toFlatty(result))) + proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} = let list = if id.len == 0: redisNil else: await get("l:" & id) diff --git a/src/routes/article.nim b/src/routes/article.nim new file mode 100644 index 0000000..0a7d1dc --- /dev/null +++ b/src/routes/article.nim @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import asyncdispatch, tables, strutils +import jester, karax/vdom +import ".."/[types, api] +import ../views/[article, general] +import router_utils + +export api, article, vdom, general, router_utils + +proc createArticleRouter*(cfg: Config) = + router articleRoute: + get "/i/article/@id": + cond @"id".allCharsInSet(Digits) + + let article = await getGraphArticle(@"id") + if article == nil: + resp Http404, showError("Article not found", cfg) + + var tweetIds: seq[string] + for e in article.entities.values: + if e.kind == "TWEET": + tweetIds.add e.tweetId + + var tweets = initTable[int64, Tweet]() + if tweetIds.len > 0: + try: + for t in await getGraphTweetResults(tweetIds): + tweets[t.id] = t + except CatchableError: + discard + + let + prefs = requestPrefs() + path = getPath() + html = renderArticle(article, tweets, path, prefs, @"id") + twitterUrl = "https://x.com/" & article.user.username & "/article/" & @"id" + resp renderMain(html, request, cfg, prefs, titleText=article.title, + twitterLink=twitterUrl) + + get "/@name/article/@id/?": + cond '.' notin @"name" + cond @"id".allCharsInSet(Digits) + redirect("/i/article/" & @"id") + + get "/@name/status/@id/article": + cond '.' notin @"name" + cond @"id".allCharsInSet(Digits) + redirect("/i/article/" & @"id") diff --git a/src/routes/community.nim b/src/routes/community.nim new file mode 100644 index 0000000..b850b6b --- /dev/null +++ b/src/routes/community.nim @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import strformat + +import jester + +import router_utils +import ".."/[types, redis_cache, api] +import ../views/[general, timeline, community] + +export community + +template respCommunity*(cmty: Community; title: string; nav, vnode: typed) = + if cmty.id.len == 0 or cmty.name.len == 0: + resp Http404, showError(&"""Community "{@"id"}" not found""", cfg) + + let html = renderCommunity(vnode, nav, cmty) + resp renderMain(html, request, cfg, prefs, titleText=title, banner=cmty.banner) + +proc createCommunityRouter*(cfg: Config) = + router community: + get "/i/communities/@id/?": + cond '.' notin @"id" + let + prefs = requestPrefs() + cmty = await getCachedCommunity(@"id") + tl = await getGraphCommunityTweets(cmty.id, "Relevance", getCursor()) + respCommunity(cmty, cmty.name, + renderCommunityTabs(QueryKind.posts, cmty), + renderTimelineTweets(tl, prefs, request.path)) + + get "/i/communities/@id/latest": + cond '.' notin @"id" + let + prefs = requestPrefs() + cmty = await getCachedCommunity(@"id") + tl = await getGraphCommunityTweets(cmty.id, "Recency", getCursor()) + respCommunity(cmty, cmty.name & " - Latest", + renderCommunityTabs(QueryKind.replies, cmty), + renderTimelineTweets(tl, prefs, request.path)) + + get "/i/communities/@id/media": + cond '.' notin @"id" + let + prefs = requestPrefs() + cmty = await getCachedCommunity(@"id") + tl = await getGraphCommunityMedia(cmty.id, getCursor()) + respCommunity(cmty, cmty.name & " - Media", + renderCommunityTabs(QueryKind.media, cmty), + renderTimelineTweets(tl, prefs, request.path)) + + get "/i/communities/@id/about": + cond '.' notin @"id" + let + prefs = requestPrefs() + cmty = await getCachedCommunity(@"id") + mods = await getCachedCommunityModerators(cmty.id) + respCommunity(cmty, cmty.name & " - About", + renderCommunityTabs(QueryKind.userList, cmty), + renderCommunityAbout(cmty, mods)) + + get "/i/communities/@id/members": + cond '.' notin @"id" + let + prefs = requestPrefs() + cmty = await getCachedCommunity(@"id") + members = await getGraphCommunityMembers(cmty.id, getCursor()) + respCommunity(cmty, cmty.name & " - Members", + renderMemberTabs(cmty, false), + renderTimelineUsers(members, prefs, request.path)) + + get "/i/communities/@id/moderators": + cond '.' notin @"id" + let + prefs = requestPrefs() + cmty = await getCachedCommunity(@"id") + mods = await getCachedCommunityModerators(cmty.id) + respCommunity(cmty, cmty.name & " - Moderators", + renderMemberTabs(cmty, true), + renderTimelineUsers(Result[User](content: mods), prefs, request.path)) + + get "/i/communities/@id/hashtag/@tag": + cond '.' notin @"id" + let + prefs = requestPrefs() + cmty = await getCachedCommunity(@"id") + tl = await getGraphCommunityHashtags(cmty.id, @"tag", getCursor()) + respCommunity(cmty, cmty.name & " - #" & @"tag", + renderHashtagHeader(cmty, @"tag"), + renderTimelineTweets(tl, prefs, request.path)) diff --git a/src/routes/media.nim b/src/routes/media.nim index df30d5f..3442916 100644 --- a/src/routes/media.nim +++ b/src/routes/media.nim @@ -1,6 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only import uri, strutils, httpclient, os, hashes, base64, re import asynchttpserver, asyncstreams, asyncfile, asyncnet +import asyncdispatch import jester @@ -15,7 +16,9 @@ const maxAge* = "max-age=604800" proc safeFetch*(url: string): Future[string] {.async.} = - let client = newAsyncHttpClient() + # maxRedirects=0: the caller already validated the host, so never follow a + # redirect off the allowlisted host (would re-open the #1411 SSRF). + let client = newAsyncHttpClient(maxRedirects = 0) try: result = await client.getContent(url) except: discard finally: client.close() @@ -30,47 +33,79 @@ template respond*(req: asynchttpserver.Request; headers) = proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} = result = Http200 - let - request = req.getNativeReq() - client = newAsyncHttpClient() + let request = req.getNativeReq() + var fetchUrl = url - try: - let res = await client.get(url) - if res.status != "200 OK": - if res.status != "404 Not Found": - echo "[media] Proxying failed, status: $1, url: $2" % [res.status, url] - return Http404 - - let hashed = $hash(url) - if request.headers.getOrDefault("If-None-Match") == hashed: - return Http304 - - let contentLength = - if res.headers.hasKey("content-length"): - res.headers["content-length", 0] + for attempt in 0 .. 2: + let client = newAsyncHttpClient(maxRedirects = 0) + var shouldRetry = false + try: + let resFut = client.get(fetchUrl) + let completed = await withTimeout(resFut, 5000) + if not completed: + if attempt < 2: + echo "[media] Retry $1/2, timeout after 5s, url: $2" % [$(attempt + 1), fetchUrl] + shouldRetry = true + else: + echo "[media] Proxying timeout after 5s, url: $1" % [fetchUrl] + return Http504 else: - "" + let res = resFut.read() + if res.status != "200 OK": + if res.status == "404 Not Found": + return Http404 + if res.status.startsWith("30") and res.headers.hasKey("location"): + let location = res.headers["location", 0] + if isTwitterUrl(location): + fetchUrl = location + shouldRetry = true + continue + else: + return Http403 + if attempt < 2: + echo "[media] Retry $1/2, status: $2, url: $3" % [$(attempt + 1), res.status, fetchUrl] + shouldRetry = true + else: + echo "[media] Proxying failed, status: $1, url: $2" % [res.status, fetchUrl] + return Http404 + else: + let hashed = $hash(url) + if request.headers.getOrDefault("If-None-Match") == hashed: + return Http304 - let headers = newHttpHeaders({ - "content-type": res.headers["content-type", 0], - "content-length": contentLength, - "cache-control": maxAge, - "etag": hashed - }) + let contentLength = + if res.headers.hasKey("content-length"): + res.headers["content-length", 0] + else: + "" - respond(request, headers) + let headers = newHttpHeaders({ + "content-type": res.headers["content-type", 0], + "content-length": contentLength, + "cache-control": maxAge, + "etag": hashed + }) - var (hasValue, data) = (true, "") - while hasValue: - (hasValue, data) = await res.bodyStream.read() - if hasValue: - await request.client.send(data) - data.setLen 0 - except HttpRequestError, ProtocolError, OSError: - echo "[media] Proxying exception, error: $1, url: $2" % [getCurrentExceptionMsg(), url] - result = Http404 - finally: - client.close() + respond(request, headers) + + var (hasValue, data) = (true, "") + while hasValue: + (hasValue, data) = await res.bodyStream.read() + if hasValue: + await request.client.send(data) + data.setLen 0 + return Http200 + except CatchableError: + if attempt < 2: + echo "[media] Retry $1/2, error: $2, url: $3" % [$(attempt + 1), getCurrentExceptionMsg(), fetchUrl] + shouldRetry = true + else: + echo "[media] Proxying exception, error: $1, url: $2" % [getCurrentExceptionMsg(), fetchUrl] + result = Http404 + finally: + client.close() + if not shouldRetry: + break template check*(code): untyped = if code != Http200: @@ -122,12 +157,12 @@ proc createMediaRouter*(cfg: Config) = get re"^\/video\/(enc)?\/?(.+)\/(.+)$": let url = decoded(request, 2) - cond "http" in url + cond isTwitterUrl(url) if getHmac(url) != request.matches[1]: resp Http403, showError("Failed to verify signature", cfg) - if ".mp4" in url or ".ts" in url or ".m4s" in url: + if ".mp4" in url or ".ts" in url or ".m4s" in url or ".aac" in url: let code = await proxyMedia(request, url) check code diff --git a/src/routes/preferences.nim b/src/routes/preferences.nim index 5886c0e..7f04de2 100644 --- a/src/routes/preferences.nim +++ b/src/routes/preferences.nim @@ -40,3 +40,6 @@ proc createPrefRouter*(cfg: Config) = savePref("hlsPlayback", "on", request) redirect(refPath()) + post "/enablemp4": + savePref("mp4Playback", "on", request) + redirect(refPath()) diff --git a/src/routes/router_utils.nim b/src/routes/router_utils.nim index 379280c..8a26193 100644 --- a/src/routes/router_utils.nim +++ b/src/routes/router_utils.nim @@ -8,8 +8,9 @@ export utils, prefs, types, uri template savePref*(pref, value: string; req: Request; expire=false) = if not expire or pref in cookies(req): + let sameSite = if cfg.useHttps: None else: Lax setCookie(pref, value, daysForward(when expire: -10 else: 360), - httpOnly=true, secure=cfg.useHttps, sameSite=None, path="/") + httpOnly=true, secure=cfg.useHttps, sameSite=sameSite, path="/") template requestPrefs*(): untyped {.dirty.} = getPrefs(cookies(request), params(request)) diff --git a/src/routes/search.nim b/src/routes/search.nim index 7d72f34..ba023fd 100644 --- a/src/routes/search.nim +++ b/src/routes/search.nim @@ -46,6 +46,7 @@ proc createSearchRouter*(cfg: Config) = redirect("/search?f=tweets&q=" & encodeUrl("#" & @"hash")) get "/opensearch": - let url = getUrlPrefix(cfg) & "/search?f=tweets&q=" - resp Http200, {"Content-Type": "application/opensearchdescription+xml"}, - generateOpenSearchXML(cfg.title, cfg.hostname, url) + let + url = getUrlPrefix(cfg) & "/search?f=tweets&q=" + headers = {"Content-Type": "application/opensearchdescription+xml"} + resp Http200, headers, generateOpenSearchXML(cfg.title, cfg.hostname, url) diff --git a/src/routes/space.nim b/src/routes/space.nim new file mode 100644 index 0000000..bd956ea --- /dev/null +++ b/src/routes/space.nim @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import asyncdispatch, strutils +import jester + +import router_utils +import ".."/[types, formatters, redis_cache] +import ../views/[general, space] +import media + +export space + +proc createSpaceRouter*(cfg: Config) = + router spaceRoute: + get "/i/spaces/@id": + cond @"id".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9'}) + let sp = await getCachedAudioSpace(@"id") + + if sp.id.len == 0: + resp Http404, showError("Space not found", cfg) + + let prefs = requestPrefs() + resp renderMain(renderSpace(sp, prefs, request.path), request, cfg, prefs, + sp.title, ogTitle=sp.title) + + get "/i/spaces/@id/stream": + cond @"id".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9'}) + let sp = await getCachedAudioSpace(@"id") + + if sp.m3u8Url.len == 0: + resp Http404 + + let manifest = await safeFetch(sp.m3u8Url) + if manifest.len == 0: + resp Http502 + + resp proxifyVideo(manifest, requestPrefs().proxyVideos, sp.m3u8Url), m3u8Mime diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index 0eb48e2..6e8969e 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -135,6 +135,39 @@ proc createTimelineRouter*(cfg: Config) = resp renderMain(aboutHtml, request, cfg, prefs, "About @" & info.username) + get "/@name/@kind/?": + cond @"kind" in ["followers", "following"] + cond '.' notin @"name" + cond @"name".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9', '_'}) + let + prefs = requestPrefs() + name = @"name" + userId = await getUserId(name) + + if userId.len == 0: + resp Http404, showError("User \"" & name & "\" not found", cfg) + if userId == "suspended": + resp showError(getSuspended(name), cfg) + + let + cursor = getCursor() + isFollowers = @"kind" == "followers" + user = await getCachedUser(name) + results = if isFollowers: await getGraphFollowers(userId, cursor) + else: await getGraphFollowing(userId, cursor) + + if user.protected: + resp renderMain(renderProtected(user.username), request, cfg, prefs, + "Protected account @" & name) + + let + tab = if isFollowers: "Followers" else: "Following" + title = (if isFollowers: "People following @" else: "People followed by @") & name + html = renderUserList(user, results, prefs, request.path, tab) + + resp renderMain(html, request, cfg, prefs, title, + images = @[user.getUserPic("_400x400")]) + get "/@name/?@tab?/?": cond '.' notin @"name" cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"] diff --git a/src/sass/_article.scss b/src/sass/_article.scss new file mode 100644 index 0000000..9488658 --- /dev/null +++ b/src/sass/_article.scss @@ -0,0 +1,278 @@ +.article-page { + max-width: 700px; + margin: 0 auto 20px; + background-color: var(--bg_panel); + + > .top-ref { + padding-top: 20px; + } + + .article-cover { + width: 100%; + display: block; + } + + .article-body { + padding: 20px; + + > :last-child { + margin-bottom: 0; + } + + .article-title { + display: block; + font-size: 2rem; + line-height: 1.3; + margin: 0 0 10px; + color: var(--fg_color); + } + + .article-author { + margin-bottom: 12px; + padding-bottom: 10px; + border-bottom: 1px solid var(--border_grey); + font-size: 14px; + + .article-author-row { + display: flex; + align-items: center; + gap: 8px; + } + + .article-avatar { + display: flex; + } + + .avatar { + width: 40px; + height: 40px; + } + + .article-author-name { + display: flex; + align-items: center; + margin-bottom: 2px; + + .fullname { + font-size: 15px; + } + + .verified-icon { + margin-left: 2px; + } + } + + .article-author-meta { + display: flex; + align-items: center; + } + + .fullname { + font-weight: 700; + color: var(--fg_color); + max-width: unset; + text-overflow: unset; + overflow: visible; + white-space: normal; + } + + .username, + .article-date-sep, + .article-date { + color: var(--fg_dark); + } + + .username { + margin-left: 0; + } + + .article-date-sep { + margin: 0 4px; + } + + .article-date { + text-decoration: none; + + &:hover { + text-decoration: underline; + } + } + + .tweet-stats { + margin-top: 6px; + + .tweet-stat { + padding-top: 0; + } + } + } + + > h1 { + display: block; + font-size: 1.8rem; + margin: 25px 0 15px; + } + + > h2 { + font-size: 1.4rem; + font-weight: bold; + margin: 20px 0 12px; + } + + > h3 { + font-size: 1.2rem; + font-weight: bold; + margin: 18px 0 10px; + } + + > p { + font-size: 16px; + line-height: 1.7; + margin: 16px 0; + word-wrap: break-word; + } + + .blockquote-attribution { + display: block; + margin-top: 0.5em; + } + + > blockquote { + border-left: 3px solid var(--accent); + padding-left: 16px; + margin: 16px 0; + color: var(--fg_faded); + font-size: 16px; + line-height: 1.7; + } + + > pre { + background-color: var(--bg_elements); + padding: 12px 16px; + border-radius: 6px; + overflow-x: auto; + margin: 16px 0; + + code { + font-family: monospace; + font-size: 14px; + color: var(--fg_color); + } + } + + code { + background-color: var(--bg_elements); + padding: 2px 5px; + border-radius: 3px; + font-family: monospace; + font-size: 0.9em; + } + + > ul, + > ol { + margin: 16px 0; + padding-left: 2em; + + li { + font-size: 16px; + line-height: 1.7; + margin: 6px 0; + } + } + + .article-media { + text-align: center; + margin: 20px 0; + + img, + video { + max-width: 100%; + border-radius: 12px; + } + + .article-media-caption { + color: var(--fg_faded); + font-size: 0.875rem; + margin-top: 6px; + } + } + + > a, + > p a, + > h1 a, + > h2 a, + > h3 a, + > blockquote a, + > ul a, + > ol a { + color: var(--accent); + text-decoration: none; + + &:hover { + text-decoration: underline; + } + } + + .article-divider { + border: none; + border-top: 1px solid var(--border_grey); + margin: 30px 0; + } + + .timeline-item { + margin: 20px 0; + border: 1px solid var(--border_grey); + border-radius: 12px; + overflow: hidden; + } + } +} + +.conversation .article-page { + max-width: 100%; + margin-bottom: 0; +} + +.article-card { + .card-image-container { + position: relative; + } + + .card-image img { + height: auto; + } + + .article-card-badge { + position: absolute; + bottom: 8px; + left: 8px; + background: rgba(0, 0, 0, 0.75); + color: #fff; + font-size: 13px; + font-weight: 700; + padding: 2px 8px; + border-radius: 4px; + } +} + +.quote .article-card { + margin: 0; + + .card-container { + border: none; + border-radius: 0; + border-top: solid 1px var(--dark_grey); + } +} + +@media (max-width: 700px) { + .article-page { + .article-body { + padding: 12px 15px 25px; + + .article-title { + font-size: 1.6rem; + } + } + } +} diff --git a/src/sass/_space.scss b/src/sass/_space.scss new file mode 100644 index 0000000..5fe2e7e --- /dev/null +++ b/src/sass/_space.scss @@ -0,0 +1,149 @@ +.space-page { + max-width: 800px; + width: 100%; + margin: 20px auto 0; +} + +.space-panel { + background-color: var(--bg_panel); + border: 1px solid var(--border_grey); + border-radius: 8px; + overflow: hidden; +} + +.space-player { + position: relative; + background: linear-gradient(135deg, #7b2a8c 0%, #9b3ab1 100%); + min-height: 140px; + display: flex; + align-items: center; + justify-content: center; + + audio { + width: 100%; + padding: 15px; + box-sizing: border-box; + + &:not([controls]) { + display: none; + } + } + + .video-overlay { + background-color: transparent; + } +} + +.space-live { + background: #e0245e; + color: white; + padding: 3px 8px; + border-radius: 4px; + font-weight: bold; + font-size: 12px; + text-transform: uppercase; + position: absolute; + top: 8px; + right: 8px; +} + +.space-info { + padding: 16px; +} + +.space-header { + display: flex; + justify-content: space-between; + align-items: flex-start; + gap: 12px; + margin-bottom: 16px; +} + +.space-title { + font-size: 18px; + font-weight: bold; + margin: 0; + line-height: 1.3; + flex: 1; +} + +.space-meta { + display: flex; + flex-direction: column; + align-items: flex-end; + gap: 4px; + flex-shrink: 0; + font-size: 14px; + color: var(--fg_faded); +} + +.listener-count { + color: var(--fg_color); +} + +.space-state { + color: var(--fg_dark); +} + +.space-participants { + border-top: 1px solid var(--border_grey); + padding-top: 12px; +} + +.space-participant { + margin-bottom: 10px; + + a { + display: flex; + align-items: center; + gap: 10px; + color: var(--fg_color); + padding: 6px 0; + } + + img { + width: 40px; + height: 40px; + border-radius: 50%; + flex-shrink: 0; + } +} + +.participant-info { + min-width: 0; +} + +.participant-name { + display: flex; + align-items: center; + gap: 4px; + + strong { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .verified-icon { + margin-bottom: 0; + position: relative; + top: -2px; + } +} + +.host-badge { + background: var(--accent); + color: white; + padding: 2px 7px; + border-radius: 3px; + font-size: 11px; + font-weight: 600; + line-height: 1; + position: relative; + top: 1px; +} + +.participant-username { + color: var(--fg_dark); + font-size: 13px; +} diff --git a/src/sass/index.scss b/src/sass/index.scss index e60b9c4..404f7d5 100644 --- a/src/sass/index.scss +++ b/src/sass/index.scss @@ -8,6 +8,8 @@ @import "timeline"; @import "search"; @import "broadcast"; +@import "space"; +@import "_article"; body { // colors @@ -159,6 +161,7 @@ body.fixed-nav .container { .verified-icon { display: inline-block; + position: relative; width: 14px; height: 14px; margin-bottom: 2px; diff --git a/src/sass/profile/_base.scss b/src/sass/profile/_base.scss index 2482460..81b3d78 100644 --- a/src/sass/profile/_base.scss +++ b/src/sass/profile/_base.scss @@ -4,6 +4,7 @@ @import "card"; @import "about-account"; @import "photo-rail"; +@import "community"; .profile-tabs { @include panel(auto, 900px); diff --git a/src/sass/profile/_community.scss b/src/sass/profile/_community.scss new file mode 100644 index 0000000..92d13c9 --- /dev/null +++ b/src/sass/profile/_community.scss @@ -0,0 +1,203 @@ +.community-header { + padding: 12px 15px; + border-bottom: 1px solid var(--border_grey); + background-color: var(--bg_panel); + + .community-name { + font-size: 22px; + margin-bottom: 6px; + + a { + color: inherit; + } + } + + .community-category { + display: inline-block; + background-color: var(--bg_elements); + border: 1px solid var(--border_grey); + border-radius: 16px; + padding: 2px 12px; + font-size: 13px; + color: var(--fg_faded); + margin-bottom: 8px; + } + + .community-description { + color: var(--fg_faded); + margin-bottom: 8px; + line-height: 1.4; + } + + .community-member-count { + font-weight: bold; + color: inherit; + } + + .community-stats { + color: var(--grey); + font-size: 14px; + } +} + +.community-about { + padding: 16px 15px 15px; + background-color: var(--bg_panel); + + h2 { + font-size: 18px; + margin: 0 0 12px; + } + + .community-info { + border-bottom: 1px solid var(--border_grey); + padding-bottom: 12px; + } + + .community-info-item { + display: flex; + gap: 10px; + padding: 8px 0; + align-items: center; + + .verified-icon { + margin-left: 2px; + } + + > .icon-container { + display: flex; + align-items: center; + justify-content: center; + font-size: 16px; + color: var(--grey); + flex-shrink: 0; + width: 26px; + height: 26px; + } + + strong { + color: var(--fg_color); + } + + a { + color: var(--accent); + } + } + + .community-rules { + border-bottom: 1px solid var(--border_grey); + padding: 16px 0 12px; + } + + .community-rules-intro { + color: var(--fg_faded); + font-size: 14px; + margin: 0 0 12px; + } + + .community-rule { + display: flex; + gap: 10px; + padding: 10px 0; + align-items: flex-start; + + .community-rule-number { + display: flex; + align-items: center; + justify-content: center; + min-width: 26px; + height: 26px; + border-radius: 50%; + background-color: var(--accent); + color: var(--fg_color); + font-weight: bold; + font-size: 13px; + flex-shrink: 0; + } + + .community-rule-content p { + margin: 4px 0 0; + color: var(--fg_faded); + font-size: 14px; + } + } + + .community-moderators { + padding-top: 16px; + + h2 { + display: flex; + align-items: center; + justify-content: space-between; + } + + .community-mods-link { + font-size: 14px; + font-weight: normal; + color: var(--accent); + } + } + + .community-moderator { + display: flex; + gap: 10px; + padding: 8px 0; + align-items: center; + + .community-mod-avatar { + width: 40px; + height: 40px; + border-radius: 50%; + } + + .community-mod-info { + display: flex; + flex-direction: column; + } + + .community-mod-name { + display: flex; + align-items: center; + font-weight: bold; + color: var(--fg_color); + + .verified-icon { + margin-left: 2px; + } + } + + .community-mod-username { + color: var(--fg_faded); + font-size: 14px; + } + } +} + +.community-tags { + display: flex; + flex-wrap: wrap; + gap: 8px; + padding: 10px 15px; + border-bottom: 1px solid var(--border_grey); + + .community-tag { + display: inline-block; + background-color: var(--bg_elements); + border: 1px solid var(--border_grey); + border-radius: 16px; + padding: 4px 12px; + font-size: 13px; + color: var(--accent); + } +} + +.community-hashtag-header { + padding: 12px 15px; + border-bottom: 1px solid var(--border_grey); + + .community-hashtag-title { + font-size: 20px; + color: var(--accent); + margin: 0; + } +} diff --git a/src/sass/tweet/_base.scss b/src/sass/tweet/_base.scss index 7606c31..93d2086 100644 --- a/src/sass/tweet/_base.scss +++ b/src/sass/tweet/_base.scss @@ -305,7 +305,7 @@ margin-top: 4px; margin-bottom: -2px; - .icon-attention { + .icon-attention-circled { margin-right: -3px; } } diff --git a/src/sass/tweet/media.scss b/src/sass/tweet/media.scss index d7b443e..3001a86 100644 --- a/src/sass/tweet/media.scss +++ b/src/sass/tweet/media.scss @@ -90,7 +90,10 @@ max-height: unset; } -.media-gif video { +.media-gif video, +.media-gif img { + width: 100%; + height: 100%; max-height: 530px; background-color: #101010; } diff --git a/src/sass/tweet/quote.scss b/src/sass/tweet/quote.scss index d6a75a9..f722455 100644 --- a/src/sass/tweet/quote.scss +++ b/src/sass/tweet/quote.scss @@ -100,7 +100,8 @@ justify-content: center; background-color: var(--bg_color); - video { + video, + img { height: unset; width: unset; max-height: 100%; diff --git a/src/sass/tweet/video.scss b/src/sass/tweet/video.scss index c20d348..28fc125 100644 --- a/src/sass/tweet/video.scss +++ b/src/sass/tweet/video.scss @@ -28,6 +28,40 @@ video { } } +.video-download { + position: absolute; + top: 8px; + right: 8px; + z-index: 2; + color: var(--accent); + text-decoration: none; + opacity: 0; + transition: opacity 0.2s; + filter: drop-shadow(0 1px 2px rgba(0, 0, 0, 0.7)); + + .icon-container { + margin: 0; + } + + .icon-download-alt { + font-size: 18px; + } +} + +.attachment:hover .video-download { + opacity: 1; + + &:hover { + color: var(--fg_color); + } +} + +@media (hover: none) { + .video-download { + opacity: 1; + } +} + .video-overlay { @include play-button; background-color: $shadow; diff --git a/src/types.nim b/src/types.nim index 848f8ae..a490316 100644 --- a/src/types.nim +++ b/src/types.nim @@ -16,6 +16,7 @@ type ApiUrl* = object endpoint*: string params*: seq[(string, string)] + skipTid*: bool ApiReq* = object oauth*: ApiUrl @@ -60,6 +61,7 @@ type rateLimited = 88 expiredToken = 89 listIdOrSlug = 112 + timelineUnavailable = 131 tweetNotFound = 144 tweetNotAuthorized = 179 forbidden = 200 @@ -127,6 +129,28 @@ type availableForReplay*: bool user*: User + SpaceParticipant* = object + userId*: string + username*: string + displayName*: string + avatarUrl*: string + isVerified*: bool + + AudioSpace* = object + id*: string + title*: string + state*: string + mediaKey*: string + m3u8Url*: string + totalLiveListeners*: int + totalReplayWatched*: int + startTime*: DateTime + endTime*: DateTime + availableForReplay*: bool + creator*: User + admins*: seq[SpaceParticipant] + speakers*: seq[SpaceParticipant] + VideoType* = enum m3u8 = "application/x-mpegURL" mp4 = "video/mp4" @@ -150,7 +174,7 @@ type variants*: seq[VideoVariant] QueryKind* = enum - posts, replies, media, users, tweets, userList + posts, replies, media, users, tweets, userList, followers, following Query* = object kind*: QueryKind @@ -197,6 +221,44 @@ type PhotoRail* = seq[GalleryPhoto] + Article* = ref object + title*: string + coverImage*: string + user*: User + time*: DateTime + stats*: TweetStats + paragraphs*: seq[ArticleParagraph] + entities*: Table[int, ArticleEntity] + media*: Table[string, ArticleMedia] + + ArticleParagraph* = object + text*: string + kind*: string + inlineStyles*: seq[ArticleStyle] + entityRanges*: seq[ArticleEntityRange] + + ArticleStyle* = object + offset*: int + length*: int + style*: string + + ArticleEntityRange* = object + offset*: int + length*: int + key*: int + + ArticleEntity* = object + kind*: string + url*: string + mediaIds*: seq[string] + tweetId*: string + markdown*: string + caption*: string + + ArticleMedia* = object + kind*: string + url*: string + Poll* = object options*: seq[string] values*: seq[int] @@ -246,6 +308,12 @@ type likes*: int views*: int + ArticlePreview* = object + title*: string + previewText*: string + coverImage*: string + tweetId*: int64 + Tweet* = ref object id*: int64 threadId*: int64 @@ -264,6 +332,7 @@ type stats*: TweetStats retweet*: Option[Tweet] attribution*: Option[User] + attributionLink*: string mediaTags*: seq[User] quote*: Option[Tweet] card*: Option[Card] @@ -273,6 +342,7 @@ type note*: string isAd*: bool isAI*: bool + articlePreview*: Option[ArticlePreview] Tweets* = seq[Tweet] @@ -315,6 +385,23 @@ type members*: int banner*: string + CommunityRule* = object + name*: string + description*: string + + Community* = object + id*: string + name*: string + description*: string + memberCount*: int + banner*: string + creator*: User + category*: string + joinPolicy*: string + createdAt*: DateTime + rules*: seq[CommunityRule] + hashtags*: seq[string] + GlobalObjects* = ref object tweets*: Table[string, Tweet] users*: Table[string, User] diff --git a/src/utils.nim b/src/utils.nim index 391e2a3..95b46de 100644 --- a/src/utils.nim +++ b/src/utils.nim @@ -40,12 +40,14 @@ proc getVidUrl*(link: string): string = &"/video/{sig}/{encodeUrl(link)}" proc getPicUrl*(link: string): string = + if link.len == 0: return if base64Media: &"/pic/enc/{encode(link, safe=true)}" else: &"/pic/{encodeUrl(link)}" proc getOrigPicUrl*(link: string): string = + if link.len == 0: return if base64Media: &"/pic/orig/enc/{encode(link, safe=true)}" else: @@ -57,8 +59,8 @@ proc filterParams*(params: Table): seq[(string, string)] = result.add p proc isTwitterUrl*(uri: Uri): bool = - uri.hostname in twitterDomains or - uri.hostname.endsWith(".video.pscp.tv") + uri.scheme in ["http", "https"] and + (uri.hostname in twitterDomains or uri.hostname.endsWith(".video.pscp.tv")) proc isTwitterUrl*(url: string): bool = isTwitterUrl(parseUri(url)) diff --git a/src/views/article.nim b/src/views/article.nim new file mode 100644 index 0000000..eedd4cf --- /dev/null +++ b/src/views/article.nim @@ -0,0 +1,248 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import strutils, strformat, tables, unicode, bitops, uri +import karax/[karaxdsl, vdom] + +import renderutils, tweet, timeline +import ".."/[types, utils, formatters] + +proc renderAtomicParagraph(paragraph: ArticleParagraph; article: Article; + tweets: Table[int64, Tweet]; path: string; + prefs: Prefs): VNode = + if paragraph.entityRanges.len == 0: + return text "" + + let er = paragraph.entityRanges[0] + if er.key notin article.entities: + return text "" + + let entity = article.entities[er.key] + + case entity.kind + of "MEDIA": + buildHtml(tdiv(class="article-media")): + for id in entity.mediaIds: + let media = article.media.getOrDefault(id) + if media.url.len == 0: + continue + case media.kind + of "ApiGif": + video(src=getVidUrl(media.url), controls="", autoplay="", loop="", + muted="") + of "ApiVideo": + video(src=getVidUrl(media.url), controls="") + else: + a(href=getOrigPicUrl(media.url), target="_blank"): + img(src=getSmallPic(media.url), alt=entity.caption, loading="lazy") + if entity.caption.len > 0: + p(class="article-media-caption"): text entity.caption + of "TWEET": + let tweet = tweets.getOrDefault( + try: parseBiggestInt(entity.tweetId) + except ValueError: 0, nil) + if tweet != nil: + renderTweet(tweet, prefs, path) + else: + text "" + of "MARKDOWN": + var content = entity.markdown + if content.startsWith("```"): + let firstNl = content.find('\n') + if firstNl >= 0: content = content[firstNl + 1 .. ^1] + if content.endsWith("```"): content = content[0 .. ^4] + content = content.strip + buildHtml(pre()): + code(): text content + of "DIVIDER": + buildHtml(hr(class="article-divider")) + else: + text "" + +proc wrapStyle(node: VNode; style: int): VNode = + result = node + if style.testBit(4): result = buildHtml(code()): result + if style.testBit(0): result = buildHtml(strong()): result + if style.testBit(1): result = buildHtml(em()): result + if style.testBit(2): result = buildHtml(del()): result + if style.testBit(3): result = buildHtml(underlined()): result + +proc addContent(target: VNode; content: string; style = 0) = + var first = true + for line in content.split('\n'): + if not first: + target.add VNode(kind: VNodeKind.br) + first = false + var pos = 0 + while pos < line.len: + let atPos = line.find('@', pos) + if atPos == -1: + target.add wrapStyle(text line[pos .. ^1], style) + break + if atPos > 0 and line[atPos - 1] in Letters + Digits + {'_'}: + target.add wrapStyle(text line[pos .. atPos], style) + pos = atPos + 1 + continue + var j = atPos + 1 + while j < line.len and j - atPos - 1 < 15 and + line[j] in Letters + Digits + {'_'}: + inc j + if j == atPos + 1: + target.add wrapStyle(text line[pos .. atPos], style) + pos = atPos + 1 + continue + if atPos > pos: + target.add wrapStyle(text line[pos ..< atPos], style) + let username = line[atPos + 1 ..< j] + let link = a.newVNode() + link.setAttr("href", "/" & username) + link.add wrapStyle(text ("@" & username), style) + target.add link + pos = j + +proc applyInlineStyles(target: VNode; runes: seq[Rune]; start, length: int; + styles: seq[ArticleStyle]) = + if styles.len == 0: + target.addContent($runes[start ..< start + length]) + return + + var + lastStyle = 0 + lastStart = start + let endPos = start + length + + for i in start ..< endPos: + var style = 0 + for sr in styles: + let + sStart = sr.offset + sEnd = sStart + sr.length + if sStart <= i and sEnd > i: + case sr.style + of "Bold": style.setBit(0) + of "Italic": style.setBit(1) + of "Strikethrough": style.setBit(2) + of "Underline": style.setBit(3) + of "Code": style.setBit(4) + else: discard + + if style != lastStyle: + if i > lastStart: + addContent(target, $runes[lastStart ..< i], lastStyle) + lastStyle = style + lastStart = i + + if lastStart < endPos: + addContent(target, $runes[lastStart ..< endPos], lastStyle) + +proc renderTextParagraph(paragraph: ArticleParagraph; article: Article): VNode = + let text = paragraph.text + + result = case paragraph.kind + of "header-one": h1.newVNode() + of "header-two": h2.newVNode() + of "header-three": h3.newVNode() + of "ordered-list-item", "unordered-list-item": li.newVNode() + of "blockquote": VNode(kind: VNodeKind.blockquote) + of "code-block": + let pre = pre.newVNode() + let code = code.newVNode() + code.add text text + pre.add code + return pre + else: p.newVNode() + + let + runes = text.toRunes + textLen = runes.len + var last = 0 + for er in paragraph.entityRanges: + if er.offset > last: + applyInlineStyles(result, runes, last, er.offset - last, + paragraph.inlineStyles) + + last = er.offset + er.length + + var target = result + if er.key in article.entities: + let entity = article.entities[er.key] + if entity.kind == "LINK": + let parsed = parseUri(entity.url) + if parsed.scheme in ["http", "https"]: + target = a.newVNode() + if parsed.isTwitterUrl: + target.setAttr("href", parsed.path) + else: + target.setAttr("href", entity.url) + + applyInlineStyles(target, runes, er.offset, er.length, + paragraph.inlineStyles) + if target != result: + result.add target + + if last < textLen: + applyInlineStyles(result, runes, last, textLen - last, + paragraph.inlineStyles) + + if paragraph.kind == "blockquote" and result.len > 0: + let lastChild = result[result.len - 1] + if lastChild.kind == VNodeKind.strong and lastChild.len > 0 and + lastChild[0].kind == VNodeKind.text: + lastChild.setAttr("class", "blockquote-attribution") + +proc renderArticle*(article: Article; tweets: Table[int64, Tweet]; + path: string; prefs: Prefs; tweetId=""): VNode = + let author = article.user + + let main = buildHtml(article(class="article-body")): + h1(class="article-title"): text article.title + + tdiv(class="article-author"): + tdiv(class="article-author-row"): + a(class="article-avatar", href=("/" & author.username)): + genImg(author.getUserPic("_bigger"), class=prefs.getAvatarClass) + tdiv(class="article-author-info"): + tdiv(class="article-author-name"): + linkUser(author, class="fullname") + verifiedIcon(author) + tdiv(class="article-author-meta"): + linkUser(author, class="username") + span(class="article-date-sep"): text " · " + a(class="article-date", + href=("/" & author.username & "/status/" & tweetId)): + text article.time.getShortTime + if not prefs.hideTweetStats: + renderStats(article.stats) + + var listKind = "" + var list: VNode = nil + + for paragraph in article.paragraphs: + let isListItem = paragraph.kind in [ + "ordered-list-item", "unordered-list-item"] + + if not isListItem and list != nil: + main.add list + list = nil + listKind = "" + + if paragraph.kind == "atomic": + main.add renderAtomicParagraph(paragraph, article, tweets, path, prefs) + elif isListItem: + if paragraph.kind != listKind: + if list != nil: + main.add list + list = if paragraph.kind == "ordered-list-item": ol.newVNode() + else: ul.newVNode() + listKind = paragraph.kind + list.add renderTextParagraph(paragraph, article) + else: + main.add renderTextParagraph(paragraph, article) + + if list != nil: + main.add list + + buildHtml(tdiv(class="article-page")): + if article.coverImage.len > 0: + a(href=getOrigPicUrl(article.coverImage), target="_blank"): + img(class="article-cover", src=getSmallPic(article.coverImage), alt="") + main + renderToTop() diff --git a/src/views/community.nim b/src/views/community.nim new file mode 100644 index 0000000..52f9041 --- /dev/null +++ b/src/views/community.nim @@ -0,0 +1,128 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import strutils, strformat, times +import karax/[karaxdsl, vdom] + +import renderutils +import ".."/[types, utils, formatters] + +proc renderCommunityTabs*(kind: QueryKind; community: Community): VNode = + let + path = &"/i/communities/{community.id}" + q = Query(kind: kind) + buildHtml(tdiv): + ul(class="tab"): + li(class=q.getTabClass(posts)): + a(href=path): text "Top" + li(class=q.getTabClass(replies)): + a(href=(path & "/latest")): text "Latest" + li(class=q.getTabClass(media)): + a(href=(path & "/media")): text "Media" + li(class=q.getTabClass(userList)): + a(href=(path & "/about")): text "About" + if community.hashtags.len > 0: + tdiv(class="community-tags"): + for tag in community.hashtags: + let bare = tag.strip(chars={'#'}) + a(class="community-tag", + href=(&"/i/communities/{community.id}/hashtag/{bare}")): + text tag + +proc renderMemberTabs*(community: Community; isModerators: bool): VNode = + let path = &"/i/communities/{community.id}" + buildHtml(ul(class="tab")): + li(class=(if not isModerators: "tab-item active" else: "tab-item")): + a(href=(path & "/members")): text "All" + li(class=(if isModerators: "tab-item active" else: "tab-item")): + a(href=(path & "/moderators")): text "Moderators" + +proc renderHashtagHeader*(community: Community; tag: string): VNode = + buildHtml(tdiv(class="community-hashtag-header")): + h2(class="community-hashtag-title"): text "#" & tag + +proc renderCommunityAbout*(community: Community; moderators: seq[User]): VNode = + buildHtml(tdiv(class="community-about")): + tdiv(class="community-info"): + h2: text "Community Info" + tdiv(class="community-info-item"): + icon "group" + if community.joinPolicy == "Open": + text "Anyone can join this Community." + else: + text "Membership is by approval only." + + tdiv(class="community-info-item"): + icon "info" + text "All Communities are publicly visible." + + tdiv(class="community-info-item"): + icon "calendar" + let + date = community.createdAt.format("MMMM d, yyyy") + creator = community.creator.username + span: + text &"Created {date} by " + a(href=(&"/{creator}")): text &"@{creator}" + if community.creator.verifiedType != none: + verifiedIcon(community.creator) + + if community.rules.len > 0: + tdiv(class="community-rules"): + h2: text "Rules" + p(class="community-rules-intro"): + text "These are set and enforced by Community admins and are in addition to " + a(href="https://help.x.com/rules-and-policies/x-rules"): text "X's rules" + text "." + + for i, rule in community.rules: + tdiv(class="community-rule"): + span(class="community-rule-number"): text $(i + 1) + tdiv(class="community-rule-content"): + strong: text rule.name + if rule.description.len > 0: + p: text rule.description + + if moderators.len > 0: + tdiv(class="community-moderators"): + h2: + text "Moderators" + a(class="community-mods-link", + href=(&"/i/communities/{community.id}/moderators")): + text "See all" + for user in moderators: + tdiv(class="community-moderator"): + a(href=(&"/{user.username}")): + genImg(user.getUserPic("_bigger"), class="community-mod-avatar") + tdiv(class="community-mod-info"): + a(href=(&"/{user.username}"), class="community-mod-name"): + text user.fullname + if user.verifiedType != none: + verifiedIcon(user) + a(href=(&"/{user.username}"), class="community-mod-username"): + text &"@{user.username}" + +proc renderCommunity*(body, nav: VNode; community: Community): VNode = + buildHtml(tdiv(class="timeline-container")): + if community.banner.len > 0: + tdiv(class="timeline-banner"): + a(href=getPicUrl(community.banner), target="_blank"): + genImg(community.banner) + + tdiv(class="community-header"): + h1(class="community-name"): + a(href=(&"/i/communities/{community.id}")): text community.name + + if community.category.len > 0: + span(class="community-category"): text community.category + + if community.description.len > 0: + tdiv(class="community-description"): + text community.description + + tdiv(class="community-stats"): + a(class="community-member-count", + href=(&"/i/communities/{community.id}/members")): + text insertSep($community.memberCount, ',') + text " Members" + + nav + body diff --git a/src/views/general.nim b/src/views/general.nim index 4110bdc..37213c2 100644 --- a/src/views/general.nim +++ b/src/views/general.nim @@ -50,8 +50,8 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; let opensearchUrl = getUrlPrefix(cfg) & "/opensearch" buildHtml(head): - link(rel="stylesheet", type="text/css", href="/css/style.css?v=35") - link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=5") + link(rel="stylesheet", type="text/css", href="/css/style.css?v=45") + link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=7") if theme.len > 0: link(rel="stylesheet", type="text/css", href=(&"/css/themes/{theme}.css")) @@ -72,7 +72,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; if prefs.hlsPlayback: script(src="/js/hls.min.js", `defer`="") - script(src="/js/hlsPlayback.js", `defer`="") + script(src="/js/hlsPlayback.js?v=1", `defer`="") if prefs.infiniteScroll: script(src="/js/infiniteScroll.js", `defer`="") @@ -84,6 +84,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; text cfg.title meta(name="viewport", content="width=device-width, initial-scale=1.0") + meta(name="referrer", content="same-origin") meta(name="theme-color", content="#1F1F1F") meta(property="og:type", content=ogType) meta(property="og:title", content=(if ogTitle.len > 0: ogTitle else: titleText)) @@ -96,6 +97,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; link(rel="preload", type="image/png", href=bannerUrl, `as`="image") for url in images: + if url.len == 0: continue let preloadUrl = if "400x400" in url: getPicUrl(url) else: getSmallPic(url) link(rel="preload", type="image/png", href=preloadUrl, `as`="image") @@ -117,13 +119,16 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc=""; # this is last so images are also preloaded # if this is done earlier, Chrome only preloads one image for some reason link(rel="preload", type="font/woff2", `as`="font", - href="/fonts/fontello.woff2?61663884", crossorigin="anonymous") + href="/fonts/fontello.woff2?59696369", crossorigin="anonymous") proc renderMain*(body: VNode; req: Request; cfg: Config; prefs=defaultPrefs; titleText=""; desc=""; ogTitle=""; rss=""; video=""; - images: seq[string] = @[]; banner=""): string = + images: seq[string] = @[]; banner=""; + twitterLink=""): string = - let twitterLink = getTwitterLink(req.path, req.params) + let twitterLink = + if twitterLink.len > 0: twitterLink + else: getTwitterLink(req.path, req.params) let node = buildHtml(html(lang="en")): renderHead(prefs, cfg, req, titleText, desc, video, images, banner, ogTitle, diff --git a/src/views/profile.nim b/src/views/profile.nim index 5b751d6..de09368 100644 --- a/src/views/profile.nim +++ b/src/views/profile.nim @@ -2,7 +2,7 @@ import strutils, strformat import karax/[karaxdsl, vdom, vstyles] -import renderutils, search +import renderutils, search, timeline import ".."/[types, utils, formatters] proc renderStat(num: int; class: string; text=""): VNode = @@ -12,6 +12,13 @@ proc renderStat(num: int; class: string; text=""): VNode = span(class="profile-stat-num"): text insertSep($num, ',') +proc renderStatLink(num: int; class, href: string): VNode = + buildHtml(li(class=class)): + a(href=href): + span(class="profile-stat-header"): text capitalizeAscii(class) + span(class="profile-stat-num"): + text insertSep($num, ',') + proc renderUserCard*(user: User; prefs: Prefs; info: AccountInfo): VNode = buildHtml(tdiv(class="profile-card")): tdiv(class="profile-card-info"): @@ -65,8 +72,8 @@ proc renderUserCard*(user: User; prefs: Prefs; info: AccountInfo): VNode = tdiv(class="profile-card-extra-links"): ul(class="profile-statlist"): renderStat(user.tweets, "posts", text="Tweets") - renderStat(user.following, "following") - renderStat(user.followers, "followers") + renderStatLink(user.following, "following", &"/{user.username}/following") + renderStatLink(user.followers, "followers", &"/{user.username}/followers") renderStat(user.likes, "likes") proc renderPhotoRail(profile: Profile): VNode = @@ -99,7 +106,7 @@ proc renderBanner(banner: string): VNode = else: a(href=getPicUrl(banner), target="_blank"): genImg(banner) -proc renderProtected(username: string): VNode = +proc renderProtected*(username: string): VNode = buildHtml(tdiv(class="timeline-container")): tdiv(class="timeline-header timeline-protected"): h2: text "This account's tweets are protected." @@ -128,3 +135,31 @@ proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode = renderProtected(profile.user.username) else: renderTweetSearch(profile.tweets, prefs, path, profile.pinned) + +proc renderFollowTabs(user: User; activeTab: string): VNode = + buildHtml(ul(class="tab")): + for tab in ["Following", "Followers"]: + li(class=(if activeTab == tab: "tab-item active" else: "tab-item")): + a(href=(&"/{user.username}/{tab.toLowerAscii()}")): text tab + +proc renderUserList*(user: User; results: Result[User]; prefs: Prefs; + path, activeTab: string): VNode = + # Check if we've loaded all results (for page 1) + var displayResults = results + if results.beginning: + let expectedCount = if activeTab == "Followers": user.followers else: user.following + if results.content.len >= expectedCount: + displayResults.bottom = "" + + buildHtml(tdiv(class="profile-tabs")): + if not prefs.hideBanner: + tdiv(class="profile-banner"): + renderBanner(user.banner) + + let sticky = if prefs.stickyProfile: " sticky" else: "" + tdiv(class=("profile-tab" & sticky)): + renderUserCard(user, prefs, AccountInfo()) + + tdiv(class="timeline-container"): + renderFollowTabs(user, activeTab) + renderTimelineUsers(displayResults, prefs, path) diff --git a/src/views/renderutils.nim b/src/views/renderutils.nim index 0bd9789..af2f05d 100644 --- a/src/views/renderutils.nim +++ b/src/views/renderutils.nim @@ -7,18 +7,20 @@ const smallWebp* = "?name=small&format=webp" const mediumWebp* = "?name=medium&format=webp" proc getSmallPic*(url: string): string = + if url.len == 0: return result = url if "?" notin url and not url.endsWith("placeholder.png"): result &= smallWebp result = getPicUrl(result) proc getMediumPic*(url: string): string = + if url.len == 0: return result = url if "?" notin url and not url.endsWith("placeholder.png"): result &= mediumWebp result = getPicUrl(result) -proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode = +proc icon*(icon: string; label=""; title=""; class=""; href=""): VNode = var c = "icon-" & icon if class.len > 0: c = &"{c} {class}" buildHtml(tdiv(class="icon-container")): @@ -27,8 +29,8 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode = else: span(class=c, title=title) - if text.len > 0: - text " " & text + if label.len > 0: + text " " & label template verifiedIcon*(user: User): untyped {.dirty.} = if user.verifiedType != VerifiedType.none: diff --git a/src/views/rss.nimf b/src/views/rss.nimf index 4c1a86f..9c703d5 100644 --- a/src/views/rss.nimf +++ b/src/views/rss.nimf @@ -12,7 +12,7 @@ #elif retweet.len > 0: prefix = &"RT by @{retweet}: " #elif tweet.reply.len > 0: prefix = &"R to @{tweet.reply[0]}: " #end if -#var text = stripHtml(tweet.text) +#var text = strutils.splitWhitespace(stripHtml(tweet.text)).join(" ") ##if unicode.runeLen(text) > 32: ## text = unicode.runeSubStr(text, 0, 32) & "..." ##end if @@ -34,6 +34,18 @@ # end case # end if #end if +#if result.len == 0 and tweet.articlePreview.isSome: +# let art = tweet.articlePreview.get() +# if art.title.len > 0: +# result = prefix & xmltree.escape(art.title) +# end if +#end if +#if result.len == 0 and tweet.card.isSome: +# let card = tweet.card.get() +# if card.kind notin {hidden, unknown} and card.title.len > 0: +# result = prefix & xmltree.escape(card.title) +# end if +#end if #end proc # #proc getDescription(desc: string; cfg: Config): string = @@ -60,6 +72,72 @@ Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)} #end case #end proc # +#proc renderRssCard(card: Card; prefs: Prefs; urlPrefix: string): string = +#let cardLink = if card.url.startsWith("/"): urlPrefix & card.url else: replaceUrls(card.url, prefs) +#let title = xmltree.escape(card.title) +
+Link
+#if cardLink.len > 0: + +#end if +#if card.image.len > 0: + +#end if +#if title.len > 0: +# if card.image.len > 0: +
+# end if +${title} +#end if +#if cardLink.len > 0: + +#end if +#if card.text.len > 0: +

${xmltree.escape(card.text)}

+#end if +#if cardLink.len > 0: +# let destText = if card.dest.len > 0: xmltree.escape(card.dest) else: xmltree.escape(cardLink) +${destText} +#elif card.dest.len > 0: +${xmltree.escape(card.dest)} +#end if +#end proc +# +#proc renderRssArticle(article: ArticlePreview; urlPrefix: string): string = +#let link = urlPrefix & "/i/article/" & $article.tweetId +
+Article
+ +#if article.coverImage.len > 0: + +#end if +#if article.title.len > 0: +# if article.coverImage.len > 0: +
+# end if +${xmltree.escape(article.title)} +#end if + +#if article.previewText.len > 0: +

${xmltree.escape(article.previewText)}

+#end if +#end proc +# +#proc renderRssPoll(poll: Poll): string = +
+Poll +

+#for i in 0 ..< poll.options.len: +# let perc = if poll.votes > 0: poll.values[i] / poll.votes * 100 else: 0.0 +# let pct = (&"{perc:.0f}").strip(chars={'.'}) +# let line = pct & "% — " & xmltree.escape(poll.options[i]) +${line}
+#end for +#let votesStr = insertSep($poll.votes, ',') +${votesStr} votes • ${xmltree.escape(poll.status)} +

+#end proc +# #proc getTweetsWithPinned(profile: Profile): seq[Tweets] = #result = profile.tweets.content #if profile.pinned.isSome and result.len > 0: @@ -82,16 +160,21 @@ Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)} #let tweet = tweet.retweet.get(tweet) #let urlPrefix = getUrlPrefix(cfg) #let text = replaceUrls(tweet.text, prefs, absolute=urlPrefix) +#if text.len > 0:

${text.replace("\n", "
\n")}

+#end if #if tweet.media.len > 0: # for media in tweet.media: ${renderRssMedia(media, tweet, urlPrefix)} # end for -#elif tweet.card.isSome: -# let card = tweet.card.get() -# if card.image.len > 0: - -# end if +#elif tweet.card.isSome and tweet.card.get().kind notin {hidden, unknown}: +${renderRssCard(tweet.card.get(), prefs, urlPrefix)} +#end if +#if tweet.articlePreview.isSome: +${renderRssArticle(tweet.articlePreview.get(), urlPrefix)} +#end if +#if tweet.poll.isSome: +${renderRssPoll(tweet.poll.get())} #end if #if tweet.note.len > 0 and not prefs.hideCommunityNotes:

Community note: ${replaceUrls(tweet.note, prefs, absolute=urlPrefix)}

diff --git a/src/views/space.nim b/src/views/space.nim new file mode 100644 index 0000000..a5cac7b --- /dev/null +++ b/src/views/space.nim @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: AGPL-3.0-only +import strutils, times +import karax/[karaxdsl, vdom] + +import renderutils +import ".."/[types, utils, formatters] + +proc renderParticipant(p: SpaceParticipant; role: string): VNode = + buildHtml(tdiv(class="space-participant")): + a(href=("/" & p.username)): + genImg(p.avatarUrl.replace("_normal", "_bigger")) + tdiv(class="participant-info"): + tdiv(class="participant-name"): + strong: text p.displayName + if p.isVerified: + tdiv(class="verified-icon blue"): + icon "circle", class="verified-icon-circle", title="Verified account" + icon "ok", class="verified-icon-check", title="Verified account" + if role.len > 0: + span(class="host-badge"): text role + span(class="participant-username"): text "@" & p.username + +proc renderSpace*(sp: AudioSpace; prefs: Prefs; path: string): VNode = + let + isLive = sp.state == "RUNNING" + source = if prefs.proxyVideos and sp.m3u8Url.startsWith("http"): + getVidUrl(sp.m3u8Url) else: sp.m3u8Url + stateText = + if isLive: "LIVE" + elif sp.endTime.year > 1: "Ended " & sp.endTime.format("MMM d, YYYY") + elif sp.state.len > 0: sp.state + else: "Ended" + durationMs = + if sp.startTime.year > 1 and sp.endTime.year > 1: + int((sp.endTime - sp.startTime).inMilliseconds) + else: 0 + duration = if durationMs > 0: getDuration(durationMs) else: "" + totalListeners = + if sp.totalReplayWatched > 0: sp.totalReplayWatched + else: sp.totalLiveListeners + + buildHtml(tdiv(class="space-page")): + tdiv(class="space-panel"): + tdiv(class="space-player"): + if sp.m3u8Url.len > 0 and prefs.hlsPlayback: + audio(data-url=source, data-autoload="false") + verbatim "
" + tdiv(class="overlay-circle"): span(class="overlay-triangle") + if isLive: + tdiv(class="space-live"): text "LIVE" + elif duration.len > 0: + tdiv(class="overlay-duration"): text duration + verbatim "
" + elif sp.m3u8Url.len > 0: + tdiv(class="video-overlay"): + buttonReferer "/enablehls", "Enable hls playback", path + if isLive: + tdiv(class="space-live"): text "LIVE" + elif duration.len > 0: + tdiv(class="overlay-duration"): text duration + elif sp.availableForReplay: + tdiv(class="video-overlay"): + p: text "Audio stream unavailable" + else: + tdiv(class="video-overlay"): + p: text "Replay is not available" + + tdiv(class="space-info"): + tdiv(class="space-header"): + h2(class="space-title"): text sp.title + tdiv(class="space-meta"): + if totalListeners > 0: + span(class="listener-count"): text insertSep($totalListeners, ',') & " listeners" + if isLive: + span(class="space-live"): text stateText + else: + span(class="space-state"): text stateText + + if sp.admins.len > 0 or sp.speakers.len > 0: + tdiv(class="space-participants"): + for admin in sp.admins: + let role = if admin.username == sp.creator.username: "Host" + else: "Co-host" + renderParticipant(admin, role) + for speaker in sp.speakers: + renderParticipant(speaker, "") diff --git a/src/views/timeline.nim b/src/views/timeline.nim index b911765..9456bf9 100644 --- a/src/views/timeline.nim +++ b/src/views/timeline.nim @@ -154,7 +154,8 @@ proc renderTimelineTweets*(results: Timeline; prefs: Prefs; path: string; else: renderThread(thread, prefs, path, bigThumb) else: for thread in filtered: - if thread.len == 1: renderTweet(thread[0], prefs, path) + if thread.len == 1: + renderTweet(thread[0], prefs, path) else: renderThread(thread, prefs, path) var cursor = getSearchMaxId(results, path) diff --git a/src/views/tweet.nim b/src/views/tweet.nim index e15cf1d..b66269a 100644 --- a/src/views/tweet.nim +++ b/src/views/tweet.nim @@ -9,17 +9,36 @@ import general const doctype = "\n" -proc renderMiniAvatar(user: User; prefs: Prefs): VNode = +proc renderMiniAvatar*(user: User; prefs: Prefs): VNode = genImg(user.getUserPic("_mini"), class=(prefs.getAvatarClass & " mini")) -proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs): VNode = +proc renderArticleCard(preview: ArticlePreview; prefs: Prefs): VNode = + let url = "/i/article/" & $preview.tweetId + buildHtml(tdiv(class="article-card card large")): + a(class="card-container", href=url): + if preview.coverImage.len > 0: + tdiv(class="card-image-container"): + tdiv(class="card-image"): + genImg(preview.coverImage) + span(class="article-card-badge"): text "Article" + tdiv(class="card-content-container"): + tdiv(class="card-content"): + h2(class="card-title"): text preview.title + if preview.previewText.len > 0: + p(class="card-description"): text preview.previewText + +proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs; + path = ""): VNode = buildHtml(tdiv): if pinned: + let pinnedLabel = + if "/i/communities/" in path: "Pinned by Community mods" + else: "Pinned Tweet" tdiv(class="pinned"): - span: icon "pin", "Pinned Tweet" + span: icon("pin", pinnedLabel) elif retweet.len > 0: tdiv(class="retweet-header"): - span: icon "retweet", retweet & " retweeted" + span: icon("retweet", retweet & " retweeted") tdiv(class="tweet-header"): a(class="tweet-avatar", href=("/" & tweet.user.username)): @@ -66,7 +85,7 @@ proc renderVideoDisabled(playbackType: VideoType; path=""): VNode = buildHtml(tdiv(class="video-overlay")): case playbackType of mp4: - p: text "mp4 playback disabled in preferences" + buttonReferer "/enablemp4", "Enable mp4 playback", path of m3u8, vmap: buttonReferer "/enablehls", "Enable hls playback", path @@ -78,6 +97,12 @@ proc renderVideoUnavailable(video: Video): VNode = else: p: text "This media is unavailable" +proc getVideoDownloadUrl(videoData: Video): string = + let mp4Vars = videoData.variants.filterIt(it.contentType == mp4) + if mp4Vars.len == 0: return "" + let best = mp4Vars.sortedByIt(it.bitrate)[^1].url + if best.startsWith("http"): getVidUrl(best) else: best + proc renderVideoAttachment(videoData: Video; prefs: Prefs; path=""; bigThumb=false): VNode = let playbackType = if not prefs.proxyVideos and videoData.hasMp4Url: mp4 @@ -108,6 +133,11 @@ proc renderVideoAttachment(videoData: Video; prefs: Prefs; path=""; bigThumb=fal if videoData.durationMs > 0: tdiv(class="overlay-duration"): text getDuration(videoData) verbatim "" + if videoData.available: + let dlUrl = getVideoDownloadUrl(videoData) + if dlUrl.len > 0: + a(class="video-download", href=dlUrl, download="", + title="Download video"): icon "download-alt" proc renderVideo*(video: Video; prefs: Prefs; path: string; bigThumb=false): VNode = let hasCardContent = video.description.len > 0 or video.title.len > 0 @@ -121,13 +151,13 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string; bigThumb=false): VNo if video.description.len > 0: p(class="card-description"): text video.description -proc renderGifAttachment(gif: Gif; prefs: Prefs): VNode = +proc renderGifAttachment(gif: Gif; prefs: Prefs; path=""): VNode = let thumb = getSmallPic(gif.thumb) buildHtml(tdiv(class="attachment")): if not prefs.mp4Playback: img(src=thumb, loading="lazy") - renderVideoDisabled(mp4) + renderVideoDisabled(mp4, path) elif prefs.autoplayGifs: video(class="gif", poster=thumb, autoplay="", muted="", loop=""): source(src=getPicUrl(gif.url), `type`="video/mp4") @@ -137,9 +167,9 @@ proc renderGifAttachment(gif: Gif; prefs: Prefs): VNode = if gif.altText.len > 0: renderAltText(gif.altText) -proc renderGif(gif: Gif; prefs: Prefs): VNode = +proc renderGif(gif: Gif; prefs: Prefs; path=""): VNode = buildHtml(tdiv(class="attachments media-gif")): - renderGifAttachment(gif, prefs) + renderGifAttachment(gif, prefs, path) proc renderMedia(media: seq[Media]; prefs: Prefs; path: string; bigThumb=false): VNode = if media.len == 0: @@ -150,7 +180,7 @@ proc renderMedia(media: seq[Media]; prefs: Prefs; path: string; bigThumb=false): if item.kind == videoMedia: return renderVideo(item.video, prefs, path, bigThumb) if item.kind == gifMedia: - return renderGif(item.gif, prefs) + return renderGif(item.gif, prefs, path) let groups = if media.len < 3: @[media] @@ -169,7 +199,7 @@ proc renderMedia(media: seq[Media]; prefs: Prefs; path: string; bigThumb=false): of videoMedia: renderVideoAttachment(mediaItem.video, prefs, path, bigThumb) of gifMedia: - renderGifAttachment(mediaItem.gif, prefs) + renderGifAttachment(mediaItem.gif, prefs, path) proc renderPoll(poll: Poll): VNode = buildHtml(tdiv(class="poll")): @@ -225,7 +255,7 @@ func formatStat(stat: int): string = if stat > 0: insertSep($stat, ',') else: "" -proc renderStats(stats: TweetStats): VNode = +proc renderStats*(stats: TweetStats): VNode = buildHtml(tdiv(class="tweet-stats")): span(class="tweet-stat"): icon "comment", formatStat(stats.replies) span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets) @@ -239,8 +269,9 @@ proc renderReply(tweet: Tweet): VNode = if i > 0: text " " a(href=("/" & u)): text "@" & u -proc renderAttribution(user: User; prefs: Prefs): VNode = - buildHtml(a(class="attribution", href=("/" & user.username))): +proc renderAttribution(user: User; prefs: Prefs; link = ""): VNode = + let href = if link.len > 0: link else: "/" & user.username + buildHtml(a(class="attribution", href=href)): renderMiniAvatar(user, prefs) strong: text user.fullname verifiedIcon(user) @@ -307,6 +338,9 @@ proc renderQuote(quote: Tweet; prefs: Prefs; path: string): VNode = if quote.media.len > 0: renderQuoteMedia(quote, prefs, path) + if quote.articlePreview.isSome: + renderArticleCard(quote.articlePreview.get(), prefs) + if quote.note.len > 0 and not prefs.hideCommunityNotes: renderCommunityNote(quote.note, prefs) @@ -322,10 +356,10 @@ proc renderDisclosures*(tweet: Tweet): VNode = buildHtml(tdiv(class="disclosures")): if tweet.isAI: span(data-disclosure="ai"): - icon "attention", "Made with AI" + icon "attention-circled", "Made with AI" if tweet.isAd: span(data-disclosure="ad"): - icon "attention", "Paid partnership (ad)" + icon "attention-circled", "Paid partnership (ad)" proc renderLocation*(tweet: Tweet): string = let (place, url) = tweet.getLocation() @@ -339,7 +373,8 @@ proc renderLocation*(tweet: Tweet): string = return $node proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; - last=false; mainTweet=false; afterTweet=false; bigThumb=false): VNode = + last=false; mainTweet=false; afterTweet=false; + bigThumb=false): VNode = var divClass = class if index == -1 or last: divClass = "thread-last " & class @@ -372,7 +407,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; a(class="tweet-link", href=getLink(tweet)) tdiv(class="tweet-body"): - renderHeader(tweet, retweet, pinned, prefs) + renderHeader(tweet, retweet, pinned, prefs, path) if not afterTweet and index == 0 and tweet.reply.len > 0 and (tweet.reply.len > 1 or tweet.reply[0] != tweet.user.username or pinned): @@ -386,11 +421,14 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0; verbatim replaceUrls(tweet.text, prefs) & renderLocation(tweet) if tweet.attribution.isSome: - renderAttribution(tweet.attribution.get(), prefs) + renderAttribution(tweet.attribution.get(), prefs, tweet.attributionLink) if tweet.card.isSome and tweet.card.get().kind != hidden: renderCard(tweet.card.get(), prefs, path) + if tweet.articlePreview.isSome: + renderArticleCard(tweet.articlePreview.get(), prefs) + if tweet.media.len > 0: renderMedia(tweet.media, prefs, path, bigThumb) diff --git a/tests/test_about_account.py b/tests/test_about_account.py new file mode 100644 index 0000000..2239d9d --- /dev/null +++ b/tests/test_about_account.py @@ -0,0 +1,76 @@ +from base import BaseTestCase, Profile +from parameterized import parameterized + + +class AboutAccount(object): + header = '.about-account-header' + name = '.about-account-name' + body = '.about-account-body' + row = '.about-account-row' + label = '.about-account-label' + value = '.about-account-value' + + +# (username, expected_labels) +# Each label is checked for presence in the page text +about_data = [ + ['jack', ['Date joined', 'Account based in', 'Connected via']], + ['NASA', ['Date joined']], + ['elonmusk', ['Date joined']], +] + +about_verified = [ + ['jack', 'Verified', 'Since '], +] + +about_affiliate = [ + ['jack', 'An affiliate of', 'Square'], + ['elonmusk', 'An affiliate of', 'X'], +] + + +class AboutAccountTest(BaseTestCase): + @parameterized.expand(about_data) + def test_about_page_has_labels(self, username, expected_labels): + """About page shows expected info labels""" + self.open_nitter(f'{username}/about') + self.assert_element_visible(AboutAccount.header) + self.assert_element_visible(AboutAccount.body) + for label in expected_labels: + self.assert_text(label, AboutAccount.body) + + @parameterized.expand(about_verified) + def test_about_verified(self, username, label, value_prefix): + """About page shows verification info for verified accounts""" + self.open_nitter(f'{username}/about') + self.assert_text(label, AboutAccount.body) + self.assert_text(value_prefix, AboutAccount.body) + + @parameterized.expand(about_affiliate) + def test_about_affiliate(self, username, label, affiliate): + """About page shows affiliate info""" + self.open_nitter(f'{username}/about') + self.assert_text(label, AboutAccount.body) + self.assert_text(f'@{affiliate}', AboutAccount.body) + + def test_about_page_title(self): + """Title contains account name""" + self.open_nitter('jack/about') + self.assert_text('jack', AboutAccount.name) + + def test_about_join_date(self): + """About page always shows join date""" + self.open_nitter('jack/about') + self.assert_text('Date joined', AboutAccount.body) + self.assert_text('March 2006', AboutAccount.body) + + def test_about_invalid_user(self): + """About page for non-existent user shows error""" + self.open_nitter('thisprofiledoesntexist/about') + self.assert_text('User "thisprofiledoesntexist" not found') + + def test_joindate_links_to_about(self): + """Join date on profile page links to about page""" + self.open_nitter('jack') + link = self.find_element(Profile.joinDate + ' a') + self.assertIn('/jack/about', link.get_attribute('href')) diff --git a/tests/test_article.py b/tests/test_article.py new file mode 100644 index 0000000..652ec76 --- /dev/null +++ b/tests/test_article.py @@ -0,0 +1,325 @@ +from base import BaseTestCase +from parameterized import parameterized + + +class ArticleSelectors: + page = '.article-page' + cover = '.article-cover' + body = '.article-body' + title = '.article-title' + author = '.article-author' + fullname = '.article-author .fullname' + username = '.article-author .username' + date = '.article-author .article-date' + avatar = '.article-author img.avatar' + verified = '.article-author .verified-icon' + media = '.article-media' + caption = '.article-media-caption' + divider = '.article-divider' + + +articles = [ + ['2064166507438059759', + '1s，秒杀一切，开源一个 X 文章发布 Skill【重磅升级】', + 'punk2898', 'Punk'], + + ['2064689664213041529', + 'SpaceX Thesis & Valuation Memorandum', + 'Dialectic_Group', 'Dialectic'], + + ['2064691088636424322', + 'Consciousness and AI: The Problem of Inner Experience', + 'CosmicOrFun', 'Cosmic Orphan'], + + ['2064755789391110154', + 'DeFi Markets Update 2026-06-10', + 'SteakhouseFi', 'Steakhouse Financial'], + + ['2064755231901319527', + 'The machine economy has a killswitch and somebody just pulled it.', + '1914ad', 'Justin Bechler HMP-028'], + + ['2062858677149675788', + 'Yakshinis', + 'CosmicOrFun', 'Cosmic Orphan'], +] + +articles_with_media = [ + ['2064166507438059759', 6], + ['2064689664213041529', 11], + ['2064755789391110154', 5], +] + +articles_with_dividers = [ + ['2064166507438059759', 1], + ['2064689664213041529', 6], +] + + +class ArticleBasicTest(BaseTestCase): + @parameterized.expand(articles) + def test_article_loads(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + self.assert_element_visible(ArticleSelectors.page) + self.assert_element_visible(ArticleSelectors.body) + self.assert_text(title, ArticleSelectors.title) + + @parameterized.expand(articles) + def test_article_author(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + self.assert_element_visible(ArticleSelectors.author) + self.assert_text(f'@{username}', ArticleSelectors.username) + + @parameterized.expand(articles) + def test_article_has_cover(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + self.assert_element_visible(ArticleSelectors.cover) + src = self.get_attribute(ArticleSelectors.cover, 'src') + self.assertIn('/pic/', src) + + @parameterized.expand(articles) + def test_article_has_date(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + date_text = self.get_text(ArticleSelectors.date) + self.assertTrue(len(date_text) > 3) + + @parameterized.expand(articles) + def test_article_author_avatar(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + self.assert_element_visible(ArticleSelectors.avatar) + src = self.get_attribute(ArticleSelectors.avatar, 'src') + self.assertIn('/pic/', src) + self.assertGreater(len(src), len('/pic/')) + + @parameterized.expand(articles) + def test_article_author_verified(self, tweet_id, title, username, fullname): + self.open_nitter(f'i/article/{tweet_id}') + self.assert_element_visible(ArticleSelectors.verified) + + def test_article_author_verified_business(self): + self.open_nitter('i/article/2064755789391110154') + self.assert_element_visible('.article-author .verified-icon.business') + + +class ArticleContentTest(BaseTestCase): + def test_article_has_paragraphs(self): + self.open_nitter('i/article/2064689664213041529') + paragraphs = self.find_elements('.article-body p') + self.assertGreater(len(paragraphs), 10) + + def test_article_has_headers(self): + self.open_nitter('i/article/2064689664213041529') + headers = self.find_elements('.article-body h1, .article-body h2') + self.assertGreater(len(headers), 5) + + def test_article_has_bold_text(self): + self.open_nitter('i/article/2064166507438059759') + bold = self.find_elements('.article-body strong') + self.assertGreater(len(bold), 0) + + def test_article_has_italic_text(self): + self.open_nitter('i/article/2064166507438059759') + italic = self.find_elements('.article-body em') + self.assertGreater(len(italic), 0) + + def test_article_has_blockquotes(self): + self.open_nitter('i/article/2064166507438059759') + self.assert_element_visible('.article-body blockquote') + + def test_article_has_lists(self): + self.open_nitter('i/article/2064166507438059759') + self.assert_element_visible('.article-body ul') + + def test_article_has_emoji_text(self): + self.open_nitter('i/article/2064166507438059759') + body = self.get_text(ArticleSelectors.body) + self.assertTrue(any(ord(c) > 0x1F000 for c in body)) + + def test_article_has_links(self): + self.open_nitter('i/article/2064691088636424322') + links = self.find_elements('.article-body a[href]') + self.assertGreater(len(links), 0) + + def test_article_twitter_links_localized(self): + self.open_nitter('i/article/2064755789391110154') + links = self.find_elements('.article-body a[href^="https://x.com"]') + self.assertEqual(len(links), 0, 'x.com links should be converted to local paths') + + @parameterized.expand(articles_with_media) + def test_article_media_count(self, tweet_id, expected_count): + self.open_nitter(f'i/article/{tweet_id}') + media = self.find_elements(ArticleSelectors.media) + self.assertEqual(len(media), expected_count) + + @parameterized.expand(articles_with_dividers) + def test_article_divider_count(self, tweet_id, expected_count): + self.open_nitter(f'i/article/{tweet_id}') + dividers = self.find_elements(ArticleSelectors.divider) + self.assertEqual(len(dividers), expected_count) + + +class ArticleMediaTest(BaseTestCase): + def test_media_images_proxied(self): + self.open_nitter('i/article/2064689664213041529') + self.assert_element_visible(ArticleSelectors.media) + img = self.find_element(f'{ArticleSelectors.media} img') + src = img.get_attribute('src') + self.assertIn('/pic/', src) + self.assertFalse(src.startswith('https://pbs.twimg.com')) + + def test_cover_image_proxied(self): + self.open_nitter('i/article/2064689664213041529') + self.assert_element_visible(ArticleSelectors.cover) + src = self.get_attribute(ArticleSelectors.cover, 'src') + self.assertIn('/pic/', src) + self.assertFalse(src.startswith('https://pbs.twimg.com')) + + def test_embedded_tweet(self): + self.open_nitter('i/article/2064755789391110154') + self.assert_element_visible('.article-body .timeline-item') + + def test_multiple_embedded_tweets(self): + self.open_nitter('i/article/2064755231901319527') + tweets = self.find_elements('.article-body .timeline-item') + self.assertGreaterEqual(len(tweets), 3) + + def test_media_caption_displayed(self): + self.open_nitter('i/article/2064689664213041529') + self.assert_element_visible(ArticleSelectors.caption) + captions = self.find_elements(ArticleSelectors.caption) + self.assertGreaterEqual(len(captions), 5) + + def test_media_caption_text(self): + self.open_nitter('i/article/2064689664213041529') + self.assert_text_visible('FIGURE 1', ArticleSelectors.caption) + + def test_media_caption_alt_attribute(self): + self.open_nitter('i/article/2064689664213041529') + img = self.find_element(f'{ArticleSelectors.media} img') + alt = img.get_attribute('alt') + self.assertGreater(len(alt), 0) + + def test_no_caption_when_absent(self): + self.open_nitter('i/article/2062858677149675788') + captions = self.find_elements(ArticleSelectors.caption) + self.assertEqual(len(captions), 0) + + +class ArticleMentionTest(BaseTestCase): + def test_mention_linkified(self): + self.open_nitter('i/article/2064755231901319527') + link = self.find_element('.article-body a[href="/ZachXBT"]') + self.assertEqual(link.text, '@ZachXBT') + + def test_multiple_mentions_linkified(self): + self.open_nitter('i/article/2064755231901319527') + links = self.find_elements('.article-body a[href^="/"]') + mention_hrefs = [l.get_attribute('href') for l in links + if l.text.startswith('@')] + usernames = [h.split('/')[-1] for h in mention_hrefs] + self.assertIn('ZachXBT', usernames) + self.assertIn('River', usernames) + + def test_mention_in_different_article(self): + self.open_nitter('i/article/2064689664213041529') + link = self.find_element('.article-body a[href="/FutureJurvetson"]') + self.assertEqual(link.text, '@FutureJurvetson') + + def test_no_spurious_whitespace_in_styled_paragraph(self): + """Styled paragraphs should not have extra whitespace from VNode serialization.""" + self.open_nitter('i/article/2064166507438059759') + source = self.get_page_source() + self.assertNotIn('white-space: pre-wrap', source) + self.assertNotIn('white-space:pre-wrap', source) + + +class ArticleCardTest(BaseTestCase): + @parameterized.expand(articles) + def test_status_page_shows_article_card(self, tweet_id, title, username, fullname): + self.open_nitter(f'{username}/status/{tweet_id}') + self.assert_element_visible('.article-card') + self.assert_text(title, '.article-card .card-title') + + def test_article_card_has_cover_image(self): + self.open_nitter('Dialectic_Group/status/2064689664213041529') + self.assert_element_visible('.article-card .card-image img') + src = self.get_attribute('.article-card .card-image img', 'src') + self.assertIn('/pic/', src) + + def test_article_card_has_badge(self): + self.open_nitter('Dialectic_Group/status/2064689664213041529') + self.assert_element_visible('.article-card-badge') + self.assert_text('Article', '.article-card-badge') + + def test_article_card_has_preview_text(self): + self.open_nitter('CosmicOrFun/status/2064691088636424322') + self.assert_element_visible('.article-card .card-description') + + def test_article_card_links_to_article(self): + self.open_nitter('punk2898/status/2064166507438059759') + href = self.get_attribute('.article-card .card-container', 'href') + self.assertIn('/article/', href) + + def test_article_url_stripped_from_tweet_text(self): + self.open_nitter('punk2898/status/2064166507438059759') + self.assert_element_visible('.article-card') + source = self.get_page_source() + # Main tweet text should not contain article URL + import re + main = re.search(r'id="m".*?tweet-content[^>]*>(.*?)', source, re.DOTALL) + self.assertIsNotNone(main) + self.assertNotIn('/article/', main.group(1)) + + +class ArticleQuotedCardTest(BaseTestCase): + """Article cards inside quoted tweets (1914ad quoting own article).""" + quoted_tweet = '1914ad/status/2064789532071891085' + quoted_article_id = '2063677483548102688' + + def test_quoted_card_visible(self): + self.open_nitter(self.quoted_tweet) + self.assert_element_visible('.quote .article-card') + + def test_quoted_card_has_title(self): + self.open_nitter(self.quoted_tweet) + self.assert_text('David Bailey Already Won', '.quote .article-card .card-title') + + def test_quoted_card_has_badge(self): + self.open_nitter(self.quoted_tweet) + self.assert_element_visible('.quote .article-card-badge') + self.assert_text('Article', '.quote .article-card-badge') + + def test_quoted_card_has_cover_image(self): + self.open_nitter(self.quoted_tweet) + self.assert_element_visible('.quote .article-card .card-image img') + src = self.get_attribute('.quote .article-card .card-image img', 'src') + self.assertIn('/pic/', src) + + def test_quoted_card_has_description(self): + self.open_nitter(self.quoted_tweet) + self.assert_element_visible('.quote .article-card .card-description') + + def test_quoted_card_links_to_article(self): + self.open_nitter(self.quoted_tweet) + href = self.get_attribute('.quote .article-card .card-container', 'href') + self.assertIn(f'/article/{self.quoted_article_id}', href) + + +class ArticleRoutingTest(BaseTestCase): + def test_username_article_route_redirects(self): + self.open_nitter('punk2898/article/2064166507438059759') + self.assert_element_visible(ArticleSelectors.page) + self.assert_text('1s', ArticleSelectors.title) + + def test_status_article_route_redirects(self): + self.open_nitter('punk2898/status/2064166507438059759/article') + self.assert_element_visible(ArticleSelectors.page) + self.assert_text('1s', ArticleSelectors.title) + + def test_invalid_id_returns_404(self): + self.open_nitter('i/article/notanumber') + self.assert_element_not_visible(ArticleSelectors.page) + + def test_nonexistent_article(self): + self.open_nitter('i/article/1') + self.assert_element_visible('.error-panel') diff --git a/tests/test_card.py b/tests/test_card.py index 129e65a..16863ea 100644 --- a/tests/test_card.py +++ b/tests/test_card.py @@ -11,18 +11,18 @@ card = [ ['voidtarget/status/1094632512926605312', 'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too)', 'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim', - 'gist.github.com', True] + 'gist.github.com', True], + + ['NASA/status/2061872347477418301', + 'Nancy Grace Roman Space Telescope - NASA Science', + 'The Nancy Grace Roman Space Telescope will settle essential questions in the areas of dark energy, exoplanets, and astrophysics.', + 'science.nasa.gov', True] ] no_thumb = [ - ['FluentAI/status/1116417904831029248', - 'LinkedIn', - 'This link will take you to a page that’s not on LinkedIn', - 'lnkd.in'], - ['Thom_Wolf/status/1122466524860702729', - 'GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in', - '', + 'GitHub - facebookresearch/XLM: PyTorch original implementation of Cross-lingual Language Model', + 'PyTorch original implementation of Cross-lingual Language Model Pretraining.', 'github.com'], ['brent_p/status/1088857328680488961', @@ -37,14 +37,9 @@ no_thumb = [ ] playable = [ - ['nim_lang/status/1118234460904919042', - 'Nim development blog 2019-03', - 'Arne (aka Krux02)* debugging: * improved nim-gdb, $ works, framefilter * alias for --debugger:native: -g* bugs: * forwarding of .pure. * sizeof union* fe...', - 'youtube.com'], - - ['nim_lang/status/1121090879823986688', - 'Nim - First natively compiled language w/ hot code-reloading at...', - '#nim #c++ #ACCUConfNim is a statically typed systems and applications programming language which offers perhaps some of the most powerful metaprogramming cap...', + ['NASA/status/2047048645845897398', + 'NASA\'s Artemis II News Conference with Moon Astronauts', + 'Live from NASA\'s Johnson Space Center in Houston', 'youtube.com'] ] @@ -72,7 +67,7 @@ class CardTest(BaseTestCase): if len(description) > 0: self.assert_text(description, c.description) - @parameterized.expand(playable) + @parameterized.expand(playable, skip_on_empty=True) def test_card_playable(self, tweet, title, description, destination): self.open_nitter(tweet) c = Card(Conversation.main + " ") diff --git a/tests/test_community.py b/tests/test_community.py new file mode 100644 index 0000000..b266044 --- /dev/null +++ b/tests/test_community.py @@ -0,0 +1,211 @@ +from base import BaseTestCase +from parameterized import parameterized + + +COMMUNITY_ID = '1493446837214187523' +COMMUNITY_PATH = f'i/communities/{COMMUNITY_ID}' + + +class CommunityTest(BaseTestCase): + def test_top_page_loads(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.community-header') + self.assert_text('Build in Public', '.community-name') + + def test_banner_visible(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.timeline-banner img') + + def test_member_count(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.community-member-count') + self.assert_text('Members', '.community-member-count') + + def test_description_visible(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.community-description') + + def test_tabs_present(self): + self.open_nitter(COMMUNITY_PATH) + tabs = self.find_elements('.tab a') + labels = [t.text for t in tabs] + self.assertEqual(labels, ['Top', 'Latest', 'Media', 'About']) + + def test_top_tab_active(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.tab .active a[href$="/' + COMMUNITY_ID + '"]') + + def test_top_has_tweets(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.timeline-item .tweet-body') + + def test_top_has_pagination(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.show-more') + self.assert_text('Load more', '.show-more') + + def test_latest_has_tweets(self): + self.open_nitter(f'{COMMUNITY_PATH}/latest') + self.assert_element_visible('.timeline-item .tweet-body') + + def test_latest_tab_active(self): + self.open_nitter(f'{COMMUNITY_PATH}/latest') + self.assert_element_visible('.tab .active a[href$="/latest"]') + + def test_media_has_tweets(self): + self.open_nitter(f'{COMMUNITY_PATH}/media') + self.assert_element_visible('.timeline-item .tweet-body') + + def test_media_tab_active(self): + self.open_nitter(f'{COMMUNITY_PATH}/media') + self.assert_element_visible('.tab .active a[href$="/media"]') + + def test_about_page(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + self.assert_element_visible('.community-about') + self.assert_text('Community Info', '.community-info h2') + + def test_about_rules(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + self.assert_element_visible('.community-rules') + self.assert_text('Rules', '.community-rules h2') + rules = self.find_elements('.community-rule') + self.assertGreater(len(rules), 0) + + def test_about_creator(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + self.assert_text('Created', '.community-info') + link = self.find_element('.community-info-item a') + self.assertTrue(link.text.startswith('@')) + self.assertGreater(len(link.text), 1) + + def test_about_tab_active(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + self.assert_element_visible('.tab .active a[href$="/about"]') + + def test_about_moderators(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + self.assert_element_visible('.community-moderators') + self.assert_text('Moderators', '.community-moderators h2') + mods = self.find_elements('.community-moderator') + self.assertGreater(len(mods), 0) + + def test_about_moderators_have_avatars(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + avatars = self.find_elements('.community-mod-avatar') + self.assertGreater(len(avatars), 0) + + def test_about_moderators_link_to_profiles(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + links = self.find_elements('.community-mod-username') + self.assertGreater(len(links), 0) + for link in links: + self.assertTrue(link.text.startswith('@')) + self.assertTrue(link.get_attribute('href').startswith('http')) + + def test_about_see_all_link(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + link = self.find_element('.community-mods-link') + self.assertEqual(link.text, 'See all') + self.assertIn('/moderators', link.get_attribute('href')) + + def test_members_page(self): + self.open_nitter(f'{COMMUNITY_PATH}/members') + self.assert_element_visible('.timeline-item') + users = self.find_elements('.timeline-item .username') + self.assertGreater(len(users), 0) + + def test_members_has_member_tabs(self): + self.open_nitter(f'{COMMUNITY_PATH}/members') + tabs = self.find_elements('.tab a') + labels = [t.text for t in tabs] + self.assertEqual(labels, ['All', 'Moderators']) + + def test_members_all_tab_active(self): + self.open_nitter(f'{COMMUNITY_PATH}/members') + self.assert_element_visible('.tab .active a[href$="/members"]') + + def test_members_count_is_link(self): + self.open_nitter(COMMUNITY_PATH) + link = self.find_element('.community-member-count') + self.assertIn('Members', link.text) + self.assertIn('/members', link.get_attribute('href')) + + def test_moderators_page(self): + self.open_nitter(f'{COMMUNITY_PATH}/moderators') + self.assert_element_visible('.timeline-item') + users = self.find_elements('.timeline-item .username') + self.assertGreater(len(users), 0) + + def test_moderators_tab_active(self): + self.open_nitter(f'{COMMUNITY_PATH}/moderators') + self.assert_element_visible('.tab .active a[href$="/moderators"]') + + def test_moderators_has_member_tabs(self): + self.open_nitter(f'{COMMUNITY_PATH}/moderators') + tabs = self.find_elements('.tab a') + labels = [t.text for t in tabs] + self.assertEqual(labels, ['All', 'Moderators']) + + def test_pinned_tweet_label(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.pinned') + self.assert_text('Pinned by Community mods', '.pinned') + + def test_hashtags_visible(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.community-tags') + tags = self.find_elements('.community-tag') + self.assertGreater(len(tags), 0) + + def test_hashtags_are_links(self): + self.open_nitter(COMMUNITY_PATH) + tags = self.find_elements('.community-tag') + for tag in tags: + href = tag.get_attribute('href') + self.assertIn('/hashtag/', href) + self.assertTrue(tag.text.startswith('#')) + + def test_hashtag_page(self): + self.open_nitter(f'{COMMUNITY_PATH}/hashtag/buildinpublic') + self.assert_element_visible('.timeline-item .tweet-body') + + def test_hashtag_shows_header(self): + self.open_nitter(f'{COMMUNITY_PATH}/hashtag/buildinpublic') + self.assert_element_visible('.community-header') + self.assert_text('Build in Public', '.community-name') + + def test_hashtag_shows_tag_title(self): + self.open_nitter(f'{COMMUNITY_PATH}/hashtag/buildinpublic') + self.assert_element_visible('.community-hashtag-header') + self.assert_text('#buildinpublic', '.community-hashtag-title') + + def test_hashtag_no_main_tabs(self): + self.open_nitter(f'{COMMUNITY_PATH}/hashtag/buildinpublic') + tabs = self.find_elements('.tab a') + tab_labels = [t.text for t in tabs] + self.assertNotIn('Top', tab_labels) + self.assertNotIn('About', tab_labels) + + def test_category_visible(self): + self.open_nitter(COMMUNITY_PATH) + self.assert_element_visible('.community-category') + + def test_about_join_policy(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + self.assert_text('Anyone can join', '.community-info') + + def test_about_visibility_note(self): + self.open_nitter(f'{COMMUNITY_PATH}/about') + self.assert_text('publicly visible', '.community-info') + + def test_404_invalid_id(self): + self.open_nitter('i/communities/999') + self.assert_element_visible('.error-panel') + self.assert_text('not found', '.error-panel') + + @parameterized.expand(['', '/latest', '/media', '/about', + '/members', '/moderators']) + def test_page_no_error(self, suffix): + self.open_nitter(f'{COMMUNITY_PATH}{suffix}') + self.assert_element_not_visible('.error-panel') diff --git a/tests/test_followers.py b/tests/test_followers.py new file mode 100644 index 0000000..ccc103b --- /dev/null +++ b/tests/test_followers.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: AGPL-3.0-only +from base import BaseTestCase, Timeline + + +class FollowersTest(BaseTestCase): + """Tests for followers and following pages""" + + def test_followers_page_loads(self): + """Test that followers page loads and shows users""" + self.open_nitter('jack/followers') + self.assert_title_contains('following @jack') + # Check for user list + self.assert_element('.timeline-item') + + def test_following_page_loads(self): + """Test that following page loads and shows users""" + self.open_nitter('jack/following') + self.assert_title_contains('followed by @jack') + # Check for user list + self.assert_element('.timeline-item') + + def test_followers_has_navigation_tabs(self): + """Test that followers page has following/followers tabs""" + self.open_nitter('jack/followers') + self.assert_element('.profile-statlist') + # Check for Following and Followers links + self.assert_element('a[href="/jack/following"]') + self.assert_element('a[href="/jack/followers"]') + + def test_following_pagination(self): + """Test that following page has load more button or scroll-to-top""" + self.open_nitter('jack/following') + # Should have either more button or top-ref (scroll to top arrow) + try: + self.assert_element('.show-more') + except: + self.assert_element('.top-ref') + + def test_nonexistent_user_followers(self): + """Test 404 for non-existent user""" + self.open_nitter('this_user_definitely_does_not_exist_12345/followers') + self.assert_element('.error-panel') + self.assert_text_visible('not found') diff --git a/tests/test_security.py b/tests/test_security.py new file mode 100644 index 0000000..65d5e9a --- /dev/null +++ b/tests/test_security.py @@ -0,0 +1,60 @@ +import subprocess +from parameterized import parameterized + +BASE_URL = 'http://localhost:8080' + + +def curl_status(url): + """Get HTTP status code using curl to avoid URL normalization by Python libs.""" + result = subprocess.run( + ['curl', '-s', '-o', '/dev/null', '-w', '%{http_code}', url], + capture_output=True, text=True, timeout=10 + ) + return int(result.stdout) + + +class TestMalformedPaths: + """Test that malformed paths don't crash the server. + + URLs like //foo are parsed as having 'foo' as the authority (host), + resulting in an empty path. Empty paths previously crashed jester's + static file handler. Now they return 400. + + URLs like //foo/bar are parsed as authority='foo', path='/bar', + so they route normally (not empty path). + """ + + @parameterized.expand([ + # These parse to empty paths -> 400 + ('//lefty_rae', 400), + ('//test', 400), + ('//anyuser', 400), + ]) + def test_empty_path_returns_400(self, path, expected_status): + """URLs that parse to empty paths should return 400, not crash.""" + status = curl_status(f'{BASE_URL}{path}') + assert status == expected_status, \ + f'Expected {expected_status} for {path}, got {status}' + + @parameterized.expand([ + ('/jack', 200), + ('/about', 200), + ('/', 200), + ]) + def test_normal_paths_work(self, path, expected_status): + """Normal paths should still work.""" + status = curl_status(f'{BASE_URL}{path}') + assert status == expected_status, \ + f'Expected {expected_status} for {path}, got {status}' + + def test_server_survives_malformed_requests(self): + """Server should handle malformed requests without crashing.""" + # These all parse to empty paths + malformed_paths = ['//a', '//b', '//c', '//user', '//test'] + for path in malformed_paths: + status = curl_status(f'{BASE_URL}{path}') + assert status == 400, f'Expected 400 for {path}, got {status}' + + # Verify server is still responding after malformed requests + status = curl_status(f'{BASE_URL}/') + assert status == 200, 'Server should still be alive' diff --git a/tests/test_space.py b/tests/test_space.py new file mode 100644 index 0000000..d2ba672 --- /dev/null +++ b/tests/test_space.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: AGPL-3.0-only +"""Integration tests for Twitter Spaces support.""" +import pytest +from seleniumbase import BaseCase + + +class TestSpacePage(BaseCase): + """Tests for /i/spaces/@id route.""" + + SPACE_ID = "1mxPaaRAwYjKN" + SPACE_URL = f"http://localhost:8080/i/spaces/{SPACE_ID}" + + def test_space_page_loads(self): + """Space page should load with title.""" + self.open(self.SPACE_URL) + self.assert_element(".space-page") + self.assert_element(".space-panel") + self.assert_text_visible("INTEL WILL MOON NEXT WEEK", ".space-title") + + def test_space_host_info(self): + """Space should display host in participants.""" + self.open(self.SPACE_URL) + self.assert_element(".space-participants") + self.assert_text_visible("bubble boi") + self.assert_element(".host-badge") + + def test_space_metadata(self): + """Space should display listener count and state.""" + self.open(self.SPACE_URL) + self.assert_element(".space-meta") + # Should show listener count (number format) + meta_text = self.get_text(".space-meta") + assert any(c.isdigit() for c in meta_text), "Should show listener count" + # Should show ended state + assert "Ended" in meta_text or "Jun" in meta_text, "Should show ended state" + + def test_space_participants(self): + """Space should display host and speakers.""" + self.open(self.SPACE_URL) + self.assert_element(".space-participants") + # Host should have badge + self.assert_element(".host-badge") + self.assert_text_visible("Host", ".host-badge") + # Should show speakers + self.assert_text_visible("CANTELOPEPEEL") + self.assert_text_visible("Based Burner Account") + self.assert_text_visible("anon invests") + + def test_space_participant_avatars(self): + """Participant avatars should load correctly.""" + self.open(self.SPACE_URL) + # Check avatars in participants section + avatars = self.find_elements(".space-participant img") + assert len(avatars) >= 4, "Should have at least 4 participant avatars" + for avatar in avatars: + src = avatar.get_attribute("src") + # Should NOT be double-encoded + assert "%2Fpic%2F" not in src, f"Avatar URL double-encoded: {src}" + # Should have valid path + assert "/pic/" in src, f"Avatar URL missing /pic/: {src}" + + def test_space_player_hls_disabled(self): + """Without HLS, should show enable button with video-overlay style.""" + self.open(self.SPACE_URL) + self.assert_element(".space-player") + self.assert_element(".video-overlay") + # Should show duration in overlay-duration + self.assert_element(".overlay-duration") + # Should have enable button + source = self.get_page_source() + assert "Enable hls playback" in source + + def test_space_player_hls_enabled(self): + """With HLS enabled, should have audio element in DOM.""" + self.open(self.SPACE_URL) + # Set HLS preference via cookie + self.add_cookie({"name": "hlsPlayback", "value": "on"}) + self.refresh() + # Check page source for audio element (hidden until play clicked) + source = self.get_page_source() + assert '