mirrors/nitter
Watch
1
0
Fork
You've already forked nitter
 0
mirror of https://github.com/zedeus/nitter synced 2026-09-05 22:59:31 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'zedeus:master' into master

Browse source
This commit is contained in:
Salastil 2026-07-02 17:53:12 -04:00 • committed by GitHub
commit 106df59291
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
71 changed files with 3652 additions and 481 deletions
Download patch file Download diff file Expand all files Collapse all files

92
.github/workflows/build-docker.yml vendored
View file

@ -7,55 +7,105 @@ on:
branches:
- master
concurrency:
group: docker-publish-${{ github.ref }}
cancel-in-progress: true
env:
IMAGE: zedeus/nitter
jobs:
tests:
uses: ./.github/workflows/run-tests.yml
secrets: inherit
build-docker-amd64:
build:
needs: [tests]
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
include:
- runner: ubuntu-24.04
platform: linux/amd64
- runner: ubuntu-24.04-arm
platform: linux/arm64
runs-on: ${{ matrix.runner }}
steps:
- name: Prepare platform name
run: echo "PLATFORM_PAIR=${platform//\//-}" >> "$GITHUB_ENV"
env:
platform: ${{ matrix.platform }}
- uses: actions/checkout@v6
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
version: latest
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push AMD64 Docker image
uses: docker/build-push-action@v3
- name: Build and push by digest
id: build
uses: docker/build-push-action@v6
with:
context: .
file: ./Dockerfile
platforms: linux/amd64
push: true
tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}
platforms: ${{ matrix.platform }}
outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
provenance: false
sbom: false
build-docker-arm64:
needs: [tests]
runs-on: ubuntu-24.04-arm
- name: Export digest
run: |
mkdir -p "${{ runner.temp }}/digests"
digest="${{ steps.build.outputs.digest }}"
touch "${{ runner.temp }}/digests/${digest#sha256:}"
- name: Upload digest
uses: actions/upload-artifact@v4
with:
name: digests-${{ env.PLATFORM_PAIR }}
path: ${{ runner.temp }}/digests/*
if-no-files-found: error
retention-days: 1
# Combine the per-arch digests into one multi-arch manifest so that
# `docker pull zedeus/nitter:latest` serves the right image on any CPU.
merge:
needs: [build]
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v6
- name: Download digests
uses: actions/download-artifact@v4
with:
path: ${{ runner.temp }}/digests
pattern: digests-*
merge-multiple: true
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
version: latest
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push ARM64 Docker image
uses: docker/build-push-action@v3
with:
context: .
file: ./Dockerfile.arm64
platforms: linux/arm64
push: true
tags: zedeus/nitter:latest-arm64,zedeus/nitter:${{ github.sha }}-arm64
- name: Create manifest list and push
working-directory: ${{ runner.temp }}/digests
run: |
docker buildx imagetools create \
-t ${{ env.IMAGE }}:latest \
-t ${{ env.IMAGE }}:latest-arm64 \
-t ${{ env.IMAGE }}:${{ github.sha }} \
$(printf '${{ env.IMAGE }}@sha256:%s ' *)
- name: Inspect image
run: docker buildx imagetools inspect ${{ env.IMAGE }}:${{ github.sha }}

25
.github/workflows/run-tests.yml vendored
View file

@ -32,10 +32,12 @@ jobs:
id: cache-nimble
uses: actions/cache@v5
with:
path: ~/.nimble
key: ${{ matrix.nim }}-nimble-v2-${{ hashFiles('*.nimble') }}
path: |
~/.nimble/pkgcache
~/.nimble/packages_official.json
key: ${{ matrix.nim }}-nimble-v6-${{ hashFiles('*.nimble') }}
restore-keys: |
${{ matrix.nim }}-nimble-v2-
${{ matrix.nim }}-nimble-v6-
- name: Setup Nim
uses: jiro4989/setup-nim-action@v2
@ -103,10 +105,12 @@ jobs:
- name: Cache Nimble Dependencies
uses: actions/cache@v5
with:
path: ~/.nimble
key: 2.2.x-nimble-v2-${{ hashFiles('*.nimble') }}
path: |
~/.nimble/pkgcache
~/.nimble/packages_official.json
key: 2.2.x-nimble-v6-${{ hashFiles('*.nimble') }}
restore-keys: |
2.2.x-nimble-v2-
2.2.x-nimble-v6-
- name: Setup Nim
uses: jiro4989/setup-nim-action@v2
@ -115,6 +119,9 @@ jobs:
use-nightlies: true
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Install Nimble dependencies
run: nimble install -y --depsOnly
- name: Download 2.2.x build artifact
uses: actions/download-artifact@v4
with:
@ -130,10 +137,8 @@ jobs:
sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf
sed -i 's/maxRetries = 1/maxRetries = 10/g' nitter.conf
# Run both Nimble tasks concurrently
nim r tools/rendermd.nim &
nim r tools/gencss.nim &
wait
nim r tools/rendermd.nim
nim r tools/gencss.nim
echo '${{ secrets.SESSIONS }}' | head -n1
echo '${{ secrets.SESSIONS }}' > ./sessions.jsonl

3
.gitignore vendored
View file

@ -15,3 +15,6 @@ sessions.json*
dump.rdb
*.bak
/tools/*.json*
nimbledeps/
nimble.paths
nimble.develop

4
Dockerfile
View file

@ -1,4 +1,4 @@
FROM nimlang/nim:2.2.0-alpine-regular as nim
FROM nimlang/nim:2.2.6-alpine-regular as nim
LABEL maintainer="setenforce@protonmail.com"
RUN apk --no-cache add libsass-dev pcre
@ -15,7 +15,7 @@ RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
FROM alpine:latest
WORKDIR /src/
RUN apk --no-cache add pcre ca-certificates
RUN apk --no-cache add pcre ca-certificates openssl
COPY --from=nim /src/nitter/nitter ./
COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
COPY --from=nim /src/nitter/public ./public

25
Dockerfile.arm64
View file

@ -1,25 +0,0 @@
FROM alpine:3.20.6 as nim
LABEL maintainer="setenforce@protonmail.com"
RUN apk --no-cache add libsass-dev pcre gcc git libc-dev nim nimble
WORKDIR /src/nitter
COPY nitter.nimble .
RUN nimble install -y --depsOnly
COPY . .
RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
&& nimble scss \
&& nimble md
FROM alpine:3.20.6
WORKDIR /src/
RUN apk --no-cache add pcre ca-certificates openssl
COPY --from=nim /src/nitter/nitter ./
COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
COPY --from=nim /src/nitter/public ./public
EXPOSE 8080
RUN adduser -h /src/ -D -s /bin/sh nitter
USER nitter
CMD ./nitter

28
README.md
View file

@ -104,9 +104,9 @@ along with the scss and md files.
# su nitter
$ git clone https://github.com/zedeus/nitter
$ cd nitter
$ nimble build -d:danger --mm:refc
$ nimble scss
$ nimble md
$ nimble -l build -d:danger --mm:refc
$ nimble -l scss
$ nimble -l md
$ cp nitter.example.conf nitter.conf
```
@ -123,12 +123,23 @@ performance reasons.
Page for the Docker image: https://hub.docker.com/r/zedeus/nitter
#### NOTE: For ARM64 support, please use the separate ARM64 docker image: [`zedeus/nitter:latest-arm64`](https://hub.docker.com/r/zedeus/nitter/tags).
#### NOTE: The published image is multi-arch — `zedeus/nitter:latest` runs natively on both `amd64` and `arm64`.
To run Nitter with Docker, you'll need to install and run Redis separately
before you can run the container. See below for how to also run Redis using
Docker.
First create your config file. The Docker commands mount it into the container,
so it has to exist on the host beforehand. If you've cloned the repo:
```bash
cp nitter.example.conf nitter.conf
```
If you're using the prebuilt image without a local clone, download
[`nitter.example.conf`](https://raw.githubusercontent.com/zedeus/nitter/master/nitter.example.conf)
and save it as `nitter.conf` instead.
To build and run Nitter in Docker:
```bash
@ -136,8 +147,6 @@ docker build -t nitter:latest .
docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest
```
Note: For ARM64, use this Dockerfile: [`Dockerfile.arm64`](https://github.com/zedeus/nitter/blob/master/Dockerfile.arm64).
A prebuilt Docker image is provided as well:
```bash
@ -151,8 +160,11 @@ Change `redisHost` from `localhost` to `nitter-redis` in `nitter.conf`, then run
docker-compose up -d
```
Note the Docker commands expect a `nitter.conf` file in the directory you run
them.
Note the Docker commands mount `nitter.conf` (and `sessions.jsonl` for
docker-compose) from the directory you run them in. If a mounted file doesn't
exist, Docker silently creates a directory in its place and the container fails
with `not a directory: Are you trying to mount a directory onto a file`. Remove
that directory and create the file as shown above.
### systemd

2
docker-compose.yml → compose.yml
View file

@ -1,5 +1,3 @@
version: "3"
services:
nitter:

4
config.nims
View file

@ -11,3 +11,7 @@ warning("HoleEnumConv", off)
hint("XDeclaredButNotUsed", off)
hint("XCannotRaiseY", off)
hint("User", off)
# begin Nimble config (version 2)
when withDir(thisDir(), system.fileExists("nimble.paths")):
include "nimble.paths"
# end Nimble config

2
nitter.example.conf
View file

@ -20,7 +20,7 @@ redisMaxConnections = 30
# you receive tons of requests per second
[Config]
hmacKey = "secretkey" # random key for cryptographic signing of video urls
hmacKey = "secretkey" # CHANGE THIS to a unique random value (e.g. `openssl rand -hex 32`); signs media urls
base64Media = false # use base64 encoding for proxied media urls
enableRSS = true # master switch, set to false to disable all RSS feeds
enableRSSUserTweets = true # /@user/rss

23
nitter.nimble
View file

@ -11,19 +11,18 @@ bin = @["nitter"]
# Dependencies
requires "nim >= 2.0.0"
requires "jester#baca3f"
requires "karax#5cf360c"
requires "sass#7dfdd03"
requires "nimcrypto#a079df9"
requires "markdown#158efe3"
requires "jester == 0.6.0"
requires "karax == 1.5.0"
requires "sass == 0.2.0"
requires "nimcrypto == 0.7.3"
requires "markdown == 0.8.8"
requires "packedjson#9e6fbb6"
requires "supersnappy#6c94198"
requires "redpool#8b7c1db"
requires "https://github.com/zedeus/redis#d0a0e6f"
requires "zippy#ca5989a"
requires "flatty#e668085"
requires "jsony#1de1f08"
requires "oauth#b8c163b"
requires "supersnappy == 2.1.4"
requires "redpool == 0.2.2"
requires "zippy == 0.10.19"
requires "flatty == 0.4.0"
requires "jsony == 1.1.6"
requires "oauth == 0.11"
# Tasks

71
public/css/fontello.css vendored
View file

@ -1,12 +1,12 @@
@font-face {
font-family: "fontello";
src: url("/fonts/fontello.eot?49059696");
src: url("/fonts/fontello.eot?59696369");
src:
url("/fonts/fontello.eot?49059696#iefix") format("embedded-opentype"),
url("/fonts/fontello.woff2?49059696") format("woff2"),
url("/fonts/fontello.woff?49059696") format("woff"),
url("/fonts/fontello.ttf?49059696") format("truetype"),
url("/fonts/fontello.svg?49059696#fontello") format("svg");
url("/fonts/fontello.eot?59696369#iefix") format("embedded-opentype"),
url("/fonts/fontello.woff2?59696369") format("woff2"),
url("/fonts/fontello.woff?59696369") format("woff"),
url("/fonts/fontello.ttf?59696369") format("truetype"),
url("/fonts/fontello.svg?59696369#fontello") format("svg");
font-weight: normal;
font-style: normal;
}
@ -36,113 +36,118 @@
-moz-osx-font-smoothing: grayscale;
}
.icon-group:before {
content: "\e0c3";
}
/* '' */
.icon-views:before {
content: "\e800";
}
/* '' */
/* '' */
.icon-heart:before {
content: "\e801";
}
/* '' */
/* '' */
.icon-quote:before {
content: "\e802";
}
/* '' */
/* '' */
.icon-comment:before {
content: "\e803";
}
/* '' */
.icon-group:before {
content: "\e804";
}
/* '' */
/* '' */
.icon-play:before {
content: "\e805";
}
/* '' */
/* '' */
.icon-link:before {
content: "\e806";
}
/* '' */
/* '' */
.icon-calendar:before {
content: "\e807";
}
/* '' */
/* '' */
.icon-location:before {
content: "\e808";
}
/* '' */
/* '' */
.icon-picture:before {
content: "\e809";
}
/* '' */
/* '' */
.icon-lock:before {
content: "\e80a";
}
/* '' */
/* '' */
.icon-down:before {
content: "\e80b";
}
/* '' */
/* '' */
.icon-retweet:before {
content: "\e80c";
}
/* '' */
/* '' */
.icon-search:before {
content: "\e80d";
}
/* '' */
/* '' */
.icon-pin:before {
content: "\e80e";
}
/* '' */
/* '' */
.icon-cog:before {
content: "\e80f";
}
/* '' */
/* '' */
.icon-rss:before {
content: "\e810";
}
/* '' */
/* '' */
.icon-ok:before {
content: "\e811";
}
/* '' */
.icon-attention:before {
/* '' */
.icon-attention-circled:before {
content: "\e812";
}
/* '' */
/* '' */
.icon-download-alt:before {
content: "\e813";
}
/* '' */
.icon-circle:before {
content: "\f111";
}
/* '' */
/* '' */
.icon-info:before {
content: "\f128";
}
/* '' */
/* '' */
.icon-bird:before {
content: "\f309";
}
/* '' */
/* '' */

BIN
public/fonts/fontello.eot
View file

Binary file not shown.

6
public/fonts/fontello.svg
View file

@ -6,6 +6,8 @@
<font id="fontello" horiz-adv-x="1000" >
<font-face font-family="fontello" font-weight="400" font-stretch="normal" units-per-em="1000" ascent="850" descent="-150" />
<missing-glyph horiz-adv-x="1000" />
<glyph glyph-name="group" unicode="&#xe0c3;" d="M0 106l0 134q0 26 18 32l171 80q-66 39-68 131 0 56 35 103 37 41 90 43 31 0 63-19-49-125 23-237-12-11-25-19l-114-55q-48-23-52-84l0-143-114 0q-25 0-27 34z m193-59l0 168q0 27 22 37l152 70 57 28q-37 23-60 66t-22 94q0 76 46 130t110 54 109-54 45-130q0-105-78-158l61-30 146-70q24-10 24-37l0-168q-2-37-37-41l-541 0q-14 2-24 14t-10 27z m473 330q68 106 22 231 31 19 66 21 49 0 90-43 35-41 35-103 0-82-65-131l168-80q18-10 18-32l0-134q0-32-27-34l-118 0 0 143q0 57-50 84l-110 53q-15 8-29 25z" horiz-adv-x="1000" />
<glyph glyph-name="views" unicode="&#xe800;" d="M180 516l0-538-180 0 0 538 180 0z m250-138l0-400-180 0 0 400 180 0z m250 344l0-744-180 0 0 744 180 0z" horiz-adv-x="680" />
<glyph glyph-name="heart" unicode="&#xe801;" d="M790 644q70-64 70-156t-70-158l-360-330-360 330q-70 66-70 158t70 156q62 58 151 58t153-58l56-52 58 52q62 58 150 58t152-58z" horiz-adv-x="860" />
@ -14,8 +16,6 @@
<glyph glyph-name="comment" unicode="&#xe803;" d="M1000 350q0-97-67-179t-182-130-251-48q-39 0-81 4-110-97-257-135-27-8-63-12-10-1-17 5t-10 16v1q-2 2 0 6t1 6 2 5l4 5t4 5 4 5q4 5 17 19t20 22 17 22 18 28 15 33 15 42q-88 50-138 123t-51 157q0 73 40 139t106 114 160 76 194 28q136 0 251-48t182-130 67-179z" horiz-adv-x="1000" />
<glyph glyph-name="group" unicode="&#xe804;" d="M0 106l0 134q0 26 18 32l171 80q-66 39-68 131 0 56 35 103 37 41 90 43 31 0 63-19-49-125 23-237-12-11-25-19l-114-55q-48-23-52-84l0-143-114 0q-25 0-27 34z m193-59l0 168q0 27 22 37l152 70 57 28q-37 23-60 66t-22 94q0 76 46 130t110 54 109-54 45-130q0-105-78-158l61-30 146-70q24-10 24-37l0-168q-2-37-37-41l-541 0q-14 2-24 14t-10 27z m473 330q68 106 22 231 31 19 66 21 49 0 90-43 35-41 35-103 0-82-65-131l168-80q18-10 18-32l0-134q0-32-27-34l-118 0 0 143q0 57-50 84l-110 53q-15 8-29 25z" horiz-adv-x="1000" />
<glyph glyph-name="play" unicode="&#xe805;" d="M772 333l-741-412q-13-7-22-2t-9 20v822q0 14 9 20t22-2l741-412q13-7 13-17t-13-17z" horiz-adv-x="785.7" />
<glyph glyph-name="link" unicode="&#xe806;" d="M294 116q14 14 34 14t36-14q32-34 0-70l-42-40q-56-56-132-56-78 0-134 56t-56 132q0 78 56 134l148 148q70 68 144 77t128-43q16-16 16-36t-16-36q-36-32-70 0-50 48-132-34l-148-146q-26-26-26-64t26-62q26-26 63-26t63 26z m450 574q56-56 56-132 0-78-56-134l-158-158q-74-72-150-72-62 0-112 50-14 14-14 34t14 36q14 14 35 14t35-14q50-48 122 24l158 156q28 28 28 64 0 38-28 62-24 26-56 31t-60-21l-50-50q-16-14-36-14t-34 14q-34 34 0 70l50 50q54 54 127 51t129-61z" horiz-adv-x="800" />
@ -44,6 +44,8 @@
<glyph glyph-name="attention-circled" unicode="&#xe812;" d="M429 779q116 0 215-58t156-156 57-215-57-215-156-156-215-58-216 58-155 156-58 215 58 215 155 156 216 58z m71-696v106q0 8-5 13t-12 5h-107q-8 0-13-5t-6-13v-106q0-8 6-13t13-6h107q7 0 12 6t5 13z m-1 192l10 346q0 7-6 10-5 5-13 5h-123q-8 0-13-5-6-3-6-10l10-346q0-6 5-10t14-4h103q8 0 13 4t6 10z" horiz-adv-x="857.1" />
<glyph glyph-name="download-alt" unicode="&#xe813;" d="M0-150l0 135 1000 0 0-135-1000 0z m88 586l228 0 0 414 370 0 0-414 228 0-414-385z" horiz-adv-x="1000" />
<glyph glyph-name="circle" unicode="&#xf111;" d="M857 350q0-117-57-215t-156-156-215-58-216 58-155 156-58 215 58 215 155 156 216 58 215-58 156-156 57-215z" horiz-adv-x="857.1" />
<glyph glyph-name="info" unicode="&#xf128;" d="M393 149v-134q0-9-7-15t-15-7h-134q-9 0-16 7t-7 15v134q0 9 7 16t16 6h134q9 0 15-6t7-16z m176 335q0-30-8-56t-20-43-31-33-32-25-34-19q-23-13-38-37t-15-37q0-10-7-18t-16-9h-134q-8 0-14 11t-6 20v26q0 46 37 87t79 60q33 16 47 32t14 42q0 24-26 41t-60 18q-36 0-60-16-20-14-60-64-7-9-17-9-7 0-14 4l-91 70q-8 6-9 14t3 16q89 148 259 148 45 0 90-17t81-46 59-72 23-88z" horiz-adv-x="571.4" />

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 7.3 KiB

After

Width:  |  Height:  |  Size: 7.4 KiB

Before After
Before After

BIN
public/fonts/fontello.ttf
View file

Binary file not shown.

BIN
public/fonts/fontello.woff
View file

Binary file not shown.

BIN
public/fonts/fontello.woff2
View file

Binary file not shown.

27
public/js/hlsPlayback.js
View file

@ -1,27 +1,30 @@
// @license http://www.gnu.org/licenses/agpl-3.0.html AGPL-3.0
// SPDX-License-Identifier: AGPL-3.0-only
function playVideo(overlay) {
const video = overlay.parentElement.querySelector('video');
const url = video.getAttribute("data-url");
const startTime = parseFloat(video.getAttribute("data-start") || "0");
video.setAttribute("controls", "");
function playMedia(overlay, tagName) {
const media = overlay.parentElement.querySelector(tagName);
const url = media.getAttribute("data-url");
const startTime = parseFloat(media.getAttribute("data-start") || "0");
media.setAttribute("controls", "");
overlay.style.display = "none";
if (Hls.isSupported()) {
var hls = new Hls({autoStartLoad: false});
hls.loadSource(url);
hls.attachMedia(video);
hls.attachMedia(media);
hls.on(Hls.Events.MANIFEST_PARSED, function () {
hls.loadLevel = hls.levels.length - 1;
hls.startLoad(startTime);
video.play();
media.play();
});
} else if (video.canPlayType('application/vnd.apple.mpegurl')) {
video.src = url;
video.addEventListener('canplay', function() {
if (startTime > 0) video.currentTime = startTime;
video.play();
} else if (media.canPlayType('application/vnd.apple.mpegurl')) {
media.src = url;
media.addEventListener('canplay', function() {
if (startTime > 0) media.currentTime = startTime;
media.play();
});
}
}
function playVideo(overlay) { playMedia(overlay, 'video'); }
function playAudio(overlay) { playMedia(overlay, 'audio'); }
// @license-end

172
src/api.nim
View file

@ -2,7 +2,7 @@
import asyncdispatch, httpclient, strutils, sequtils, sugar
import packedjson
import types, query, formatters, consts, apiutils, parser, utils
import experimental/parser as newParser
import experimental/parser
# Helper to generate params object for GraphQL requests
proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
@ -11,13 +11,18 @@ proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
if fieldToggles.len > 0:
result.add ("fieldToggles", fieldToggles)
proc apiUrl(endpoint, variables: string; fieldToggles = ""): ApiUrl =
return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles))
proc apiUrl(endpoint, variables: string; fieldToggles = ""; skipTid = false): ApiUrl =
return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles), skipTid: skipTid)
proc apiReq(endpoint, variables: string; fieldToggles = ""): ApiReq =
let url = apiUrl(endpoint, variables, fieldToggles)
proc apiReq(endpoint, variables: string; fieldToggles = ""; skipTid = false): ApiReq =
let url = apiUrl(endpoint, variables, fieldToggles, skipTid)
return ApiReq(cookie: url, oauth: url)
proc cursorParam(after: string): string =
## JSON-escape the user-supplied cursor so it cannot break out of the GraphQL
## variables object (same input-validation class as the #1411 media SSRF).
if after.len > 0: "\"cursor\":" & $(%after) & "," else: ""
proc mediaUrl(id, cursor: string; count=20): ApiReq =
result = ApiReq(
cookie: apiUrl(graphUserMedia, userMediaVars % [id, cursor, $count]),
@ -25,33 +30,24 @@ proc mediaUrl(id, cursor: string; count=20): ApiReq =
)
proc userTweetsUrl(id: string; cursor: string): ApiReq =
result = ApiReq(
# cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"])
)
# might change this in the future pending testing
result.cookie = result.oauth
return apiReq(graphUserTweetsV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles)
proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq =
let cookieVars = userTweetsAndRepliesVars % [id, cursor]
result = ApiReq(
cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"])
)
return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles, skipTid=true)
proc tweetDetailUrl(id: string; cursor: string): ApiReq =
let cookieVars = tweetDetailVars % [id, cursor]
result = ApiReq(
return apiReq(graphTweet, tweetVars % [id, cursor])
# let cookieVars = tweetDetailVars % [id, cursor]
# result = ApiReq(
# cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
cookie: apiUrl(graphTweet, tweetVars % [id, cursor]),
oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
)
# oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
# )
proc userUrl(username: string): ApiReq =
let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
let cookieVars = $(%*{"screen_name": username, "withGrokTranslatedBio": false})
result = ApiReq(
cookie: apiUrl(graphUser, cookieVars, tweetDetailFieldToggles),
oauth: apiUrl(graphUserV2, """{"screen_name": "$1"}""" % username)
oauth: apiUrl(graphUserV2, $(%*{"screen_name": username}))
)
proc getGraphUser*(username: string): Future[User] {.async.} =
@ -69,7 +65,7 @@ proc getGraphUserById*(id: string): Future[User] {.async.} =
proc getAboutAccount*(username: string): Future[AccountInfo] {.async.} =
if username.len == 0: return
let
url = apiReq(graphAboutAccount, """{"screenName":"$1"}""" % username)
url = apiReq(graphAboutAccount, $(%*{"screenName": username}))
js = await fetch(url)
result = parseAboutAccount(js)
@ -80,7 +76,7 @@ proc restReq(endpoint: string; params: seq[(string, string)] = @[]): ApiReq =
proc getBroadcastInfo*(id: string): Future[Broadcast] {.async.} =
if id.len == 0: return
let
req = apiReq(graphBroadcast, """{"id":"$1"}""" % id)
req = apiReq(graphBroadcast, $(%*{"id": id}))
js = await fetch(req)
result = parseBroadcastInfo(js)
@ -92,10 +88,23 @@ proc fetchBroadcastStream*(mediaKey: string): Future[string] {.async.} =
result = streamJs{"source", "noRedirectPlaybackUrl"}.getStr(
streamJs{"source", "location"}.getStr)
proc getAudioSpace*(id: string): Future[AudioSpace] {.async.} =
if id.len == 0: return
let
variables = %*{
"id": id,
"isMetatagsQuery": false,
"withReplays": true,
"withListeners": true
}
req = apiReq(graphAudioSpace, $variables)
js = await fetch(req)
result = parseAudioSpace(js)
proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
cursor = cursorParam(after)
url = case kind
of TimelineKind.tweets: userTweetsUrl(id, cursor)
of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor)
@ -103,10 +112,61 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi
js = await fetch(url)
result = parseGraphTimeline(js, after)
proc getGraphCommunity*(id: string): Future[Community] {.async.} =
if id.len == 0: return
let
url = apiReq(graphCommunity, $(%*{"communityId": id}))
js = await fetch(url)
result = parseGraphCommunity(js)
proc getGraphCommunityTweets*(id: string; rankingMode: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = cursorParam(after)
url = apiReq(graphCommunityTweets, communityTweetsVars % [id, cursor, rankingMode])
js = await fetch(url)
result = parseGraphCommunityTimeline(js, after)
proc getGraphCommunityMedia*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = cursorParam(after)
url = apiReq(graphCommunityMedia, communityMediaVars % [id, cursor])
js = await fetch(url)
result = parseGraphCommunityTimeline(js, after)
proc communitySliceReq(endpoint, variables: string): ApiReq =
let url = ApiUrl(endpoint: endpoint, params: @[("variables", variables)])
ApiReq(cookie: url, oauth: url)
proc getGraphCommunityMembers*(id: string; after=""): Future[Result[User]] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: $(%after) else: "null"
url = communitySliceReq(graphCommunityMembers, communityMembersVars % [id, cursor])
js = await fetch(url)
result = parseGraphCommunityMembers(js, after)
proc getGraphCommunityModerators*(id: string): Future[Result[User]] {.async.} =
if id.len == 0: return
let
url = communitySliceReq(graphCommunityModerators, communityMembersVars % [id, "null"])
js = await fetch(url)
result = parseGraphCommunityMembers(js)
proc getGraphCommunityHashtags*(id, hashtag: string; after=""): Future[Timeline] {.async.} =
if id.len == 0 or hashtag.len == 0: return
let
safeTag = multiReplace(hashtag, ("\"", ""), ("\\", ""))
cursor = cursorParam(after)
url = apiReq(graphCommunityHashtags, communityHashtagsVars % [id, cursor, safeTag])
js = await fetch(url)
result = parseGraphCommunityTimeline(js, after)
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
cursor = cursorParam(after)
url = apiReq(graphListTweets, restIdVars % [id, cursor, "20"])
js = await fetch(url)
result = parseGraphTimeline(js, after).tweets
@ -120,7 +180,7 @@ proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
proc getGraphList*(id: string): Future[List] {.async.} =
let
url = apiReq(graphListById, """{"listId": "$1"}""" % id)
url = apiReq(graphListById, $(%*{"listId": id}))
js = await fetch(url)
result = parseGraphList(js)
@ -141,17 +201,39 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
js = await fetchRaw(url)
result = parseGraphListMembers(js, after)
proc getGraphUserConnections(userId: string; endpoint: string; kind: QueryKind;
after=""): Future[Result[User]] {.async.} =
if userId.len == 0: return
var variables = %*{
"userId": userId,
"count": 20,
"includePromotedContent": false,
"withGrokTranslatedBio": true
}
if after.len > 0:
variables["cursor"] = %after
let
url = apiReq(endpoint, $variables)
js = await fetchRaw(url)
result = parseGraphFollowers(js, after, kind)
proc getGraphFollowers*(userId: string; after=""): Future[Result[User]] {.async.} =
result = await getGraphUserConnections(userId, graphFollowers, followers, after)
proc getGraphFollowing*(userId: string; after=""): Future[Result[User]] {.async.} =
result = await getGraphUserConnections(userId, graphFollowing, following, after)
proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
if id.len == 0: return
let
url = apiReq(graphTweetResult, """{"rest_id": "$1"}""" % id)
url = apiReq(graphTweetResult, $(%*{"rest_id": id}))
js = await fetch(url)
result = parseGraphTweetResult(js)
proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
cursor = cursorParam(after)
js = await fetch(tweetDetailUrl(id, cursor))
result = parseGraphConversation(js, id)
@ -184,13 +266,13 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
var
variables = %*{
"rawQuery": q,
"query_source": "typedQuery",
"count": 20,
"querySource": "typed_query",
"product": "Latest",
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false
"withGrokTranslatedBio":true,
"withQuickPromoteEligibilityTweetFields":false
}
if after.len > 0 and maxId.len == 0:
variables["cursor"] = % after
let
@ -212,12 +294,11 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
var
variables = %*{
"rawQuery": query.text,
"query_source": "typedQuery",
"count": 20,
"querySource": "typed_query",
"product": "People",
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false
"withGrokTranslatedBio":true,
"withQuickPromoteEligibilityTweetFields":false
}
if after.len > 0:
variables["cursor"] = % after
@ -234,6 +315,21 @@ proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
let js = await fetch(mediaUrl(id, "", 30))
result = parseGraphPhotoRail(js)
proc getGraphArticle*(id: string): Future[Article] {.async.} =
if id.len == 0: return
let
url = apiReq(graphTweetResultByRestId, articleVars % id, articleFieldToggles)
json = await fetchRaw(url)
result = parseGraphArticle(json)
proc getGraphTweetResults*(ids: seq[string]): Future[seq[Tweet]] {.async.} =
if ids.len == 0: return
let
idsJson = "[" & ids.mapIt("\"" & it & "\"").join(",") & "]"
url = apiReq(graphTweetResultsByRestIds, articleBatchVars % idsJson, articleFieldToggles)
js = await fetch(url)
result = parseGraphTweetResults(js)
proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
let client = newAsyncHttpClient(maxRedirects=0)
try:

54
src/apiutils.nim
View file

@ -1,6 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-only
import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
import jsony, packedjson, zippy, oauth1
import jsony, packedjson, zippy, oauth/oauth1
import types, auth, consts, parserutils, http_pool, tid
import experimental/types/common
@ -8,6 +8,7 @@ const
rlRemaining = "x-rate-limit-remaining"
rlReset = "x-rate-limit-reset"
rlLimit = "x-rate-limit-limit"
npCache = "x-np-cache"
errorsToSkip = {null, doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
var
@ -63,7 +64,7 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
proc getCookieHeader(authToken, ct0: string): string =
"auth_token=" & authToken & "; ct0=" & ct0
proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
proc genHeaders*(session: Session, url: Uri, skipTid: bool): Future[HttpHeaders] {.async.} =
result = newHttpHeaders({
"accept": "*/*",
"accept-encoding": "gzip",
@ -84,13 +85,14 @@ proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
result["x-twitter-auth-type"] = "OAuth2Session"
result["x-csrf-token"] = session.ct0
result["cookie"] = getCookieHeader(session.authToken, session.ct0)
result["referer"] = "https://x.com/"
result["sec-ch-ua"] = """"Google Chrome";v="142", "Chromium";v="142", "Not A(Brand";v="24""""
result["sec-ch-ua-mobile"] = "?0"
result["sec-ch-ua-platform"] = "Windows"
result["sec-fetch-dest"] = "empty"
result["sec-fetch-mode"] = "cors"
result["sec-fetch-site"] = "same-site"
if disableTid or "/1.1/" in url.path:
result["sec-fetch-site"] = "same-origin"
if disableTid or skipTid or "/1.1/" in url.path:
result["authorization"] = bearerToken2
else:
result["authorization"] = bearerToken
@ -114,7 +116,12 @@ template fetchImpl(result, fetchBody) {.dirty.} =
try:
var resp: AsyncResponse
pool.use(await genHeaders(session, url)):
let skipTid = case session.kind
of oauth: req.oauth.skipTid
of cookie: req.cookie.skipTid
let headers = await genHeaders(session, url, skipTid)
pool.use(headers):
template getContent =
# TODO: this is a temporary simple implementation
if apiProxy.len > 0 and "/1.1/" notin url.path:
@ -130,10 +137,11 @@ template fetchImpl(result, fetchBody) {.dirty.} =
raise newException(BadClientError, "Bad client")
if resp.status == $Http404 and result.len == 0:
echo "[sessions] transient 404 (empty body), retrying: ", url.path
echo "[sessions] transient 404 (empty body), retrying: ", url.path, ", session: ", session.pretty
raise rateLimitError()
if resp.headers.hasKey(rlRemaining):
let cacheStatus = resp.headers.getOrDefault(npCache)
if cacheStatus notin ["HIT", "STALE"] and resp.headers.hasKey(rlRemaining):
let
remaining = parseInt(resp.headers[rlRemaining])
reset = parseInt(resp.headers[rlReset])
@ -147,7 +155,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
if result.startsWith("{\"errors"):
let errors = result.fromJson(Errors)
if errors notin errorsToSkip:
echo "Fetch error, API: ", url.path, ", errors: ", errors
echo "Fetch error, API: ", url.path, ", errors: ", errors, ", session: ", session.pretty
if errors in {expiredToken, badToken, locked}:
invalidate(session)
raise rateLimitError()
@ -162,7 +170,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
fetchBody
if resp.status == $Http400:
echo "ERROR 400, ", url.path, ": ", result
echo "ERROR 400, ", url.path, ": ", result, ", session: ", session.pretty
raise newException(InternalError, $url)
except InternalError as e:
raise e
@ -177,21 +185,33 @@ template fetchImpl(result, fetchBody) {.dirty.} =
finally:
release(session)
template retry(bod) =
template retry(bod) {.dirty.} =
var session: Session
var retrySuccess = false
for i in 0 ..< maxRetries:
try:
session = nil
bod
retrySuccess = true
break
except RateLimitError:
echo "[sessions] Rate limited, retrying ", req.cookie.endpoint,
let api = if session.isNil: req.cookie.endpoint
else: req.endpoint(session)
if session.isNil:
echo "[sessions] Rate limited, retrying ", api,
" request (", i, "/", maxRetries, ")..."
else:
echo "[sessions] Rate limited, retrying ", api,
" request (", i, "/", maxRetries, ")..., session: ", session.pretty
session = nil
if retryDelayMs > 0:
await sleepAsync(retryDelayMs)
if not retrySuccess:
raise rateLimitError()
proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
retry:
var
body: string
var body: string
session = await getAndValidateSession(req)
let url = req.toUrl(session.kind)
@ -200,22 +220,22 @@ proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
if body.startsWith('{') or body.startsWith('['):
result = parseJson(body)
else:
echo resp.status, ": ", body, " --- url: ", url
echo resp.status, ": ", body, " --- url: ", url, ", session: ", session.pretty
result = newJNull()
let error = result.getError
if error != null and error notin errorsToSkip:
echo "Fetch error, API: ", url.path, ", error: ", error
echo "Fetch error, API: ", url.path, ", error: ", error, ", session: ", session.pretty
if error in {expiredToken, badToken, locked}:
invalidate(session)
raise rateLimitError()
proc fetchRaw*(req: ApiReq): Future[string] {.async.} =
retry:
var session = await getAndValidateSession(req)
session = await getAndValidateSession(req)
let url = req.toUrl(session.kind)
fetchImpl result:
if not (result.startsWith('{') or result.startsWith('[')):
echo resp.status, ": ", result, " --- url: ", url
echo resp.status, ": ", result, " --- url: ", url, ", session: ", session.pretty
result.setLen(0)

17
src/auth.nim
View file

@ -18,7 +18,7 @@ proc setMaxConcurrentReqs*(reqs: int) =
template log(str: varargs[string, `$`]) =
echo "[sessions] ", str.join("")
proc endpoint(req: ApiReq; session: Session): string =
proc endpoint*(req: ApiReq; session: Session): string =
case session.kind
of oauth: req.oauth.endpoint
of cookie: req.cookie.endpoint
@ -50,6 +50,8 @@ proc getSessionPoolHealth*(): JsonNode =
oldest = now.int64
newest = 0'i64
average = 0'i64
oauthTotal, cookieTotal = 0
oauthLimited, cookieLimited = 0
for session in sessionPool:
let created = snowflakeToEpoch(session.id)
@ -59,8 +61,15 @@ proc getSessionPoolHealth*(): JsonNode =
oldest = created
average += created
case session.kind
of oauth: inc oauthTotal
of cookie: inc cookieTotal
if session.limited:
limited.incl session.id
case session.kind
of oauth: inc oauthLimited
of cookie: inc cookieLimited
for api in session.apis.keys:
let
@ -84,6 +93,8 @@ proc getSessionPoolHealth*(): JsonNode =
"sessions": %*{
"total": sessionPool.len,
"limited": limited.card,
"oauth": %*{"total": oauthTotal, "limited": oauthLimited},
"cookie": %*{"total": cookieTotal, "limited": cookieLimited},
"oldest": $fromUnix(oldest),
"newest": $fromUnix(newest),
"average": $fromUnix(average)
@ -100,6 +111,7 @@ proc getSessionPoolDebug*(): JsonNode =
for session in sessionPool:
let sessionJson = %*{
"kind": $session.kind,
"apis": newJObject(),
"pending": session.pending,
}
@ -173,7 +185,10 @@ proc getSession*(req: ApiReq): Future[Session] {.async.} =
if not result.isNil and result.isReady(req):
inc result.pending
else:
if result.isNil:
log "no sessions available for API: ", req.cookie.endpoint
else:
log "no sessions available for API: ", req.endpoint(result), ", last tried: ", result.pretty
raise noSessionsError()
proc setLimited*(session: Session; req: ApiReq) =

209
src/consts.nim
View file

@ -7,109 +7,86 @@ const
bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
bearerToken2* = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery"
graphUserById* = "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery"
graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2"
graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2"
graphUserTweets* = "oRJs8SLCRNRbQzuZG93_oA/UserTweets"
graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
graphUserMedia* = "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
graphTweet* = "b4pV7sWOe97RncwHcGESUA/ConversationTimeline"
graphTweetDetail* = "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail"
graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
graphTweetEditHistory* = "upS9teTSG45aljmP9oTuXA/TweetEditHistory"
graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
graphAboutAccount* = "zs_jFPFT78rBpXv9Z3U2YQ/AboutAccountQuery"
graphUser* = "IGgvgiOx4QZndDHuD3x9TQ/UserByScreenName"
graphUserV2* = "-ZzAG_Bckx16LMbEvHC3lg/UserResultByScreenNameQuery"
graphUserById* = "-DAaa9jPxPswYeI2fZ9rug/UserResultByIdQuery"
graphUserTweetsV2* = "LE3eTyeqhBh2g-fX85O2eQ/UserWithProfileTweetsQueryV2"
graphUserTweetsAndRepliesV2* = "AcYHjc_YAx-9_rKWdMsKvA/UserWithProfileTweetsAndRepliesQueryV2"
graphUserTweets* = "PNd0vlufvrcIwrAnBYKE9g/UserTweets"
graphUserTweetsAndReplies* = "EqtpEwt0CoQXmDfq5DKH0A/UserTweetsAndReplies"
graphUserMedia* = "g_rGPF0fLON-M9cyVjXuzA/UserMedia"
graphUserMediaV2* = "WK111rbR0vM0ZX4lyZCYjw/MediaTimelineV2"
graphTweet* = "OZMbEnEa96AN8Pq6HyTWdw/ConversationTimeline"
graphTweetDetail* = "6uCvnic3m5reVuehkvHa3w/TweetDetail"
graphTweetResult* = "xYOrBQoTlfKJJPsX76MZEw/TweetResultByIdQuery"
graphTweetEditHistory* = "MGElmrYILE8wUfI8GorUYA/TweetEditHistory"
graphSearchTimeline* = "-TFXKoMnMTKdEXcCn-eahw/SearchTimeline"
graphBroadcast* = "0nMmbMh-_JwwRRFNXkyH3Q/BroadcastQuery"
graphListById* = "t9AbdyHaJVfjL9jsODwgpQ/ListByRestId"
graphListBySlug* = "LDQpQ89B5ipR8izCKrWU0g/ListBySlug"
graphListMembers* = "EM7YRaM3gCnzDESmchA7RA/ListMembers"
graphListTweets* = "0QJtcuMzVywHGAWD6Dtjlw/ListTimeline"
graphAboutAccount* = "zUnx-DLN9dkwOkNhTLySjg/AboutAccountQuery"
graphCommunity* = "-ElI1vg3dYbttVMhBhGdLw/CommunityQuery"
graphCommunityTweets* = "Mvs5UOOEkpXVMDZtUcxR-Q/CommunityTweetsTimeline"
graphCommunityMedia* = "Bt9XYnY7D3OcmZE5lhdx-A/CommunityMediaTimeline"
graphCommunityMembers* = "WSbJGJjZaVasSj9bnqSZSA/membersSliceTimeline_Query"
graphCommunityModerators* = "GBMT3GOWy5dYsYC4XJfvow/moderatorsSliceTimeline_Query"
graphCommunityHashtags* = "40DyrMxfCknGuZwE-keW_Q/CommunityHashtagsTimeline"
graphTweetResultByRestId* = "qtXMy1p5Y62uCskc_NUPJw/TweetResultByRestId"
graphTweetResultsByRestIds* = "Sc9EUQTZNEH-wzegn-nHvQ/TweetResultsByRestIds"
graphBroadcast* = "FJLCzpXCLPM1jUZqmM7oEA/BroadcastQuery"
graphAudioSpace* = "rWRLsOhNJ2xjpI1tREYurQ/AudioSpaceById"
restLiveStream* = "1.1/live_video_stream/status/"
graphFollowers* = "9jsVJ9l2uXUIKslHvJqIhw/Followers"
graphFollowing* = "OLm4oHZBfqWx8jbcEhWoFw/Following"
gqlFeatures* = """{
"android_ad_formats_media_component_render_overlay_enabled": false,
"android_graphql_skip_api_media_color_palette": false,
"android_professional_link_spotlight_display_enabled": false,
"articles_api_enabled": false,
"articles_preview_enabled": true,
"blue_business_profile_image_shape_enabled": false,
"c9s_tweet_anatomy_moderator_badge_enabled": true,
"commerce_android_shop_module_enabled": false,
"communities_web_enable_tweet_community_results_fetch": true,
"creator_subscriptions_quote_tweet_preview_enabled": false,
"creator_subscriptions_subscription_count_enabled": false,
"creator_subscriptions_tweet_preview_api_enabled": true,
"freedom_of_speech_not_reach_fetch_enabled": true,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
"grok_android_analyze_trend_fetch_enabled": false,
"grok_translations_community_note_auto_translation_is_enabled": false,
"grok_translations_community_note_translation_is_enabled": false,
"grok_translations_post_auto_translation_is_enabled": false,
"grok_translations_timeline_user_bio_auto_translation_is_enabled": false,
"hidden_profile_likes_enabled": false,
"highlights_tweets_tab_ui_enabled": false,
"immersive_video_status_linkable_timestamps": false,
"interactive_text_enabled": false,
"longform_notetweets_consumption_enabled": true,
"longform_notetweets_inline_media_enabled": true,
"longform_notetweets_richtext_consumption_enabled": true,
"longform_notetweets_rich_text_read_enabled": true,
"mobile_app_spotlight_module_enabled": false,
"payments_enabled": false,
"post_ctas_fetch_enabled": true,
"premium_content_api_read_enabled": false,
"rweb_video_screen_enabled": false,
"rweb_cashtags_enabled": true,
"profile_label_improvements_pcf_label_in_post_enabled": true,
"profile_label_improvements_pcf_label_in_profile_enabled": false,
"responsive_web_edit_tweet_api_enabled": true,
"responsive_web_enhance_cards_enabled": false,
"responsive_web_graphql_exclude_directive_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"responsive_web_profile_redirect_enabled": false,
"rweb_tipjar_consumption_enabled": false,
"verified_phone_label_enabled": false,
"creator_subscriptions_tweet_preview_api_enabled": true,
"responsive_web_graphql_timeline_navigation_enabled": true,
"responsive_web_grok_analysis_button_from_backend": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"premium_content_api_read_enabled": false,
"communities_web_enable_tweet_community_results_fetch": true,
"c9s_tweet_anatomy_moderator_badge_enabled": true,
"c9s_list_members_action_api_enabled": false,
"c9s_superc9s_indication_enabled": false,
"responsive_web_grok_analyze_button_fetch_trends_enabled": false,
"responsive_web_grok_analyze_post_followups_enabled": true,
"rweb_cashtags_composer_attachment_enabled": true,
"responsive_web_jetfuel_frame": true,
"responsive_web_grok_share_attachment_enabled": true,
"responsive_web_grok_annotations_enabled": true,
"responsive_web_grok_community_note_auto_translation_is_enabled": false,
"articles_preview_enabled": true,
"responsive_web_edit_tweet_api_enabled": true,
"rweb_conversational_replies_downvote_enabled": false,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
"view_counts_everywhere_api_enabled": true,
"longform_notetweets_consumption_enabled": true,
"responsive_web_twitter_article_tweet_consumption_enabled": true,
"content_disclosure_indicator_enabled": true,
"content_disclosure_ai_generated_indicator_enabled": true,
"responsive_web_grok_show_grok_translated_post": true,
"responsive_web_grok_analysis_button_from_backend": true,
"post_ctas_fetch_enabled": true,
"freedom_of_speech_not_reach_fetch_enabled": true,
"standardized_nudges_misinfo": true,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
"longform_notetweets_rich_text_read_enabled": true,
"longform_notetweets_inline_media_enabled": false,
"responsive_web_grok_image_annotation_enabled": true,
"responsive_web_grok_imagine_annotation_enabled": true,
"responsive_web_grok_share_attachment_enabled": true,
"responsive_web_grok_show_grok_translated_post": false,
"responsive_web_jetfuel_frame": true,
"responsive_web_media_download_video_enabled": false,
"responsive_web_profile_redirect_enabled": false,
"responsive_web_text_conversations_enabled": false,
"responsive_web_twitter_article_notes_tab_enabled": false,
"responsive_web_twitter_article_tweet_consumption_enabled": true,
"responsive_web_twitter_blue_verified_badge_is_enabled": true,
"rweb_lists_timeline_redesign_enabled": true,
"rweb_tipjar_consumption_enabled": true,
"rweb_video_screen_enabled": false,
"rweb_video_timestamps_enabled": false,
"spaces_2022_h2_clipping": true,
"spaces_2022_h2_spaces_communities": true,
"standardized_nudges_misinfo": true,
"subscriptions_feature_can_gift_premium": false,
"subscriptions_verification_info_enabled": true,
"subscriptions_verification_info_is_identity_verified_enabled": false,
"subscriptions_verification_info_reason_enabled": true,
"subscriptions_verification_info_verified_since_enabled": true,
"super_follow_badge_privacy_enabled": false,
"super_follow_exclusive_tweet_notifications_enabled": false,
"super_follow_tweet_api_enabled": false,
"super_follow_user_api_enabled": false,
"tweet_awards_web_tipping_enabled": false,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
"tweetypie_unmention_optimization_enabled": false,
"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
"unified_cards_destination_url_params_enabled": false,
"verified_phone_label_enabled": false,
"vibe_api_enabled": false,
"view_counts_everywhere_api_enabled": true,
"hidden_profile_subscriptions_enabled": false
"responsive_web_grok_community_note_auto_translation_is_enabled": true,
"responsive_web_enhance_cards_enabled": false
}""".replace(" ", "").replace("\n", "")
tweetVars* = """{
@ -143,7 +120,7 @@ const
restIdVars* = """{
"rest_id": "$1", $2
"count": $3
}"""
}""".replace(" ", "").replace("\n", "")
userMediaVars* = """{
"userId": "$1", $2
@ -170,6 +147,50 @@ const
"withVoice": true
}""".replace(" ", "").replace("\n", "")
articleVars* = """{
"tweetId": "$1",
"includePromotedContent": false,
"withBirdwatchNotes": true,
"withVoice": true,
"withCommunity": true
}""".replace(" ", "").replace("\n", "")
articleBatchVars* = """{
"tweetIds": $1,
"includePromotedContent": false,
"withBirdwatchNotes": true,
"withVoice": true,
"withCommunity": true
}""".replace(" ", "").replace("\n", "")
articleFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withArticleSummaryText":true,"withArticleVoiceOver":true}"""
communityTweetsVars* = """{
"communityId": "$1", $2
"count": 20,
"displayLocation": "Community",
"rankingMode": "$3",
"withCommunity": true
}""".replace(" ", "").replace("\n", "")
communityMediaVars* = """{
"communityId": "$1", $2
"count": 20,
"withCommunity": true
}""".replace(" ", "").replace("\n", "")
communityMembersVars* = """{
"communityId": "$1",
"cursor": $2
}""".replace(" ", "").replace("\n", "")
communityHashtagsVars* = """{
"communityId": "$1", $2
"count": 20,
"hashtags": ["$3"],
"withCommunity": true
}""".replace(" ", "").replace("\n", "")
userFieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}"""
userTweetsFieldToggles* = """{"withArticlePlainText":false}"""
userTweetsFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false}"""
tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}"""

4
src/experimental/parser.nim
View file

@ -1,2 +1,2 @@
import parser/[user, graphql]
export user, graphql
import parser/[user, graphql, article]
export user, graphql, article

88
src/experimental/parser/article.nim Normal file
View file

@ -0,0 +1,88 @@
# SPDX-License-Identifier: AGPL-3.0-only
import std/[strutils, tables, times, options]
import jsony
import utils, graphql, ../types/article
from ../../types import Article, ArticleParagraph, ArticleEntity, ArticleMedia,
User, TweetStats
proc parseGraphArticle*(json: string): Article =
if json.len == 0 or json[0] != '{':
return
var raw: GraphArticle
try:
raw = json.fromJson(GraphArticle)
except CatchableError:
return
let
tweet = raw.data.tweetResult.result
article = tweet.article.articleResults.result
if article.title.len == 0:
return
let publishedAt = article.metadata.firstPublishedAtSecs
var articleTime: DateTime
if publishedAt > 0:
articleTime = publishedAt.int64.fromUnix.utc
elif tweet.legacy.createdAt.len > 0:
articleTime = parseTwitterDate(tweet.legacy.createdAt)
result = Article(
title: article.title,
coverImage: getImageUrl(article.coverMedia.mediaInfo.originalImgUrl),
time: articleTime,
user: parseUserResult(tweet.core.userResults.result),
)
result.stats = TweetStats(
replies: tweet.legacy.replyCount,
retweets: tweet.legacy.retweetCount,
likes: tweet.legacy.favoriteCount,
)
if tweet.views.count.len > 0:
try: result.stats.views = parseInt(tweet.views.count)
except ValueError: discard
for blk in article.contentState.blocks:
result.paragraphs.add ArticleParagraph(
text: blk.text,
kind: blk.blockKind,
inlineStyles: blk.inlineStyleRanges,
entityRanges: blk.entityRanges,
)
for entry in article.contentState.entityMap:
let key = try: parseInt(entry.key) except ValueError: continue
var entity = ArticleEntity(kind: entry.value.entityKind)
case entity.kind
of "LINK": entity.url = entry.value.data.url
of "MEDIA":
for mi in entry.value.data.mediaItems:
entity.mediaIds.add mi.mediaId
entity.caption = entry.value.data.caption
of "TWEET": entity.tweetId = entry.value.data.tweetId
of "MARKDOWN": entity.markdown = entry.value.data.markdown
else: discard
result.entities[key] = entity
for me in article.mediaEntities:
let typeName = me.mediaInfo.typeName
var media = ArticleMedia(kind: typeName)
if me.mediaInfo.videoInfo.isSome:
let variants = me.mediaInfo.videoInfo.get.variants
case typeName
of "ApiGif":
if variants.len > 0:
media.url = variants[0].url
of "ApiVideo":
var bestBitrate = -1
for v in variants:
if v.bitrate > bestBitrate:
bestBitrate = v.bitrate
media.url = v.url
else: discard
elif typeName == "ApiImage":
media.url = getImageUrl(me.mediaInfo.originalImgUrl)
result.media[me.mediaId] = media

24
src/experimental/parser/graphql.nim
View file

@ -1,6 +1,6 @@
import options, strutils
import jsony
import user, utils, ../types/[graphuser, graphlistmembers]
import user, utils, ../types/[graphuser, graphlistmembers, graphfollowers]
from ../../types import User, VerifiedType, Result, Query, QueryKind
proc parseUserResult*(userResult: UserResult): User =
@ -67,3 +67,25 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] =
of TimelineTimelineCursor:
if entry.content.cursorType == "Bottom":
result.bottom = entry.content.value
proc parseGraphFollowers*(json, cursor: string; kind: QueryKind): Result[User] =
result = Result[User](
beginning: cursor.len == 0,
query: Query(kind: kind)
)
if json.len == 0 or json[0] != '{':
return
let raw = json.fromJson(GraphFollowers)
for instruction in raw.data.user.result.timeline.timeline.instructions:
if instruction.kind == "TimelineAddEntries":
for entry in instruction.entries:
case entry.content.entryType
of TimelineTimelineItem:
let userResult = entry.content.itemContent.userResults.result
if userResult.restId.len > 0:
result.content.add parseUserResult(userResult)
of TimelineTimelineCursor:
if entry.content.cursorType == "Bottom":
result.bottom = entry.content.value

79
src/experimental/types/article.nim Normal file
View file

@ -0,0 +1,79 @@
import std/options
import graphuser
from ../../types import ArticleStyle, ArticleEntityRange
type
GraphArticle* = object
data*: tuple[tweetResult: tuple[result: TweetResultNode]]
TweetResultNode* = object
article*: tuple[articleResults: tuple[result: ArticleResultNode]]
legacy*: TweetLegacy
core*: tuple[userResults: UserData]
views*: tuple[count: string]
TweetLegacy* = object
createdAt*: string
replyCount*: int
retweetCount*: int
favoriteCount*: int
ArticleResultNode* = object
title*: string
coverMedia*: tuple[mediaInfo: MediaInfoNode]
contentState*: ContentState
metadata*: tuple[firstPublishedAtSecs: int]
mediaEntities*: seq[RawMediaEntity]
ContentState* = object
blocks*: seq[ContentBlock]
entityMap*: seq[EntityMapEntry]
ContentBlock* = object
text*: string
blockKind*: string
inlineStyleRanges*: seq[ArticleStyle]
entityRanges*: seq[ArticleEntityRange]
EntityMapEntry* = object
key*: string
value*: EntityMapValue
EntityMapValue* = object
entityKind*: string
data*: EntityDataNode
EntityDataNode* = object
url*: string
mediaItems*: seq[tuple[mediaId: string]]
tweetId*: string
markdown*: string
caption*: string
RawMediaEntity* = object
mediaId*: string
mediaInfo*: MediaInfoNode
MediaInfoNode* = object
typeName*: string
originalImgUrl*: string
videoInfo*: Option[VideoInfoNode]
VideoInfoNode* = object
variants*: seq[VideoVariant]
VideoVariant* = object
url*: string
bitrate*: int
proc renameHook*(v: var ContentBlock; fieldName: var string) =
if fieldName == "type":
fieldName = "blockKind"
proc renameHook*(v: var EntityMapValue; fieldName: var string) =
if fieldName == "type":
fieldName = "entityKind"
proc renameHook*(v: var MediaInfoNode; fieldName: var string) =
if fieldName == "__typename":
fieldName = "typeName"

17
src/experimental/types/graphfollowers.nim Normal file
View file

@ -0,0 +1,17 @@
# SPDX-License-Identifier: AGPL-3.0-only
import graphlistmembers
type
GraphFollowers* = object
data*: tuple[user: UserWrapper]
UserWrapper = object
result*: UserResultWrapper
UserResultWrapper = object
timeline*: tuple[timeline: graphlistmembers.Timeline]
# Hook to normalize snake_case field from API to camelCase used by shared types
proc renameHook*(v: var Content; fieldName: var string) =
if fieldName == "user_results":
fieldName = "userResults"

6
src/experimental/types/graphlistmembers.nim
View file

@ -7,10 +7,10 @@ type
List = object
membersTimeline*: tuple[timeline: Timeline]
Timeline = object
Timeline* = object
instructions*: seq[Instruction]
Instruction = object
Instruction* = object
kind*: string
entries*: seq[tuple[content: Content]]
@ -18,7 +18,7 @@ type
TimelineTimelineItem
TimelineTimelineCursor
Content = object
Content* = object
case entryType*: ContentEntryType
of TimelineTimelineItem:
itemContent*: tuple[userResults: UserData]

18
src/formatters.nim
View file

@ -140,25 +140,30 @@ proc pageDesc*(user: User): string =
"The latest tweets from " & user.fullname
proc getJoinDate*(user: User): string =
if user.joinDate.year == 0: return ""
user.joinDate.format("'Joined' MMMM YYYY")
proc getJoinDateFull*(user: User): string =
if user.joinDate.year == 0: return ""
user.joinDate.format("h:mm tt - d MMM YYYY")
proc getTime*(tweet: Tweet): string =
if tweet.time.year == 0: return ""
tweet.time.format("MMM d', 'YYYY' · 'h:mm tt' UTC'")
proc getRfc822Time*(tweet: Tweet): string =
if tweet.time.year == 0: return ""
tweet.time.format("ddd', 'dd MMM yyyy HH:mm:ss 'GMT'")
proc getShortTime*(tweet: Tweet): string =
proc getShortTime*(time: DateTime): string =
if time.year == 0: return ""
let now = now()
let since = now - tweet.time
let since = now - time
if now.year != tweet.time.year:
result = tweet.time.format("d MMM yyyy")
if now.year != time.year:
result = time.format("d MMM yyyy")
elif since.inDays >= 1:
result = tweet.time.format("MMM d")
result = time.format("MMM d")
elif since.inHours >= 1:
result = $since.inHours & "h"
elif since.inMinutes >= 1:
@ -168,6 +173,9 @@ proc getShortTime*(tweet: Tweet): string =
else:
result = "now"
proc getShortTime*(tweet: Tweet): string =
getShortTime(tweet.time)
proc getDuration*(ms: int): string =
let
sec = int(round(ms / 1000))

23
src/nitter.nim
View file

@ -2,15 +2,15 @@
import asyncdispatch, strformat, logging
from net import Port
from htmlgen import a
from os import getEnv
from os import getEnv, normalizedPath
import jester
import types, config, prefs, formatters, redis_cache, http_pool, auth, apiutils
import views/[general, about]
import routes/[
preferences, timeline, status, media, search, rss, list, debug,
unsupported, embed, resolver, broadcast, router_utils]
preferences, timeline, status, media, search, rss, list, community, debug,
unsupported, embed, resolver, broadcast, space, article, router_utils]
const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
const issuesUrl = "https://github.com/zedeus/nitter/issues"
@ -34,6 +34,10 @@ stdout.flushFile
updateDefaultPrefs(fullCfg)
setCacheTimes(cfg)
setHmacKey(cfg.hmacKey)
if cfg.hmacKey.len == 0 or cfg.hmacKey == "secretkey":
stderr.write "WARNING: insecure default 'hmacKey' in nitter.conf; " &
"set a unique random value to stop media URL signatures being forgeable.\n"
stderr.flushFile
setProxyEncoding(cfg.base64Media)
setMaxHttpConns(cfg.httpMaxConns)
setHttpProxy(cfg.proxy, cfg.proxyAuth)
@ -48,27 +52,35 @@ waitFor initRedisPool(cfg)
stdout.write &"Connected to Redis at {cfg.redisHost}:{cfg.redisPort}\n"
stdout.flushFile
createArticleRouter(cfg)
createUnsupportedRouter(cfg)
createResolverRouter(cfg)
createPrefRouter(cfg)
createTimelineRouter(cfg)
createListRouter(cfg)
createCommunityRouter(cfg)
createStatusRouter(cfg)
createSearchRouter(cfg)
createMediaRouter(cfg)
createEmbedRouter(cfg)
createRssRouter(cfg)
createBroadcastRouter(cfg)
createSpaceRouter(cfg)
createDebugRouter(cfg)
settings:
port = Port(cfg.port)
staticDir = cfg.staticDir
staticDir = normalizedPath(cfg.staticDir)
bindAddr = cfg.address
reusePort = true
maxBody = 64 * 1024
routes:
before:
# Reject malformed paths
if request.path.len == 0 or request.path[0] != '/':
halt Http400
# skip all file URLs
cond "." notin request.path
applyUrlPrefs()
@ -113,15 +125,18 @@ routes:
resp Http429, showError(
&"Instance has no auth tokens, or is fully rate limited.<br>Use {link} or try again later.", cfg)
extend articleRoute, ""
extend rss, ""
extend status, ""
extend search, ""
extend timeline, ""
extend media, ""
extend list, ""
extend community, ""
extend preferences, ""
extend resolver, ""
extend embed, ""
extend broadcastRoute, ""
extend spaceRoute, ""
extend debug, ""
extend unsupported, ""

237
src/parser.nim
View file

@ -1,10 +1,10 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, options, times, math, tables
import strutils, options, times, math, tables, uri
import packedjson, packedjson/deserialiser
import types, parserutils, utils
import experimental/parser/unifiedcard
proc parseGraphTweet(js: JsonNode): Tweet
proc parseGraphTweet*(js: JsonNode): Tweet
proc parseVerifiedType(s: string; current: VerifiedType): VerifiedType =
try: parseEnum[VerifiedType](s)
@ -46,10 +46,10 @@ proc parseUser(js: JsonNode; id=""): User =
proc parseGraphUser(js: JsonNode): User =
var user = js{"user_result", "result"}
if user.isNull:
user = ? js{"user_results", "result"}
user = js{"user_results", "result"}
if user.isNull:
if js{"core"}.notNull and js{"legacy"}.notNull:
if js{"core"}.notNull:
user = js
else:
return
@ -61,6 +61,7 @@ proc parseGraphUser(js: JsonNode): User =
# fallback to support UserMedia/recent GraphQL updates
if result.username.len == 0:
result.id = user{"rest_id"}.getStr
result.username = user{"core", "screen_name"}.getStr
result.fullname = user{"core", "name"}.getStr
result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
@ -130,6 +131,81 @@ proc parseBroadcastInfo*(js: JsonNode): Broadcast =
user: parseGraphUser(bc)
)
proc parseSpaceParticipant(js: JsonNode): SpaceParticipant =
result = SpaceParticipant(
userId: js{"user_results", "rest_id"}.getStr,
username: js{"twitter_screen_name"}.getStr,
displayName: js{"display_name"}.getStr,
avatarUrl: js{"avatar_url"}.getStr,
isVerified: js{"is_verified"}.getBool or
js{"user_results", "result", "is_blue_verified"}.getBool
)
proc parseAudioSpace*(js: JsonNode): AudioSpace =
let space = ? js{"data", "audioSpace"}
let meta = space{"metadata"}
result = AudioSpace(
id: meta{"rest_id"}.getStr,
title: meta{"title"}.getStr,
state: meta{"state"}.getStr.toUpperAscii,
mediaKey: meta{"media_key"}.getStr,
totalLiveListeners: meta{"total_live_listeners"}.getInt,
totalReplayWatched: meta{"total_replay_watched"}.getInt,
availableForReplay: meta{"is_space_available_for_replay"}.getBool
)
let startedAt = meta{"started_at"}.getInt(0)
if startedAt > 0:
result.startTime = fromUnix(startedAt div 1000).utc()
let endedAtStr = meta{"ended_at"}.getStr
if endedAtStr.len > 0:
try:
let endedAt = parseBiggestInt(endedAtStr)
if endedAt > 0:
result.endTime = fromUnix(endedAt div 1000).utc()
except ValueError:
discard
result.creator = parseGraphUser(meta{"creator_results", "result"})
for admin in space{"participants", "admins"}:
result.admins.add parseSpaceParticipant(admin)
for speaker in space{"participants", "speakers"}:
result.speakers.add parseSpaceParticipant(speaker)
proc parseGraphCommunity*(js: JsonNode): Community =
if js.isNull: return
let c = ? js{"data", "communityResults", "result"}
result = Community(
id: c{"rest_id"}.getStr(c{"id_str"}.getStr),
name: c{"name"}.getStr,
description: c{"description"}.getStr,
memberCount: c{"member_count"}.getInt,
joinPolicy: c{"join_policy"}.getStr,
category: c{"primary_community_topic", "topic_name"}.getStr,
banner: c{"custom_banner_media", "media_info", "original_img_url"}.getImageStr,
creator: parseGraphUser(c{"creator_results", "result"}),
)
let createdMs = c{"created_at"}.getInt(0)
if createdMs > 0:
result.createdAt = fromUnix(createdMs div 1000).utc()
for rule in c{"rules"}:
result.rules.add CommunityRule(
name: rule{"name"}.getStr,
description: rule{"description"}.getStr
)
for item in c{"trending_hashtags_slice", "items"}:
let tag = item{"hashtag"}.getStr
if tag.len > 0:
result.hashtags.add tag
proc parseGraphList*(js: JsonNode): List =
if js.isNull: return
@ -228,6 +304,10 @@ proc parseLegacyMediaEntities(js: JsonNode; result: var Tweet) =
result.attribution = some(parseUser(user))
else:
result.attribution = some(parseGraphUser(user))
# Set attribution link from expanded_url (strip /video/N suffix)
let expanded = m{"expanded_url"}.getStr
if expanded.len > 0:
result.attributionLink = expanded.parseUri.path.replace("/video/1", "")
of "animated_gif":
result.media.addMedia(Gif(
url: m{"video_info", "variants"}[0]{"url"}.getImageStr,
@ -236,11 +316,6 @@ proc parseLegacyMediaEntities(js: JsonNode; result: var Tweet) =
))
else: discard
with url, m{"url"}:
if result.text.endsWith(url.getStr):
result.text.removeSuffix(url.getStr)
result.text = result.text.strip()
proc parseMediaEntities(js: JsonNode; result: var Tweet) =
with mediaEntities, js{"media_entities"}:
var parsedMedia: MediaEntities
@ -261,6 +336,18 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) =
durationMs: mediaInfo{"duration_millis"}.getInt,
variants: parseVideoVariants(mediaInfo{"variants"})
))
# Parse source user for video attribution
with sourceUser, mediaEntity{"source_user_results", "result"}:
if result.attribution.isNone:
let expanded = mediaEntity{"expanded_url"}.getStr
if expanded.len > 0:
result.attributionLink = expanded.parseUri.path.replace("/video/1", "")
result.attribution = some(User(
id: sourceUser{"rest_id"}.getStr,
fullname: sourceUser{"core", "name"}.getStr,
userPic: sourceUser{"avatar", "image_url"}.getImageStr.replace("_normal", "")
))
of "ApiGif":
parsedMedia.addMedia(Gif(
url: mediaInfo{"variants"}[0]{"url"}.getImageStr,
@ -269,23 +356,9 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) =
))
else: discard
if "expanded_url" in mediaEntity:
let expandedUrl = js.getExpandedUrl
if result.text.endsWith(expandedUrl):
result.text.removeSuffix(expandedUrl)
result.text = result.text.strip()
if mediaEntities.len > 0 and parsedMedia.len == mediaEntities.len:
result.media = parsedMedia
# Remove media URLs from text
with mediaList, js{"legacy", "entities", "media"}:
for url in mediaList:
let expandedUrl = url.getExpandedUrl
if result.text.endsWith(expandedUrl):
result.text.removeSuffix(expandedUrl)
result.text = result.text.strip()
proc parsePromoVideo(js: JsonNode): Video =
result = Video(
thumb: js{"player_image_large"}.getImageVal,
@ -362,7 +435,13 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
result.url = vals{"player_url"}.getStrVal
if "youtube.com" in result.url:
result.url = result.url.replace("/embed/", "/watch?v=")
of audiospace, unknown:
of audiospace:
let spaceId = vals{"id"}.getStrVal
if spaceId.len > 0:
result.url = "/i/spaces/" & spaceId
result.title = "Twitter Space"
result.text = "Click to view Space"
of unknown:
result.title = "This card type is not supported."
else: discard
@ -428,13 +507,13 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
# graphql
with rt, js{"retweeted_status_result", "result"}:
# needed due to weird edgecase where the actual tweet data isn't included
if "legacy" in rt:
if "legacy" in rt or "rest_id" in rt:
result.retweet = some parseGraphTweet(rt)
return
with reposts, js{"repostedStatusResults"}:
with rt, reposts{"result"}:
if "legacy" in rt:
if "legacy" in rt or "rest_id" in rt:
result.retweet = some parseGraphTweet(rt)
return
@ -449,7 +528,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
result.poll = some parsePoll(jsCard)
elif name == "amplify":
result.media.addMedia(parsePromoVideo(jsCard{"binding_values"}))
else:
elif name.len > 0 and jsCard{"binding_values"}.notNull:
result.card = some parseCard(jsCard, js{"entities", "urls"})
result.expandTweetEntities(js)
@ -469,7 +548,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
result.text.removeSuffix(" Learn more.")
result.available = false
proc parseGraphTweet(js: JsonNode): Tweet =
proc parseGraphTweet*(js: JsonNode): Tweet =
if js.kind == JNull:
return Tweet()
@ -506,7 +585,7 @@ proc parseGraphTweet(js: JsonNode): Tweet =
"binding_values": %bindingObj
}
var replyId = 0
var replyId: int64 = 0
with restId, js{"reply_to_results", "rest_id"}:
replyId = restId.getId
@ -537,14 +616,40 @@ proc parseGraphTweet(js: JsonNode): Tweet =
result.poll = some parsePoll(jsCard)
elif name == "amplify":
result.media.addMedia(parsePromoVideo(jsCard{"binding_values"}))
else:
elif name.len > 0 and jsCard{"binding_values"}.notNull:
result.card = some parseCard(jsCard, js{"url_entities"})
result.expandTweetEntitiesV2(js)
parseMediaEntities(js, result)
if result.attribution.isNone:
parseLegacyMediaEntities(js{"legacy"}, result)
let hasArticle = js{"article", "article_results", "result", "title"}.getStr.len > 0
result.expandTweetEntitiesV2(js, hasArticle)
# Strip video source URL from text (for videos from other tweets)
with mediaEntities, js{"media_entities"}:
for m in mediaEntities:
if "source_status_id_str" in m:
let mediaUrl = m{"url"}.getStr
if mediaUrl.len > 0:
let idx = result.text.rfind(mediaUrl)
if idx >= 0:
result.text = result.text[0 ..< idx].strip()
break
else:
result = parseTweet(js{"legacy"}, jsCard, replyId)
result.id = js{"rest_id"}.getId
with artNode, js{"article", "article_results", "result"}:
let artTitle = artNode{"title"}.getStr
if artTitle.len > 0:
result.articlePreview = some ArticlePreview(
title: artTitle,
previewText: artNode{"preview_text"}.getStr,
coverImage: artNode{"cover_media_results", "result", "media_info", "original_img_url"}.getImageStr,
tweetId: result.id
)
result.user = parseGraphUser(js{"core"})
if result.reply.len == 0:
@ -559,6 +664,20 @@ proc parseGraphTweet(js: JsonNode): Tweet =
parseMediaEntities(js, result)
# Hide card if it's redundant with attribution (same video shown via embed)
if result.attribution.isSome and result.card.isSome:
let cardUri = get(result.card).url.parseUri
if cardUri.isTwitterUrl:
let cardPath = cardUri.path.replace("/video/1", "")
if cardPath.len > 0 and cardPath == result.attributionLink:
get(result.card).kind = hidden
# Handle retweets - check both legacy and top-level paths
with reposts, js{"legacy", "repostedStatusResults"}:
with rt, reposts{"result"}:
if "legacy" in rt or "rest_id" in rt:
result.retweet = some parseGraphTweet(rt)
with quoted, js{"quoted_status_result", "result"}:
result.quote = some(parseGraphTweet(quoted))
@ -600,6 +719,16 @@ proc parseGraphTweetResult*(js: JsonNode): Tweet =
with tweet, js{"data", "tweet_result", "result"}:
result = parseGraphTweet(tweet)
proc parseGraphTweetResults*(js: JsonNode): seq[Tweet] =
let results = js{"data", "tweetResult"}
if results.kind != JArray: return
for item in results:
let tweet = item{"result"}
if tweet.isNull: continue
let t = parseGraphTweet(tweet)
if t != nil:
result.add t
proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
result = Conversation(replies: Result[Chain](beginning: true))
@ -807,3 +936,49 @@ proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] =
elif typ == "TimelineReplaceEntry":
if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"):
result.bottom = instruction{"entry", "content", "value"}.getStr
proc parseGraphCommunityTimeline*(js: JsonNode; after=""): Timeline =
result = Timeline(beginning: after.len == 0)
let communityResult = js{"data", "communityResults", "result"}
let instructions = ? select(
communityResult{"ranked_community_timeline", "timeline", "instructions"},
communityResult{"community_media_timeline", "timeline", "instructions"},
communityResult{"community_filtered_timeline", "timeline", "instructions"}
)
if instructions.len == 0:
return
for i in instructions:
if i{"entries"}.notNull:
for e in i{"entries"}:
let entryId = e.getEntryId
if entryId.startsWith("tweet") or entryId.startsWith("profile-grid") or
entryId.startsWith("communities-grid"):
for tweet in extractTweetsFromEntry(e):
result.content.add tweet
elif entryId.startsWith("cursor-bottom"):
result.bottom = e{"content", "value"}.getStr
if after.len == 0 and i.getTypeName == "TimelinePinEntry":
var tweets = extractTweetsFromEntry(i{"entry"})
for tweet in tweets.mitems:
tweet.pinned = true
if tweets.len > 0:
result.content.insert(tweets, 0)
proc parseGraphCommunityMembers*(js: JsonNode; after=""): Result[User] =
result = Result[User](beginning: after.len == 0)
let r = js{"data", "communityResults", "result"}
let slice = if not r{"members_slice"}.isNull: r{"members_slice"}
else: r{"moderators_slice"}
for item in slice{"items_results"}:
let user = parseGraphUser(item{"result"})
if user.username.len > 0:
result.content.add user
let cursor = slice{"slice_info", "next_cursor"}.getStr
if cursor.len > 0:
result.bottom = cursor

49
src/parserutils.nim
View file

@ -185,12 +185,16 @@ proc extractSlice(js: JsonNode): Slice[int] =
result = js["indices"][0].getInt ..< js["indices"][1].getInt
proc extractUrls(result: var seq[ReplaceSlice]; js: JsonNode;
textLen: int; hideTwitter = false) =
textLen: int; hideTwitter = false;
hideArticle = false) =
let
url = js.getExpandedUrl
slice = js.extractSlice
if hideTwitter and slice.b.succ >= textLen and url.isTwitterUrl:
if hideArticle and url.isTwitterUrl and "/article/" in url:
if slice.a < textLen:
result.add ReplaceSlice(kind: rkRemove, slice: slice)
elif hideTwitter and slice.b.succ >= textLen and url.isTwitterUrl:
if slice.a < textLen:
result.add ReplaceSlice(kind: rkRemove, slice: slice)
else:
@ -202,15 +206,28 @@ proc extractHashtags(result: var seq[ReplaceSlice]; js: JsonNode) =
proc replacedWith(runes: seq[Rune]; repls: openArray[ReplaceSlice];
textSlice: Slice[int]): string =
let
runeLen = runes.len
safeStart = max(0, textSlice.a)
safeEnd = min(runeLen, textSlice.b)
var validRepls: seq[ReplaceSlice]
for rep in repls:
if rep.slice.a >= 0 and rep.slice.b >= 0 and rep.slice.b < runeLen and rep.slice.a <= rep.slice.b:
validRepls.add rep
template extractLowerBound(i: int; idx): int =
if i > 0: repls[idx].slice.b.succ else: textSlice.a
if i > 0: min(validRepls[idx].slice.b.succ, runeLen) else: safeStart
result = newStringOfCap(runes.len)
for i, rep in repls:
result.add $runes[extractLowerBound(i, i - 1) ..< rep.slice.a]
for i, rep in validRepls:
let lower = extractLowerBound(i, i - 1)
if lower < rep.slice.a:
result.add $runes[lower ..< rep.slice.a]
case rep.kind
of rkHashtag:
if rep.slice.a.succ <= rep.slice.b:
let
name = $runes[rep.slice.a.succ .. rep.slice.b]
symbol = $runes[rep.slice.a]
@ -222,8 +239,8 @@ proc replacedWith(runes: seq[Rune]; repls: openArray[ReplaceSlice];
of rkRemove:
discard
let rest = extractLowerBound(repls.len, ^1) ..< textSlice.b
if rest.a <= rest.b:
let rest = extractLowerBound(validRepls.len, ^1) ..< safeEnd
if rest.a >= 0 and rest.a <= rest.b and rest.b <= runeLen:
result.add $runes[rest]
proc deduplicate(s: var seq[ReplaceSlice]) =
@ -326,10 +343,11 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
replyTo = reply.getStr
tweet.reply.add replyTo
tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote or hasJobCard)
tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo,
hasQuote or hasJobCard)
proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: Slice[int];
hasRedundantLink=false) =
hasRedundantLink=false; hasArticle=false) =
let hasCard = tweet.card.isSome
var replacements = newSeq[ReplaceSlice]()
@ -340,7 +358,8 @@ proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: S
if urlStr.len == 0 or urlStr notin text:
continue
replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink)
replacements.extractUrls(u, textSlice.b, hideTwitter = hasRedundantLink,
hideArticle = hasArticle)
if hasCard and u{"url"}.getStr == get(tweet.card).url:
get(tweet.card).url = u.getExpandedUrl
@ -371,22 +390,26 @@ proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: S
tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false)
proc expandTweetEntitiesV2*(tweet: Tweet; js: JsonNode) =
proc expandTweetEntitiesV2*(tweet: Tweet; js: JsonNode; hasArticle=false) =
let
textRange = js{"details", "display_text_range"}
textSlice = textRange{0}.getInt .. textRange{1}.getInt
hasQuote = "quoted_tweet_results" in js
hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails
hasAttribution = tweet.attribution.isSome
tweet.expandTextEntitiesV2(js, tweet.text, textSlice, hasQuote or hasJobCard)
tweet.expandTextEntitiesV2(js, tweet.text, textSlice,
hasQuote or hasJobCard or hasAttribution,
hasArticle)
proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) =
let
entities = ? js{"entity_set"}
text = js{"text"}.getStr.multiReplace(("<", unicodeOpen), (">", unicodeClose))
textSlice = 0..text.runeLen
hasAttribution = tweet.attribution.isSome
tweet.expandTextEntities(entities, text, textSlice)
tweet.expandTextEntities(entities, text, textSlice, hasRedundantLink=hasAttribution)
tweet.text = tweet.text.multiReplace((unicodeOpen, xmlOpen), (unicodeClose, xmlClose))

41
src/redis_cache.nim
View file

@ -144,8 +144,9 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} =
else:
let user = await getGraphUserById(userId)
result = user.username
if result.len > 0:
await setEx(key, baseCacheTime, result)
if result.len > 0 and user.id.len > 0:
if user.id.len > 0:
await all(cacheUserId(result, user.id), cache(user))
# proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
@ -172,6 +173,21 @@ proc getCachedBroadcast*(id: string): Future[Broadcast] {.async.} =
await cache(result)
result.m3u8Url = await fetchBroadcastStream(result.mediaKey)
proc cache*(data: AudioSpace) {.async.} =
if data.id.len == 0: return
let ttl = if data.state == "RUNNING": baseCacheTime div 6 else: baseCacheTime
await setEx("sp:" & data.id, ttl, compress(toFlatty(data)))
proc getCachedAudioSpace*(id: string): Future[AudioSpace] {.async.} =
if id.len == 0: return
let cached = await get("sp:" & id)
if cached != redisNil:
cached.deserialize(AudioSpace)
else:
result = await getAudioSpace(id)
await cache(result)
result.m3u8Url = await fetchBroadcastStream(result.mediaKey)
proc cache*(data: AccountInfo; name: string) {.async.} =
await setEx("ai:" & toLower(name), baseCacheTime * 24, compress(toFlatty(data)))
@ -194,6 +210,29 @@ proc getCachedPhotoRail*(id: string): Future[PhotoRail] {.async.} =
result = await getPhotoRail(id)
await cache(result, id)
proc cache*(data: Community) {.async.} =
if data.id.len == 0: return
await setEx("cm:" & data.id, listCacheTime, compress(toFlatty(data)))
proc getCachedCommunity*(id: string): Future[Community] {.async.} =
if id.len == 0: return
let cached = await get("cm:" & id)
if cached != redisNil:
cached.deserialize(Community)
else:
result = await getGraphCommunity(id)
await cache(result)
proc getCachedCommunityModerators*(id: string): Future[seq[User]] {.async.} =
if id.len == 0: return
let cached = await get("cmm:" & id)
if cached != redisNil:
cached.deserialize(seq[User])
else:
let mods = await getGraphCommunityModerators(id)
result = mods.content
await setEx("cmm:" & id, listCacheTime, compress(toFlatty(result)))
proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} =
let list = if id.len == 0: redisNil
else: await get("l:" & id)

48
src/routes/article.nim Normal file
View file

@ -0,0 +1,48 @@
# SPDX-License-Identifier: AGPL-3.0-only
import asyncdispatch, tables, strutils
import jester, karax/vdom
import ".."/[types, api]
import ../views/[article, general]
import router_utils
export api, article, vdom, general, router_utils
proc createArticleRouter*(cfg: Config) =
router articleRoute:
get "/i/article/@id":
cond @"id".allCharsInSet(Digits)
let article = await getGraphArticle(@"id")
if article == nil:
resp Http404, showError("Article not found", cfg)
var tweetIds: seq[string]
for e in article.entities.values:
if e.kind == "TWEET":
tweetIds.add e.tweetId
var tweets = initTable[int64, Tweet]()
if tweetIds.len > 0:
try:
for t in await getGraphTweetResults(tweetIds):
tweets[t.id] = t
except CatchableError:
discard
let
prefs = requestPrefs()
path = getPath()
html = renderArticle(article, tweets, path, prefs, @"id")
twitterUrl = "https://x.com/" & article.user.username & "/article/" & @"id"
resp renderMain(html, request, cfg, prefs, titleText=article.title,
twitterLink=twitterUrl)
get "/@name/article/@id/?":
cond '.' notin @"name"
cond @"id".allCharsInSet(Digits)
redirect("/i/article/" & @"id")
get "/@name/status/@id/article":
cond '.' notin @"name"
cond @"id".allCharsInSet(Digits)
redirect("/i/article/" & @"id")

89
src/routes/community.nim Normal file
View file

@ -0,0 +1,89 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strformat
import jester
import router_utils
import ".."/[types, redis_cache, api]
import ../views/[general, timeline, community]
export community
template respCommunity*(cmty: Community; title: string; nav, vnode: typed) =
if cmty.id.len == 0 or cmty.name.len == 0:
resp Http404, showError(&"""Community "{@"id"}" not found""", cfg)
let html = renderCommunity(vnode, nav, cmty)
resp renderMain(html, request, cfg, prefs, titleText=title, banner=cmty.banner)
proc createCommunityRouter*(cfg: Config) =
router community:
get "/i/communities/@id/?":
cond '.' notin @"id"
let
prefs = requestPrefs()
cmty = await getCachedCommunity(@"id")
tl = await getGraphCommunityTweets(cmty.id, "Relevance", getCursor())
respCommunity(cmty, cmty.name,
renderCommunityTabs(QueryKind.posts, cmty),
renderTimelineTweets(tl, prefs, request.path))
get "/i/communities/@id/latest":
cond '.' notin @"id"
let
prefs = requestPrefs()
cmty = await getCachedCommunity(@"id")
tl = await getGraphCommunityTweets(cmty.id, "Recency", getCursor())
respCommunity(cmty, cmty.name & " - Latest",
renderCommunityTabs(QueryKind.replies, cmty),
renderTimelineTweets(tl, prefs, request.path))
get "/i/communities/@id/media":
cond '.' notin @"id"
let
prefs = requestPrefs()
cmty = await getCachedCommunity(@"id")
tl = await getGraphCommunityMedia(cmty.id, getCursor())
respCommunity(cmty, cmty.name & " - Media",
renderCommunityTabs(QueryKind.media, cmty),
renderTimelineTweets(tl, prefs, request.path))
get "/i/communities/@id/about":
cond '.' notin @"id"
let
prefs = requestPrefs()
cmty = await getCachedCommunity(@"id")
mods = await getCachedCommunityModerators(cmty.id)
respCommunity(cmty, cmty.name & " - About",
renderCommunityTabs(QueryKind.userList, cmty),
renderCommunityAbout(cmty, mods))
get "/i/communities/@id/members":
cond '.' notin @"id"
let
prefs = requestPrefs()
cmty = await getCachedCommunity(@"id")
members = await getGraphCommunityMembers(cmty.id, getCursor())
respCommunity(cmty, cmty.name & " - Members",
renderMemberTabs(cmty, false),
renderTimelineUsers(members, prefs, request.path))
get "/i/communities/@id/moderators":
cond '.' notin @"id"
let
prefs = requestPrefs()
cmty = await getCachedCommunity(@"id")
mods = await getCachedCommunityModerators(cmty.id)
respCommunity(cmty, cmty.name & " - Moderators",
renderMemberTabs(cmty, true),
renderTimelineUsers(Result[User](content: mods), prefs, request.path))
get "/i/communities/@id/hashtag/@tag":
cond '.' notin @"id"
let
prefs = requestPrefs()
cmty = await getCachedCommunity(@"id")
tl = await getGraphCommunityHashtags(cmty.id, @"tag", getCursor())
respCommunity(cmty, cmty.name & " - #" & @"tag",
renderHashtagHeader(cmty, @"tag"),
renderTimelineTweets(tl, prefs, request.path))

59
src/routes/media.nim
View file

@ -1,6 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
import uri, strutils, httpclient, os, hashes, base64, re
import asynchttpserver, asyncstreams, asyncfile, asyncnet
import asyncdispatch
import jester
@ -15,7 +16,9 @@ const
maxAge* = "max-age=604800"
proc safeFetch*(url: string): Future[string] {.async.} =
let client = newAsyncHttpClient()
# maxRedirects=0: the caller already validated the host, so never follow a
# redirect off the allowlisted host (would re-open the #1411 SSRF).
let client = newAsyncHttpClient(maxRedirects = 0)
try: result = await client.getContent(url)
except: discard
finally: client.close()
@ -30,17 +33,42 @@ template respond*(req: asynchttpserver.Request; headers) =
proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
result = Http200
let
request = req.getNativeReq()
client = newAsyncHttpClient()
let request = req.getNativeReq()
var fetchUrl = url
for attempt in 0 .. 2:
let client = newAsyncHttpClient(maxRedirects = 0)
var shouldRetry = false
try:
let res = await client.get(url)
let resFut = client.get(fetchUrl)
let completed = await withTimeout(resFut, 5000)
if not completed:
if attempt < 2:
echo "[media] Retry $1/2, timeout after 5s, url: $2" % [$(attempt + 1), fetchUrl]
shouldRetry = true
else:
echo "[media] Proxying timeout after 5s, url: $1" % [fetchUrl]
return Http504
else:
let res = resFut.read()
if res.status != "200 OK":
if res.status != "404 Not Found":
echo "[media] Proxying failed, status: $1, url: $2" % [res.status, url]
if res.status == "404 Not Found":
return Http404
if res.status.startsWith("30") and res.headers.hasKey("location"):
let location = res.headers["location", 0]
if isTwitterUrl(location):
fetchUrl = location
shouldRetry = true
continue
else:
return Http403
if attempt < 2:
echo "[media] Retry $1/2, status: $2, url: $3" % [$(attempt + 1), res.status, fetchUrl]
shouldRetry = true
else:
echo "[media] Proxying failed, status: $1, url: $2" % [res.status, fetchUrl]
return Http404
else:
let hashed = $hash(url)
if request.headers.getOrDefault("If-None-Match") == hashed:
return Http304
@ -66,11 +94,18 @@ proc proxyMedia*(req: jester.Request; url: string): Future[HttpCode] {.async.} =
if hasValue:
await request.client.send(data)
data.setLen 0
except HttpRequestError, ProtocolError, OSError:
echo "[media] Proxying exception, error: $1, url: $2" % [getCurrentExceptionMsg(), url]
return Http200
except CatchableError:
if attempt < 2:
echo "[media] Retry $1/2, error: $2, url: $3" % [$(attempt + 1), getCurrentExceptionMsg(), fetchUrl]
shouldRetry = true
else:
echo "[media] Proxying exception, error: $1, url: $2" % [getCurrentExceptionMsg(), fetchUrl]
result = Http404
finally:
client.close()
if not shouldRetry:
break
template check*(code): untyped =
if code != Http200:
@ -122,12 +157,12 @@ proc createMediaRouter*(cfg: Config) =
get re"^\/video\/(enc)?\/?(.+)\/(.+)$":
let url = decoded(request, 2)
cond "http" in url
cond isTwitterUrl(url)
if getHmac(url) != request.matches[1]:
resp Http403, showError("Failed to verify signature", cfg)
if ".mp4" in url or ".ts" in url or ".m4s" in url:
if ".mp4" in url or ".ts" in url or ".m4s" in url or ".aac" in url:
let code = await proxyMedia(request, url)
check code

3
src/routes/preferences.nim
View file

@ -40,3 +40,6 @@ proc createPrefRouter*(cfg: Config) =
savePref("hlsPlayback", "on", request)
redirect(refPath())
post "/enablemp4":
savePref("mp4Playback", "on", request)
redirect(refPath())

3
src/routes/router_utils.nim
View file

@ -8,8 +8,9 @@ export utils, prefs, types, uri
template savePref*(pref, value: string; req: Request; expire=false) =
if not expire or pref in cookies(req):
let sameSite = if cfg.useHttps: None else: Lax
setCookie(pref, value, daysForward(when expire: -10 else: 360),
httpOnly=true, secure=cfg.useHttps, sameSite=None, path="/")
httpOnly=true, secure=cfg.useHttps, sameSite=sameSite, path="/")
template requestPrefs*(): untyped {.dirty.} =
getPrefs(cookies(request), params(request))

7
src/routes/search.nim
View file

@ -46,6 +46,7 @@ proc createSearchRouter*(cfg: Config) =
redirect("/search?f=tweets&q=" & encodeUrl("#" & @"hash"))
get "/opensearch":
let url = getUrlPrefix(cfg) & "/search?f=tweets&q="
resp Http200, {"Content-Type": "application/opensearchdescription+xml"},
generateOpenSearchXML(cfg.title, cfg.hostname, url)
let
url = getUrlPrefix(cfg) & "/search?f=tweets&q="
headers = {"Content-Type": "application/opensearchdescription+xml"}
resp Http200, headers, generateOpenSearchXML(cfg.title, cfg.hostname, url)

36
src/routes/space.nim Normal file
View file

@ -0,0 +1,36 @@
# SPDX-License-Identifier: AGPL-3.0-only
import asyncdispatch, strutils
import jester
import router_utils
import ".."/[types, formatters, redis_cache]
import ../views/[general, space]
import media
export space
proc createSpaceRouter*(cfg: Config) =
router spaceRoute:
get "/i/spaces/@id":
cond @"id".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9'})
let sp = await getCachedAudioSpace(@"id")
if sp.id.len == 0:
resp Http404, showError("Space not found", cfg)
let prefs = requestPrefs()
resp renderMain(renderSpace(sp, prefs, request.path), request, cfg, prefs,
sp.title, ogTitle=sp.title)
get "/i/spaces/@id/stream":
cond @"id".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9'})
let sp = await getCachedAudioSpace(@"id")
if sp.m3u8Url.len == 0:
resp Http404
let manifest = await safeFetch(sp.m3u8Url)
if manifest.len == 0:
resp Http502
resp proxifyVideo(manifest, requestPrefs().proxyVideos, sp.m3u8Url), m3u8Mime

33
src/routes/timeline.nim
View file

@ -135,6 +135,39 @@ proc createTimelineRouter*(cfg: Config) =
resp renderMain(aboutHtml, request, cfg, prefs,
"About @" & info.username)
get "/@name/@kind/?":
cond @"kind" in ["followers", "following"]
cond '.' notin @"name"
cond @"name".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9', '_'})
let
prefs = requestPrefs()
name = @"name"
userId = await getUserId(name)
if userId.len == 0:
resp Http404, showError("User \"" & name & "\" not found", cfg)
if userId == "suspended":
resp showError(getSuspended(name), cfg)
let
cursor = getCursor()
isFollowers = @"kind" == "followers"
user = await getCachedUser(name)
results = if isFollowers: await getGraphFollowers(userId, cursor)
else: await getGraphFollowing(userId, cursor)
if user.protected:
resp renderMain(renderProtected(user.username), request, cfg, prefs,
"Protected account @" & name)
let
tab = if isFollowers: "Followers" else: "Following"
title = (if isFollowers: "People following @" else: "People followed by @") & name
html = renderUserList(user, results, prefs, request.path, tab)
resp renderMain(html, request, cfg, prefs, title,
images = @[user.getUserPic("_400x400")])
get "/@name/?@tab?/?":
cond '.' notin @"name"
cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]

278
src/sass/_article.scss Normal file
View file

@ -0,0 +1,278 @@
.article-page {
max-width: 700px;
margin: 0 auto 20px;
background-color: var(--bg_panel);
> .top-ref {
padding-top: 20px;
}
.article-cover {
width: 100%;
display: block;
}
.article-body {
padding: 20px;
> :last-child {
margin-bottom: 0;
}
.article-title {
display: block;
font-size: 2rem;
line-height: 1.3;
margin: 0 0 10px;
color: var(--fg_color);
}
.article-author {
margin-bottom: 12px;
padding-bottom: 10px;
border-bottom: 1px solid var(--border_grey);
font-size: 14px;
.article-author-row {
display: flex;
align-items: center;
gap: 8px;
}
.article-avatar {
display: flex;
}
.avatar {
width: 40px;
height: 40px;
}
.article-author-name {
display: flex;
align-items: center;
margin-bottom: 2px;
.fullname {
font-size: 15px;
}
.verified-icon {
margin-left: 2px;
}
}
.article-author-meta {
display: flex;
align-items: center;
}
.fullname {
font-weight: 700;
color: var(--fg_color);
max-width: unset;
text-overflow: unset;
overflow: visible;
white-space: normal;
}
.username,
.article-date-sep,
.article-date {
color: var(--fg_dark);
}
.username {
margin-left: 0;
}
.article-date-sep {
margin: 0 4px;
}
.article-date {
text-decoration: none;
&:hover {
text-decoration: underline;
}
}
.tweet-stats {
margin-top: 6px;
.tweet-stat {
padding-top: 0;
}
}
}
> h1 {
display: block;
font-size: 1.8rem;
margin: 25px 0 15px;
}
> h2 {
font-size: 1.4rem;
font-weight: bold;
margin: 20px 0 12px;
}
> h3 {
font-size: 1.2rem;
font-weight: bold;
margin: 18px 0 10px;
}
> p {
font-size: 16px;
line-height: 1.7;
margin: 16px 0;
word-wrap: break-word;
}
.blockquote-attribution {
display: block;
margin-top: 0.5em;
}
> blockquote {
border-left: 3px solid var(--accent);
padding-left: 16px;
margin: 16px 0;
color: var(--fg_faded);
font-size: 16px;
line-height: 1.7;
}
> pre {
background-color: var(--bg_elements);
padding: 12px 16px;
border-radius: 6px;
overflow-x: auto;
margin: 16px 0;
code {
font-family: monospace;
font-size: 14px;
color: var(--fg_color);
}
}
code {
background-color: var(--bg_elements);
padding: 2px 5px;
border-radius: 3px;
font-family: monospace;
font-size: 0.9em;
}
> ul,
> ol {
margin: 16px 0;
padding-left: 2em;
li {
font-size: 16px;
line-height: 1.7;
margin: 6px 0;
}
}
.article-media {
text-align: center;
margin: 20px 0;
img,
video {
max-width: 100%;
border-radius: 12px;
}
.article-media-caption {
color: var(--fg_faded);
font-size: 0.875rem;
margin-top: 6px;
}
}
> a,
> p a,
> h1 a,
> h2 a,
> h3 a,
> blockquote a,
> ul a,
> ol a {
color: var(--accent);
text-decoration: none;
&:hover {
text-decoration: underline;
}
}
.article-divider {
border: none;
border-top: 1px solid var(--border_grey);
margin: 30px 0;
}
.timeline-item {
margin: 20px 0;
border: 1px solid var(--border_grey);
border-radius: 12px;
overflow: hidden;
}
}
}
.conversation .article-page {
max-width: 100%;
margin-bottom: 0;
}
.article-card {
.card-image-container {
position: relative;
}
.card-image img {
height: auto;
}
.article-card-badge {
position: absolute;
bottom: 8px;
left: 8px;
background: rgba(0, 0, 0, 0.75);
color: #fff;
font-size: 13px;
font-weight: 700;
padding: 2px 8px;
border-radius: 4px;
}
}
.quote .article-card {
margin: 0;
.card-container {
border: none;
border-radius: 0;
border-top: solid 1px var(--dark_grey);
}
}
@media (max-width: 700px) {
.article-page {
.article-body {
padding: 12px 15px 25px;
.article-title {
font-size: 1.6rem;
}
}
}
}

149
src/sass/_space.scss Normal file
View file

@ -0,0 +1,149 @@
.space-page {
max-width: 800px;
width: 100%;
margin: 20px auto 0;
}
.space-panel {
background-color: var(--bg_panel);
border: 1px solid var(--border_grey);
border-radius: 8px;
overflow: hidden;
}
.space-player {
position: relative;
background: linear-gradient(135deg, #7b2a8c 0%, #9b3ab1 100%);
min-height: 140px;
display: flex;
align-items: center;
justify-content: center;
audio {
width: 100%;
padding: 15px;
box-sizing: border-box;
&:not([controls]) {
display: none;
}
}
.video-overlay {
background-color: transparent;
}
}
.space-live {
background: #e0245e;
color: white;
padding: 3px 8px;
border-radius: 4px;
font-weight: bold;
font-size: 12px;
text-transform: uppercase;
position: absolute;
top: 8px;
right: 8px;
}
.space-info {
padding: 16px;
}
.space-header {
display: flex;
justify-content: space-between;
align-items: flex-start;
gap: 12px;
margin-bottom: 16px;
}
.space-title {
font-size: 18px;
font-weight: bold;
margin: 0;
line-height: 1.3;
flex: 1;
}
.space-meta {
display: flex;
flex-direction: column;
align-items: flex-end;
gap: 4px;
flex-shrink: 0;
font-size: 14px;
color: var(--fg_faded);
}
.listener-count {
color: var(--fg_color);
}
.space-state {
color: var(--fg_dark);
}
.space-participants {
border-top: 1px solid var(--border_grey);
padding-top: 12px;
}
.space-participant {
margin-bottom: 10px;
a {
display: flex;
align-items: center;
gap: 10px;
color: var(--fg_color);
padding: 6px 0;
}
img {
width: 40px;
height: 40px;
border-radius: 50%;
flex-shrink: 0;
}
}
.participant-info {
min-width: 0;
}
.participant-name {
display: flex;
align-items: center;
gap: 4px;
strong {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.verified-icon {
margin-bottom: 0;
position: relative;
top: -2px;
}
}
.host-badge {
background: var(--accent);
color: white;
padding: 2px 7px;
border-radius: 3px;
font-size: 11px;
font-weight: 600;
line-height: 1;
position: relative;
top: 1px;
}
.participant-username {
color: var(--fg_dark);
font-size: 13px;
}

3
src/sass/index.scss
View file

@ -8,6 +8,8 @@
@import "timeline";
@import "search";
@import "broadcast";
@import "space";
@import "_article";
body {
// colors
@ -159,6 +161,7 @@ body.fixed-nav .container {
.verified-icon {
display: inline-block;
position: relative;
width: 14px;
height: 14px;
margin-bottom: 2px;

1
src/sass/profile/_base.scss
View file

@ -4,6 +4,7 @@
@import "card";
@import "about-account";
@import "photo-rail";
@import "community";
.profile-tabs {
@include panel(auto, 900px);

203
src/sass/profile/_community.scss Normal file
View file

@ -0,0 +1,203 @@
.community-header {
padding: 12px 15px;
border-bottom: 1px solid var(--border_grey);
background-color: var(--bg_panel);
.community-name {
font-size: 22px;
margin-bottom: 6px;
a {
color: inherit;
}
}
.community-category {
display: inline-block;
background-color: var(--bg_elements);
border: 1px solid var(--border_grey);
border-radius: 16px;
padding: 2px 12px;
font-size: 13px;
color: var(--fg_faded);
margin-bottom: 8px;
}
.community-description {
color: var(--fg_faded);
margin-bottom: 8px;
line-height: 1.4;
}
.community-member-count {
font-weight: bold;
color: inherit;
}
.community-stats {
color: var(--grey);
font-size: 14px;
}
}
.community-about {
padding: 16px 15px 15px;
background-color: var(--bg_panel);
h2 {
font-size: 18px;
margin: 0 0 12px;
}
.community-info {
border-bottom: 1px solid var(--border_grey);
padding-bottom: 12px;
}
.community-info-item {
display: flex;
gap: 10px;
padding: 8px 0;
align-items: center;
.verified-icon {
margin-left: 2px;
}
> .icon-container {
display: flex;
align-items: center;
justify-content: center;
font-size: 16px;
color: var(--grey);
flex-shrink: 0;
width: 26px;
height: 26px;
}
strong {
color: var(--fg_color);
}
a {
color: var(--accent);
}
}
.community-rules {
border-bottom: 1px solid var(--border_grey);
padding: 16px 0 12px;
}
.community-rules-intro {
color: var(--fg_faded);
font-size: 14px;
margin: 0 0 12px;
}
.community-rule {
display: flex;
gap: 10px;
padding: 10px 0;
align-items: flex-start;
.community-rule-number {
display: flex;
align-items: center;
justify-content: center;
min-width: 26px;
height: 26px;
border-radius: 50%;
background-color: var(--accent);
color: var(--fg_color);
font-weight: bold;
font-size: 13px;
flex-shrink: 0;
}
.community-rule-content p {
margin: 4px 0 0;
color: var(--fg_faded);
font-size: 14px;
}
}
.community-moderators {
padding-top: 16px;
h2 {
display: flex;
align-items: center;
justify-content: space-between;
}
.community-mods-link {
font-size: 14px;
font-weight: normal;
color: var(--accent);
}
}
.community-moderator {
display: flex;
gap: 10px;
padding: 8px 0;
align-items: center;
.community-mod-avatar {
width: 40px;
height: 40px;
border-radius: 50%;
}
.community-mod-info {
display: flex;
flex-direction: column;
}
.community-mod-name {
display: flex;
align-items: center;
font-weight: bold;
color: var(--fg_color);
.verified-icon {
margin-left: 2px;
}
}
.community-mod-username {
color: var(--fg_faded);
font-size: 14px;
}
}
}
.community-tags {
display: flex;
flex-wrap: wrap;
gap: 8px;
padding: 10px 15px;
border-bottom: 1px solid var(--border_grey);
.community-tag {
display: inline-block;
background-color: var(--bg_elements);
border: 1px solid var(--border_grey);
border-radius: 16px;
padding: 4px 12px;
font-size: 13px;
color: var(--accent);
}
}
.community-hashtag-header {
padding: 12px 15px;
border-bottom: 1px solid var(--border_grey);
.community-hashtag-title {
font-size: 20px;
color: var(--accent);
margin: 0;
}
}

2
src/sass/tweet/_base.scss
View file

@ -305,7 +305,7 @@
margin-top: 4px;
margin-bottom: -2px;
.icon-attention {
.icon-attention-circled {
margin-right: -3px;
}
}

5
src/sass/tweet/media.scss
View file

@ -90,7 +90,10 @@
max-height: unset;
}
.media-gif video {
.media-gif video,
.media-gif img {
width: 100%;
height: 100%;
max-height: 530px;
background-color: #101010;
}

3
src/sass/tweet/quote.scss
View file

@ -100,7 +100,8 @@
justify-content: center;
background-color: var(--bg_color);
video {
video,
img {
height: unset;
width: unset;
max-height: 100%;

34
src/sass/tweet/video.scss
View file

@ -28,6 +28,40 @@ video {
}
}
.video-download {
position: absolute;
top: 8px;
right: 8px;
z-index: 2;
color: var(--accent);
text-decoration: none;
opacity: 0;
transition: opacity 0.2s;
filter: drop-shadow(0 1px 2px rgba(0, 0, 0, 0.7));
.icon-container {
margin: 0;
}
.icon-download-alt {
font-size: 18px;
}
}
.attachment:hover .video-download {
opacity: 1;
&:hover {
color: var(--fg_color);
}
}
@media (hover: none) {
.video-download {
opacity: 1;
}
}
.video-overlay {
@include play-button;
background-color: $shadow;

89
src/types.nim
View file

@ -16,6 +16,7 @@ type
ApiUrl* = object
endpoint*: string
params*: seq[(string, string)]
skipTid*: bool
ApiReq* = object
oauth*: ApiUrl
@ -60,6 +61,7 @@ type
rateLimited = 88
expiredToken = 89
listIdOrSlug = 112
timelineUnavailable = 131
tweetNotFound = 144
tweetNotAuthorized = 179
forbidden = 200
@ -127,6 +129,28 @@ type
availableForReplay*: bool
user*: User
SpaceParticipant* = object
userId*: string
username*: string
displayName*: string
avatarUrl*: string
isVerified*: bool
AudioSpace* = object
id*: string
title*: string
state*: string
mediaKey*: string
m3u8Url*: string
totalLiveListeners*: int
totalReplayWatched*: int
startTime*: DateTime
endTime*: DateTime
availableForReplay*: bool
creator*: User
admins*: seq[SpaceParticipant]
speakers*: seq[SpaceParticipant]
VideoType* = enum
m3u8 = "application/x-mpegURL"
mp4 = "video/mp4"
@ -150,7 +174,7 @@ type
variants*: seq[VideoVariant]
QueryKind* = enum
posts, replies, media, users, tweets, userList
posts, replies, media, users, tweets, userList, followers, following
Query* = object
kind*: QueryKind
@ -197,6 +221,44 @@ type
PhotoRail* = seq[GalleryPhoto]
Article* = ref object
title*: string
coverImage*: string
user*: User
time*: DateTime
stats*: TweetStats
paragraphs*: seq[ArticleParagraph]
entities*: Table[int, ArticleEntity]
media*: Table[string, ArticleMedia]
ArticleParagraph* = object
text*: string
kind*: string
inlineStyles*: seq[ArticleStyle]
entityRanges*: seq[ArticleEntityRange]
ArticleStyle* = object
offset*: int
length*: int
style*: string
ArticleEntityRange* = object
offset*: int
length*: int
key*: int
ArticleEntity* = object
kind*: string
url*: string
mediaIds*: seq[string]
tweetId*: string
markdown*: string
caption*: string
ArticleMedia* = object
kind*: string
url*: string
Poll* = object
options*: seq[string]
values*: seq[int]
@ -246,6 +308,12 @@ type
likes*: int
views*: int
ArticlePreview* = object
title*: string
previewText*: string
coverImage*: string
tweetId*: int64
Tweet* = ref object
id*: int64
threadId*: int64
@ -264,6 +332,7 @@ type
stats*: TweetStats
retweet*: Option[Tweet]
attribution*: Option[User]
attributionLink*: string
mediaTags*: seq[User]
quote*: Option[Tweet]
card*: Option[Card]
@ -273,6 +342,7 @@ type
note*: string
isAd*: bool
isAI*: bool
articlePreview*: Option[ArticlePreview]
Tweets* = seq[Tweet]
@ -315,6 +385,23 @@ type
members*: int
banner*: string
CommunityRule* = object
name*: string
description*: string
Community* = object
id*: string
name*: string
description*: string
memberCount*: int
banner*: string
creator*: User
category*: string
joinPolicy*: string
createdAt*: DateTime
rules*: seq[CommunityRule]
hashtags*: seq[string]
GlobalObjects* = ref object
tweets*: Table[string, Tweet]
users*: Table[string, User]

6
src/utils.nim
View file

@ -40,12 +40,14 @@ proc getVidUrl*(link: string): string =
&"/video/{sig}/{encodeUrl(link)}"
proc getPicUrl*(link: string): string =
if link.len == 0: return
if base64Media:
&"/pic/enc/{encode(link, safe=true)}"
else:
&"/pic/{encodeUrl(link)}"
proc getOrigPicUrl*(link: string): string =
if link.len == 0: return
if base64Media:
&"/pic/orig/enc/{encode(link, safe=true)}"
else:
@ -57,8 +59,8 @@ proc filterParams*(params: Table): seq[(string, string)] =
result.add p
proc isTwitterUrl*(uri: Uri): bool =
uri.hostname in twitterDomains or
uri.hostname.endsWith(".video.pscp.tv")
uri.scheme in ["http", "https"] and
(uri.hostname in twitterDomains or uri.hostname.endsWith(".video.pscp.tv"))
proc isTwitterUrl*(url: string): bool =
isTwitterUrl(parseUri(url))

248
src/views/article.nim Normal file
View file

@ -0,0 +1,248 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, strformat, tables, unicode, bitops, uri
import karax/[karaxdsl, vdom]
import renderutils, tweet, timeline
import ".."/[types, utils, formatters]
proc renderAtomicParagraph(paragraph: ArticleParagraph; article: Article;
tweets: Table[int64, Tweet]; path: string;
prefs: Prefs): VNode =
if paragraph.entityRanges.len == 0:
return text ""
let er = paragraph.entityRanges[0]
if er.key notin article.entities:
return text ""
let entity = article.entities[er.key]
case entity.kind
of "MEDIA":
buildHtml(tdiv(class="article-media")):
for id in entity.mediaIds:
let media = article.media.getOrDefault(id)
if media.url.len == 0:
continue
case media.kind
of "ApiGif":
video(src=getVidUrl(media.url), controls="", autoplay="", loop="",
muted="")
of "ApiVideo":
video(src=getVidUrl(media.url), controls="")
else:
a(href=getOrigPicUrl(media.url), target="_blank"):
img(src=getSmallPic(media.url), alt=entity.caption, loading="lazy")
if entity.caption.len > 0:
p(class="article-media-caption"): text entity.caption
of "TWEET":
let tweet = tweets.getOrDefault(
try: parseBiggestInt(entity.tweetId)
except ValueError: 0, nil)
if tweet != nil:
renderTweet(tweet, prefs, path)
else:
text ""
of "MARKDOWN":
var content = entity.markdown
if content.startsWith("```"):
let firstNl = content.find('\n')
if firstNl >= 0: content = content[firstNl + 1 .. ^1]
if content.endsWith("```"): content = content[0 .. ^4]
content = content.strip
buildHtml(pre()):
code(): text content
of "DIVIDER":
buildHtml(hr(class="article-divider"))
else:
text ""
proc wrapStyle(node: VNode; style: int): VNode =
result = node
if style.testBit(4): result = buildHtml(code()): result
if style.testBit(0): result = buildHtml(strong()): result
if style.testBit(1): result = buildHtml(em()): result
if style.testBit(2): result = buildHtml(del()): result
if style.testBit(3): result = buildHtml(underlined()): result
proc addContent(target: VNode; content: string; style = 0) =
var first = true
for line in content.split('\n'):
if not first:
target.add VNode(kind: VNodeKind.br)
first = false
var pos = 0
while pos < line.len:
let atPos = line.find('@', pos)
if atPos == -1:
target.add wrapStyle(text line[pos .. ^1], style)
break
if atPos > 0 and line[atPos - 1] in Letters + Digits + {'_'}:
target.add wrapStyle(text line[pos .. atPos], style)
pos = atPos + 1
continue
var j = atPos + 1
while j < line.len and j - atPos - 1 < 15 and
line[j] in Letters + Digits + {'_'}:
inc j
if j == atPos + 1:
target.add wrapStyle(text line[pos .. atPos], style)
pos = atPos + 1
continue
if atPos > pos:
target.add wrapStyle(text line[pos ..< atPos], style)
let username = line[atPos + 1 ..< j]
let link = a.newVNode()
link.setAttr("href", "/" & username)
link.add wrapStyle(text ("@" & username), style)
target.add link
pos = j
proc applyInlineStyles(target: VNode; runes: seq[Rune]; start, length: int;
styles: seq[ArticleStyle]) =
if styles.len == 0:
target.addContent($runes[start ..< start + length])
return
var
lastStyle = 0
lastStart = start
let endPos = start + length
for i in start ..< endPos:
var style = 0
for sr in styles:
let
sStart = sr.offset
sEnd = sStart + sr.length
if sStart <= i and sEnd > i:
case sr.style
of "Bold": style.setBit(0)
of "Italic": style.setBit(1)
of "Strikethrough": style.setBit(2)
of "Underline": style.setBit(3)
of "Code": style.setBit(4)
else: discard
if style != lastStyle:
if i > lastStart:
addContent(target, $runes[lastStart ..< i], lastStyle)
lastStyle = style
lastStart = i
if lastStart < endPos:
addContent(target, $runes[lastStart ..< endPos], lastStyle)
proc renderTextParagraph(paragraph: ArticleParagraph; article: Article): VNode =
let text = paragraph.text
result = case paragraph.kind
of "header-one": h1.newVNode()
of "header-two": h2.newVNode()
of "header-three": h3.newVNode()
of "ordered-list-item", "unordered-list-item": li.newVNode()
of "blockquote": VNode(kind: VNodeKind.blockquote)
of "code-block":
let pre = pre.newVNode()
let code = code.newVNode()
code.add text text
pre.add code
return pre
else: p.newVNode()
let
runes = text.toRunes
textLen = runes.len
var last = 0
for er in paragraph.entityRanges:
if er.offset > last:
applyInlineStyles(result, runes, last, er.offset - last,
paragraph.inlineStyles)
last = er.offset + er.length
var target = result
if er.key in article.entities:
let entity = article.entities[er.key]
if entity.kind == "LINK":
let parsed = parseUri(entity.url)
if parsed.scheme in ["http", "https"]:
target = a.newVNode()
if parsed.isTwitterUrl:
target.setAttr("href", parsed.path)
else:
target.setAttr("href", entity.url)
applyInlineStyles(target, runes, er.offset, er.length,
paragraph.inlineStyles)
if target != result:
result.add target
if last < textLen:
applyInlineStyles(result, runes, last, textLen - last,
paragraph.inlineStyles)
if paragraph.kind == "blockquote" and result.len > 0:
let lastChild = result[result.len - 1]
if lastChild.kind == VNodeKind.strong and lastChild.len > 0 and
lastChild[0].kind == VNodeKind.text:
lastChild.setAttr("class", "blockquote-attribution")
proc renderArticle*(article: Article; tweets: Table[int64, Tweet];
path: string; prefs: Prefs; tweetId=""): VNode =
let author = article.user
let main = buildHtml(article(class="article-body")):
h1(class="article-title"): text article.title
tdiv(class="article-author"):
tdiv(class="article-author-row"):
a(class="article-avatar", href=("/" & author.username)):
genImg(author.getUserPic("_bigger"), class=prefs.getAvatarClass)
tdiv(class="article-author-info"):
tdiv(class="article-author-name"):
linkUser(author, class="fullname")
verifiedIcon(author)
tdiv(class="article-author-meta"):
linkUser(author, class="username")
span(class="article-date-sep"): text " · "
a(class="article-date",
href=("/" & author.username & "/status/" & tweetId)):
text article.time.getShortTime
if not prefs.hideTweetStats:
renderStats(article.stats)
var listKind = ""
var list: VNode = nil
for paragraph in article.paragraphs:
let isListItem = paragraph.kind in [
"ordered-list-item", "unordered-list-item"]
if not isListItem and list != nil:
main.add list
list = nil
listKind = ""
if paragraph.kind == "atomic":
main.add renderAtomicParagraph(paragraph, article, tweets, path, prefs)
elif isListItem:
if paragraph.kind != listKind:
if list != nil:
main.add list
list = if paragraph.kind == "ordered-list-item": ol.newVNode()
else: ul.newVNode()
listKind = paragraph.kind
list.add renderTextParagraph(paragraph, article)
else:
main.add renderTextParagraph(paragraph, article)
if list != nil:
main.add list
buildHtml(tdiv(class="article-page")):
if article.coverImage.len > 0:
a(href=getOrigPicUrl(article.coverImage), target="_blank"):
img(class="article-cover", src=getSmallPic(article.coverImage), alt="")
main
renderToTop()

128
src/views/community.nim Normal file
View file

@ -0,0 +1,128 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, strformat, times
import karax/[karaxdsl, vdom]
import renderutils
import ".."/[types, utils, formatters]
proc renderCommunityTabs*(kind: QueryKind; community: Community): VNode =
let
path = &"/i/communities/{community.id}"
q = Query(kind: kind)
buildHtml(tdiv):
ul(class="tab"):
li(class=q.getTabClass(posts)):
a(href=path): text "Top"
li(class=q.getTabClass(replies)):
a(href=(path & "/latest")): text "Latest"
li(class=q.getTabClass(media)):
a(href=(path & "/media")): text "Media"
li(class=q.getTabClass(userList)):
a(href=(path & "/about")): text "About"
if community.hashtags.len > 0:
tdiv(class="community-tags"):
for tag in community.hashtags:
let bare = tag.strip(chars={'#'})
a(class="community-tag",
href=(&"/i/communities/{community.id}/hashtag/{bare}")):
text tag
proc renderMemberTabs*(community: Community; isModerators: bool): VNode =
let path = &"/i/communities/{community.id}"
buildHtml(ul(class="tab")):
li(class=(if not isModerators: "tab-item active" else: "tab-item")):
a(href=(path & "/members")): text "All"
li(class=(if isModerators: "tab-item active" else: "tab-item")):
a(href=(path & "/moderators")): text "Moderators"
proc renderHashtagHeader*(community: Community; tag: string): VNode =
buildHtml(tdiv(class="community-hashtag-header")):
h2(class="community-hashtag-title"): text "#" & tag
proc renderCommunityAbout*(community: Community; moderators: seq[User]): VNode =
buildHtml(tdiv(class="community-about")):
tdiv(class="community-info"):
h2: text "Community Info"
tdiv(class="community-info-item"):
icon "group"
if community.joinPolicy == "Open":
text "Anyone can join this Community."
else:
text "Membership is by approval only."
tdiv(class="community-info-item"):
icon "info"
text "All Communities are publicly visible."
tdiv(class="community-info-item"):
icon "calendar"
let
date = community.createdAt.format("MMMM d, yyyy")
creator = community.creator.username
span:
text &"Created {date} by "
a(href=(&"/{creator}")): text &"@{creator}"
if community.creator.verifiedType != none:
verifiedIcon(community.creator)
if community.rules.len > 0:
tdiv(class="community-rules"):
h2: text "Rules"
p(class="community-rules-intro"):
text "These are set and enforced by Community admins and are in addition to "
a(href="https://help.x.com/rules-and-policies/x-rules"): text "X's rules"
text "."
for i, rule in community.rules:
tdiv(class="community-rule"):
span(class="community-rule-number"): text $(i + 1)
tdiv(class="community-rule-content"):
strong: text rule.name
if rule.description.len > 0:
p: text rule.description
if moderators.len > 0:
tdiv(class="community-moderators"):
h2:
text "Moderators"
a(class="community-mods-link",
href=(&"/i/communities/{community.id}/moderators")):
text "See all"
for user in moderators:
tdiv(class="community-moderator"):
a(href=(&"/{user.username}")):
genImg(user.getUserPic("_bigger"), class="community-mod-avatar")
tdiv(class="community-mod-info"):
a(href=(&"/{user.username}"), class="community-mod-name"):
text user.fullname
if user.verifiedType != none:
verifiedIcon(user)
a(href=(&"/{user.username}"), class="community-mod-username"):
text &"@{user.username}"
proc renderCommunity*(body, nav: VNode; community: Community): VNode =
buildHtml(tdiv(class="timeline-container")):
if community.banner.len > 0:
tdiv(class="timeline-banner"):
a(href=getPicUrl(community.banner), target="_blank"):
genImg(community.banner)
tdiv(class="community-header"):
h1(class="community-name"):
a(href=(&"/i/communities/{community.id}")): text community.name
if community.category.len > 0:
span(class="community-category"): text community.category
if community.description.len > 0:
tdiv(class="community-description"):
text community.description
tdiv(class="community-stats"):
a(class="community-member-count",
href=(&"/i/communities/{community.id}/members")):
text insertSep($community.memberCount, ',')
text " Members"
nav
body

17
src/views/general.nim
View file

@ -50,8 +50,8 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
buildHtml(head):
link(rel="stylesheet", type="text/css", href="/css/style.css?v=35")
link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=5")
link(rel="stylesheet", type="text/css", href="/css/style.css?v=45")
link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=7")
if theme.len > 0:
link(rel="stylesheet", type="text/css", href=(&"/css/themes/{theme}.css"))
@ -72,7 +72,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
if prefs.hlsPlayback:
script(src="/js/hls.min.js", `defer`="")
script(src="/js/hlsPlayback.js", `defer`="")
script(src="/js/hlsPlayback.js?v=1", `defer`="")
if prefs.infiniteScroll:
script(src="/js/infiniteScroll.js", `defer`="")
@ -84,6 +84,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
text cfg.title
meta(name="viewport", content="width=device-width, initial-scale=1.0")
meta(name="referrer", content="same-origin")
meta(name="theme-color", content="#1F1F1F")
meta(property="og:type", content=ogType)
meta(property="og:title", content=(if ogTitle.len > 0: ogTitle else: titleText))
@ -96,6 +97,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
link(rel="preload", type="image/png", href=bannerUrl, `as`="image")
for url in images:
if url.len == 0: continue
let preloadUrl = if "400x400" in url: getPicUrl(url)
else: getSmallPic(url)
link(rel="preload", type="image/png", href=preloadUrl, `as`="image")
@ -117,13 +119,16 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
# this is last so images are also preloaded
# if this is done earlier, Chrome only preloads one image for some reason
link(rel="preload", type="font/woff2", `as`="font",
href="/fonts/fontello.woff2?61663884", crossorigin="anonymous")
href="/fonts/fontello.woff2?59696369", crossorigin="anonymous")
proc renderMain*(body: VNode; req: Request; cfg: Config; prefs=defaultPrefs;
titleText=""; desc=""; ogTitle=""; rss=""; video="";
images: seq[string] = @[]; banner=""): string =
images: seq[string] = @[]; banner="";
twitterLink=""): string =
let twitterLink = getTwitterLink(req.path, req.params)
let twitterLink =
if twitterLink.len > 0: twitterLink
else: getTwitterLink(req.path, req.params)
let node = buildHtml(html(lang="en")):
renderHead(prefs, cfg, req, titleText, desc, video, images, banner, ogTitle,

43
src/views/profile.nim
View file

@ -2,7 +2,7 @@
import strutils, strformat
import karax/[karaxdsl, vdom, vstyles]
import renderutils, search
import renderutils, search, timeline
import ".."/[types, utils, formatters]
proc renderStat(num: int; class: string; text=""): VNode =
@ -12,6 +12,13 @@ proc renderStat(num: int; class: string; text=""): VNode =
span(class="profile-stat-num"):
text insertSep($num, ',')
proc renderStatLink(num: int; class, href: string): VNode =
buildHtml(li(class=class)):
a(href=href):
span(class="profile-stat-header"): text capitalizeAscii(class)
span(class="profile-stat-num"):
text insertSep($num, ',')
proc renderUserCard*(user: User; prefs: Prefs; info: AccountInfo): VNode =
buildHtml(tdiv(class="profile-card")):
tdiv(class="profile-card-info"):
@ -65,8 +72,8 @@ proc renderUserCard*(user: User; prefs: Prefs; info: AccountInfo): VNode =
tdiv(class="profile-card-extra-links"):
ul(class="profile-statlist"):
renderStat(user.tweets, "posts", text="Tweets")
renderStat(user.following, "following")
renderStat(user.followers, "followers")
renderStatLink(user.following, "following", &"/{user.username}/following")
renderStatLink(user.followers, "followers", &"/{user.username}/followers")
renderStat(user.likes, "likes")
proc renderPhotoRail(profile: Profile): VNode =
@ -99,7 +106,7 @@ proc renderBanner(banner: string): VNode =
else:
a(href=getPicUrl(banner), target="_blank"): genImg(banner)
proc renderProtected(username: string): VNode =
proc renderProtected*(username: string): VNode =
buildHtml(tdiv(class="timeline-container")):
tdiv(class="timeline-header timeline-protected"):
h2: text "This account's tweets are protected."
@ -128,3 +135,31 @@ proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
renderProtected(profile.user.username)
else:
renderTweetSearch(profile.tweets, prefs, path, profile.pinned)
proc renderFollowTabs(user: User; activeTab: string): VNode =
buildHtml(ul(class="tab")):
for tab in ["Following", "Followers"]:
li(class=(if activeTab == tab: "tab-item active" else: "tab-item")):
a(href=(&"/{user.username}/{tab.toLowerAscii()}")): text tab
proc renderUserList*(user: User; results: Result[User]; prefs: Prefs;
path, activeTab: string): VNode =
# Check if we've loaded all results (for page 1)
var displayResults = results
if results.beginning:
let expectedCount = if activeTab == "Followers": user.followers else: user.following
if results.content.len >= expectedCount:
displayResults.bottom = ""
buildHtml(tdiv(class="profile-tabs")):
if not prefs.hideBanner:
tdiv(class="profile-banner"):
renderBanner(user.banner)
let sticky = if prefs.stickyProfile: " sticky" else: ""
tdiv(class=("profile-tab" & sticky)):
renderUserCard(user, prefs, AccountInfo())
tdiv(class="timeline-container"):
renderFollowTabs(user, activeTab)
renderTimelineUsers(displayResults, prefs, path)

8
src/views/renderutils.nim
View file

@ -7,18 +7,20 @@ const smallWebp* = "?name=small&format=webp"
const mediumWebp* = "?name=medium&format=webp"
proc getSmallPic*(url: string): string =
if url.len == 0: return
result = url
if "?" notin url and not url.endsWith("placeholder.png"):
result &= smallWebp
result = getPicUrl(result)
proc getMediumPic*(url: string): string =
if url.len == 0: return
result = url
if "?" notin url and not url.endsWith("placeholder.png"):
result &= mediumWebp
result = getPicUrl(result)
proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
proc icon*(icon: string; label=""; title=""; class=""; href=""): VNode =
var c = "icon-" & icon
if class.len > 0: c = &"{c} {class}"
buildHtml(tdiv(class="icon-container")):
@ -27,8 +29,8 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
else:
span(class=c, title=title)
if text.len > 0:
text " " & text
if label.len > 0:
text " " & label
template verifiedIcon*(user: User): untyped {.dirty.} =
if user.verifiedType != VerifiedType.none:

93
src/views/rss.nimf
View file

@ -12,7 +12,7 @@
#elif retweet.len > 0: prefix = &"RT by @{retweet}: "
#elif tweet.reply.len > 0: prefix = &"R to @{tweet.reply[0]}: "
#end if
#var text = stripHtml(tweet.text)
#var text = strutils.splitWhitespace(stripHtml(tweet.text)).join(" ")
##if unicode.runeLen(text) > 32:
## text = unicode.runeSubStr(text, 0, 32) & "..."
##end if
@ -34,6 +34,18 @@
# end case
# end if
#end if
#if result.len == 0 and tweet.articlePreview.isSome:
# let art = tweet.articlePreview.get()
# if art.title.len > 0:
# result = prefix & xmltree.escape(art.title)
# end if
#end if
#if result.len == 0 and tweet.card.isSome:
# let card = tweet.card.get()
# if card.kind notin {hidden, unknown} and card.title.len > 0:
# result = prefix & xmltree.escape(card.title)
# end if
#end if
#end proc
#
#proc getDescription(desc: string; cfg: Config): string =
@ -60,6 +72,72 @@ Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)}
#end case
#end proc
#
#proc renderRssCard(card: Card; prefs: Prefs; urlPrefix: string): string =
#let cardLink = if card.url.startsWith("/"): urlPrefix & card.url else: replaceUrls(card.url, prefs)
#let title = xmltree.escape(card.title)
<hr/>
<b>Link</b><br>
#if cardLink.len > 0:
<a href="${xmltree.escape(cardLink)}">
#end if
#if card.image.len > 0:
<img src="${urlPrefix}${getPicUrl(card.image)}" style="max-width:250px;" />
#end if
#if title.len > 0:
# if card.image.len > 0:
<br>
# end if
<b>${title}</b>
#end if
#if cardLink.len > 0:
</a>
#end if
#if card.text.len > 0:
<p>${xmltree.escape(card.text)}</p>
#end if
#if cardLink.len > 0:
# let destText = if card.dest.len > 0: xmltree.escape(card.dest) else: xmltree.escape(cardLink)
<small><a href="${xmltree.escape(cardLink)}">${destText}</a></small>
#elif card.dest.len > 0:
<small>${xmltree.escape(card.dest)}</small>
#end if
#end proc
#
#proc renderRssArticle(article: ArticlePreview; urlPrefix: string): string =
#let link = urlPrefix & "/i/article/" & $article.tweetId
<hr/>
<b>Article</b><br>
<a href="${link}">
#if article.coverImage.len > 0:
<img src="${urlPrefix}${getPicUrl(article.coverImage)}" style="max-width:250px;" />
#end if
#if article.title.len > 0:
# if article.coverImage.len > 0:
<br>
# end if
<b>${xmltree.escape(article.title)}</b>
#end if
</a>
#if article.previewText.len > 0:
<p>${xmltree.escape(article.previewText)}</p>
#end if
#end proc
#
#proc renderRssPoll(poll: Poll): string =
<hr/>
<b>Poll</b>
<p>
#for i in 0 ..< poll.options.len:
# let perc = if poll.votes > 0: poll.values[i] / poll.votes * 100 else: 0.0
# let pct = (&"{perc:.0f}").strip(chars={'.'})
# let line = pct & "% — " & xmltree.escape(poll.options[i])
${line}<br>
#end for
#let votesStr = insertSep($poll.votes, ',')
<i>${votesStr} votes • ${xmltree.escape(poll.status)}</i>
</p>
#end proc
#
#proc getTweetsWithPinned(profile: Profile): seq[Tweets] =
#result = profile.tweets.content
#if profile.pinned.isSome and result.len > 0:
@ -82,16 +160,21 @@ Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)}
#let tweet = tweet.retweet.get(tweet)
#let urlPrefix = getUrlPrefix(cfg)
#let text = replaceUrls(tweet.text, prefs, absolute=urlPrefix)
#if text.len > 0:
<p>${text.replace("\n", "<br>\n")}</p>
#end if
#if tweet.media.len > 0:
# for media in tweet.media:
${renderRssMedia(media, tweet, urlPrefix)}
# end for
#elif tweet.card.isSome:
# let card = tweet.card.get()
# if card.image.len > 0:
<img src="${urlPrefix}${getPicUrl(card.image)}" style="max-width:250px;" />
#elif tweet.card.isSome and tweet.card.get().kind notin {hidden, unknown}:
${renderRssCard(tweet.card.get(), prefs, urlPrefix)}
#end if
#if tweet.articlePreview.isSome:
${renderRssArticle(tweet.articlePreview.get(), urlPrefix)}
#end if
#if tweet.poll.isSome:
${renderRssPoll(tweet.poll.get())}
#end if
#if tweet.note.len > 0 and not prefs.hideCommunityNotes:
<p><b>Community note:</b> ${replaceUrls(tweet.note, prefs, absolute=urlPrefix)}</p>

86
src/views/space.nim Normal file
View file

@ -0,0 +1,86 @@
# SPDX-License-Identifier: AGPL-3.0-only
import strutils, times
import karax/[karaxdsl, vdom]
import renderutils
import ".."/[types, utils, formatters]
proc renderParticipant(p: SpaceParticipant; role: string): VNode =
buildHtml(tdiv(class="space-participant")):
a(href=("/" & p.username)):
genImg(p.avatarUrl.replace("_normal", "_bigger"))
tdiv(class="participant-info"):
tdiv(class="participant-name"):
strong: text p.displayName
if p.isVerified:
tdiv(class="verified-icon blue"):
icon "circle", class="verified-icon-circle", title="Verified account"
icon "ok", class="verified-icon-check", title="Verified account"
if role.len > 0:
span(class="host-badge"): text role
span(class="participant-username"): text "@" & p.username
proc renderSpace*(sp: AudioSpace; prefs: Prefs; path: string): VNode =
let
isLive = sp.state == "RUNNING"
source = if prefs.proxyVideos and sp.m3u8Url.startsWith("http"):
getVidUrl(sp.m3u8Url) else: sp.m3u8Url
stateText =
if isLive: "LIVE"
elif sp.endTime.year > 1: "Ended " & sp.endTime.format("MMM d, YYYY")
elif sp.state.len > 0: sp.state
else: "Ended"
durationMs =
if sp.startTime.year > 1 and sp.endTime.year > 1:
int((sp.endTime - sp.startTime).inMilliseconds)
else: 0
duration = if durationMs > 0: getDuration(durationMs) else: ""
totalListeners =
if sp.totalReplayWatched > 0: sp.totalReplayWatched
else: sp.totalLiveListeners
buildHtml(tdiv(class="space-page")):
tdiv(class="space-panel"):
tdiv(class="space-player"):
if sp.m3u8Url.len > 0 and prefs.hlsPlayback:
audio(data-url=source, data-autoload="false")
verbatim "<div class=\"video-overlay\" onclick=\"playAudio(this)\">"
tdiv(class="overlay-circle"): span(class="overlay-triangle")
if isLive:
tdiv(class="space-live"): text "LIVE"
elif duration.len > 0:
tdiv(class="overlay-duration"): text duration
verbatim "</div>"
elif sp.m3u8Url.len > 0:
tdiv(class="video-overlay"):
buttonReferer "/enablehls", "Enable hls playback", path
if isLive:
tdiv(class="space-live"): text "LIVE"
elif duration.len > 0:
tdiv(class="overlay-duration"): text duration
elif sp.availableForReplay:
tdiv(class="video-overlay"):
p: text "Audio stream unavailable"
else:
tdiv(class="video-overlay"):
p: text "Replay is not available"
tdiv(class="space-info"):
tdiv(class="space-header"):
h2(class="space-title"): text sp.title
tdiv(class="space-meta"):
if totalListeners > 0:
span(class="listener-count"): text insertSep($totalListeners, ',') & " listeners"
if isLive:
span(class="space-live"): text stateText
else:
span(class="space-state"): text stateText
if sp.admins.len > 0 or sp.speakers.len > 0:
tdiv(class="space-participants"):
for admin in sp.admins:
let role = if admin.username == sp.creator.username: "Host"
else: "Co-host"
renderParticipant(admin, role)
for speaker in sp.speakers:
renderParticipant(speaker, "")

3
src/views/timeline.nim
View file

@ -154,7 +154,8 @@ proc renderTimelineTweets*(results: Timeline; prefs: Prefs; path: string;
else: renderThread(thread, prefs, path, bigThumb)
else:
for thread in filtered:
if thread.len == 1: renderTweet(thread[0], prefs, path)
if thread.len == 1:
renderTweet(thread[0], prefs, path)
else: renderThread(thread, prefs, path)
var cursor = getSearchMaxId(results, path)

76
src/views/tweet.nim
View file

@ -9,17 +9,36 @@ import general
const doctype = "<!DOCTYPE html>\n"
proc renderMiniAvatar(user: User; prefs: Prefs): VNode =
proc renderMiniAvatar*(user: User; prefs: Prefs): VNode =
genImg(user.getUserPic("_mini"), class=(prefs.getAvatarClass & " mini"))
proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs): VNode =
proc renderArticleCard(preview: ArticlePreview; prefs: Prefs): VNode =
let url = "/i/article/" & $preview.tweetId
buildHtml(tdiv(class="article-card card large")):
a(class="card-container", href=url):
if preview.coverImage.len > 0:
tdiv(class="card-image-container"):
tdiv(class="card-image"):
genImg(preview.coverImage)
span(class="article-card-badge"): text "Article"
tdiv(class="card-content-container"):
tdiv(class="card-content"):
h2(class="card-title"): text preview.title
if preview.previewText.len > 0:
p(class="card-description"): text preview.previewText
proc renderHeader(tweet: Tweet; retweet: string; pinned: bool; prefs: Prefs;
path = ""): VNode =
buildHtml(tdiv):
if pinned:
let pinnedLabel =
if "/i/communities/" in path: "Pinned by Community mods"
else: "Pinned Tweet"
tdiv(class="pinned"):
span: icon "pin", "Pinned Tweet"
span: icon("pin", pinnedLabel)
elif retweet.len > 0:
tdiv(class="retweet-header"):
span: icon "retweet", retweet & " retweeted"
span: icon("retweet", retweet & " retweeted")
tdiv(class="tweet-header"):
a(class="tweet-avatar", href=("/" & tweet.user.username)):
@ -66,7 +85,7 @@ proc renderVideoDisabled(playbackType: VideoType; path=""): VNode =
buildHtml(tdiv(class="video-overlay")):
case playbackType
of mp4:
p: text "mp4 playback disabled in preferences"
buttonReferer "/enablemp4", "Enable mp4 playback", path
of m3u8, vmap:
buttonReferer "/enablehls", "Enable hls playback", path
@ -78,6 +97,12 @@ proc renderVideoUnavailable(video: Video): VNode =
else:
p: text "This media is unavailable"
proc getVideoDownloadUrl(videoData: Video): string =
let mp4Vars = videoData.variants.filterIt(it.contentType == mp4)
if mp4Vars.len == 0: return ""
let best = mp4Vars.sortedByIt(it.bitrate)[^1].url
if best.startsWith("http"): getVidUrl(best) else: best
proc renderVideoAttachment(videoData: Video; prefs: Prefs; path=""; bigThumb=false): VNode =
let
playbackType = if not prefs.proxyVideos and videoData.hasMp4Url: mp4
@ -108,6 +133,11 @@ proc renderVideoAttachment(videoData: Video; prefs: Prefs; path=""; bigThumb=fal
if videoData.durationMs > 0:
tdiv(class="overlay-duration"): text getDuration(videoData)
verbatim "</div>"
if videoData.available:
let dlUrl = getVideoDownloadUrl(videoData)
if dlUrl.len > 0:
a(class="video-download", href=dlUrl, download="",
title="Download video"): icon "download-alt"
proc renderVideo*(video: Video; prefs: Prefs; path: string; bigThumb=false): VNode =
let hasCardContent = video.description.len > 0 or video.title.len > 0
@ -121,13 +151,13 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string; bigThumb=false): VNo
if video.description.len > 0:
p(class="card-description"): text video.description
proc renderGifAttachment(gif: Gif; prefs: Prefs): VNode =
proc renderGifAttachment(gif: Gif; prefs: Prefs; path=""): VNode =
let thumb = getSmallPic(gif.thumb)
buildHtml(tdiv(class="attachment")):
if not prefs.mp4Playback:
img(src=thumb, loading="lazy")
renderVideoDisabled(mp4)
renderVideoDisabled(mp4, path)
elif prefs.autoplayGifs:
video(class="gif", poster=thumb, autoplay="", muted="", loop=""):
source(src=getPicUrl(gif.url), `type`="video/mp4")
@ -137,9 +167,9 @@ proc renderGifAttachment(gif: Gif; prefs: Prefs): VNode =
if gif.altText.len > 0:
renderAltText(gif.altText)
proc renderGif(gif: Gif; prefs: Prefs): VNode =
proc renderGif(gif: Gif; prefs: Prefs; path=""): VNode =
buildHtml(tdiv(class="attachments media-gif")):
renderGifAttachment(gif, prefs)
renderGifAttachment(gif, prefs, path)
proc renderMedia(media: seq[Media]; prefs: Prefs; path: string; bigThumb=false): VNode =
if media.len == 0:
@ -150,7 +180,7 @@ proc renderMedia(media: seq[Media]; prefs: Prefs; path: string; bigThumb=false):
if item.kind == videoMedia:
return renderVideo(item.video, prefs, path, bigThumb)
if item.kind == gifMedia:
return renderGif(item.gif, prefs)
return renderGif(item.gif, prefs, path)
let
groups = if media.len < 3: @[media]
@ -169,7 +199,7 @@ proc renderMedia(media: seq[Media]; prefs: Prefs; path: string; bigThumb=false):
of videoMedia:
renderVideoAttachment(mediaItem.video, prefs, path, bigThumb)
of gifMedia:
renderGifAttachment(mediaItem.gif, prefs)
renderGifAttachment(mediaItem.gif, prefs, path)
proc renderPoll(poll: Poll): VNode =
buildHtml(tdiv(class="poll")):
@ -225,7 +255,7 @@ func formatStat(stat: int): string =
if stat > 0: insertSep($stat, ',')
else: ""
proc renderStats(stats: TweetStats): VNode =
proc renderStats*(stats: TweetStats): VNode =
buildHtml(tdiv(class="tweet-stats")):
span(class="tweet-stat"): icon "comment", formatStat(stats.replies)
span(class="tweet-stat"): icon "retweet", formatStat(stats.retweets)
@ -239,8 +269,9 @@ proc renderReply(tweet: Tweet): VNode =
if i > 0: text " "
a(href=("/" & u)): text "@" & u
proc renderAttribution(user: User; prefs: Prefs): VNode =
buildHtml(a(class="attribution", href=("/" & user.username))):
proc renderAttribution(user: User; prefs: Prefs; link = ""): VNode =
let href = if link.len > 0: link else: "/" & user.username
buildHtml(a(class="attribution", href=href)):
renderMiniAvatar(user, prefs)
strong: text user.fullname
verifiedIcon(user)
@ -307,6 +338,9 @@ proc renderQuote(quote: Tweet; prefs: Prefs; path: string): VNode =
if quote.media.len > 0:
renderQuoteMedia(quote, prefs, path)
if quote.articlePreview.isSome:
renderArticleCard(quote.articlePreview.get(), prefs)
if quote.note.len > 0 and not prefs.hideCommunityNotes:
renderCommunityNote(quote.note, prefs)
@ -322,10 +356,10 @@ proc renderDisclosures*(tweet: Tweet): VNode =
buildHtml(tdiv(class="disclosures")):
if tweet.isAI:
span(data-disclosure="ai"):
icon "attention", "Made with AI"
icon "attention-circled", "Made with AI"
if tweet.isAd:
span(data-disclosure="ad"):
icon "attention", "Paid partnership (ad)"
icon "attention-circled", "Paid partnership (ad)"
proc renderLocation*(tweet: Tweet): string =
let (place, url) = tweet.getLocation()
@ -339,7 +373,8 @@ proc renderLocation*(tweet: Tweet): string =
return $node
proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
last=false; mainTweet=false; afterTweet=false; bigThumb=false): VNode =
last=false; mainTweet=false; afterTweet=false;
bigThumb=false): VNode =
var divClass = class
if index == -1 or last:
divClass = "thread-last " & class
@ -372,7 +407,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
a(class="tweet-link", href=getLink(tweet))
tdiv(class="tweet-body"):
renderHeader(tweet, retweet, pinned, prefs)
renderHeader(tweet, retweet, pinned, prefs, path)
if not afterTweet and index == 0 and tweet.reply.len > 0 and
(tweet.reply.len > 1 or tweet.reply[0] != tweet.user.username or pinned):
@ -386,11 +421,14 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
verbatim replaceUrls(tweet.text, prefs) & renderLocation(tweet)
if tweet.attribution.isSome:
renderAttribution(tweet.attribution.get(), prefs)
renderAttribution(tweet.attribution.get(), prefs, tweet.attributionLink)
if tweet.card.isSome and tweet.card.get().kind != hidden:
renderCard(tweet.card.get(), prefs, path)
if tweet.articlePreview.isSome:
renderArticleCard(tweet.articlePreview.get(), prefs)
if tweet.media.len > 0:
renderMedia(tweet.media, prefs, path, bigThumb)

76
tests/test_about_account.py Normal file
View file

@ -0,0 +1,76 @@
from base import BaseTestCase, Profile
from parameterized import parameterized
class AboutAccount(object):
header = '.about-account-header'
name = '.about-account-name'
body = '.about-account-body'
row = '.about-account-row'
label = '.about-account-label'
value = '.about-account-value'
# (username, expected_labels)
# Each label is checked for presence in the page text
about_data = [
['jack', ['Date joined', 'Account based in', 'Connected via']],
['NASA', ['Date joined']],
['elonmusk', ['Date joined']],
]
about_verified = [
['jack', 'Verified', 'Since '],
]
about_affiliate = [
['jack', 'An affiliate of', 'Square'],
['elonmusk', 'An affiliate of', 'X'],
]
class AboutAccountTest(BaseTestCase):
@parameterized.expand(about_data)
def test_about_page_has_labels(self, username, expected_labels):
"""About page shows expected info labels"""
self.open_nitter(f'{username}/about')
self.assert_element_visible(AboutAccount.header)
self.assert_element_visible(AboutAccount.body)
for label in expected_labels:
self.assert_text(label, AboutAccount.body)
@parameterized.expand(about_verified)
def test_about_verified(self, username, label, value_prefix):
"""About page shows verification info for verified accounts"""
self.open_nitter(f'{username}/about')
self.assert_text(label, AboutAccount.body)
self.assert_text(value_prefix, AboutAccount.body)
@parameterized.expand(about_affiliate)
def test_about_affiliate(self, username, label, affiliate):
"""About page shows affiliate info"""
self.open_nitter(f'{username}/about')
self.assert_text(label, AboutAccount.body)
self.assert_text(f'@{affiliate}', AboutAccount.body)
def test_about_page_title(self):
"""Title contains account name"""
self.open_nitter('jack/about')
self.assert_text('jack', AboutAccount.name)
def test_about_join_date(self):
"""About page always shows join date"""
self.open_nitter('jack/about')
self.assert_text('Date joined', AboutAccount.body)
self.assert_text('March 2006', AboutAccount.body)
def test_about_invalid_user(self):
"""About page for non-existent user shows error"""
self.open_nitter('thisprofiledoesntexist/about')
self.assert_text('User "thisprofiledoesntexist" not found')
def test_joindate_links_to_about(self):
"""Join date on profile page links to about page"""
self.open_nitter('jack')
link = self.find_element(Profile.joinDate + ' a')
self.assertIn('/jack/about', link.get_attribute('href'))

325
tests/test_article.py Normal file
View file

@ -0,0 +1,325 @@
from base import BaseTestCase
from parameterized import parameterized
class ArticleSelectors:
page = '.article-page'
cover = '.article-cover'
body = '.article-body'
title = '.article-title'
author = '.article-author'
fullname = '.article-author .fullname'
username = '.article-author .username'
date = '.article-author .article-date'
avatar = '.article-author img.avatar'
verified = '.article-author .verified-icon'
media = '.article-media'
caption = '.article-media-caption'
divider = '.article-divider'
articles = [
['2064166507438059759',
'1s，秒杀一切，开源一个 X 文章发布 Skill【重磅升级】',
'punk2898', 'Punk'],
['2064689664213041529',
'SpaceX Thesis & Valuation Memorandum',
'Dialectic_Group', 'Dialectic'],
['2064691088636424322',
'Consciousness and AI: The Problem of Inner Experience',
'CosmicOrFun', 'Cosmic Orphan'],
['2064755789391110154',
'DeFi Markets Update 2026-06-10',
'SteakhouseFi', 'Steakhouse Financial'],
['2064755231901319527',
'The machine economy has a killswitch and somebody just pulled it.',
'1914ad', 'Justin Bechler HMP-028'],
['2062858677149675788',
'Yakshinis',
'CosmicOrFun', 'Cosmic Orphan'],
]
articles_with_media = [
['2064166507438059759', 6],
['2064689664213041529', 11],
['2064755789391110154', 5],
]
articles_with_dividers = [
['2064166507438059759', 1],
['2064689664213041529', 6],
]
class ArticleBasicTest(BaseTestCase):
@parameterized.expand(articles)
def test_article_loads(self, tweet_id, title, username, fullname):
self.open_nitter(f'i/article/{tweet_id}')
self.assert_element_visible(ArticleSelectors.page)
self.assert_element_visible(ArticleSelectors.body)
self.assert_text(title, ArticleSelectors.title)
@parameterized.expand(articles)
def test_article_author(self, tweet_id, title, username, fullname):
self.open_nitter(f'i/article/{tweet_id}')
self.assert_element_visible(ArticleSelectors.author)
self.assert_text(f'@{username}', ArticleSelectors.username)
@parameterized.expand(articles)
def test_article_has_cover(self, tweet_id, title, username, fullname):
self.open_nitter(f'i/article/{tweet_id}')
self.assert_element_visible(ArticleSelectors.cover)
src = self.get_attribute(ArticleSelectors.cover, 'src')
self.assertIn('/pic/', src)
@parameterized.expand(articles)
def test_article_has_date(self, tweet_id, title, username, fullname):
self.open_nitter(f'i/article/{tweet_id}')
date_text = self.get_text(ArticleSelectors.date)
self.assertTrue(len(date_text) > 3)
@parameterized.expand(articles)
def test_article_author_avatar(self, tweet_id, title, username, fullname):
self.open_nitter(f'i/article/{tweet_id}')
self.assert_element_visible(ArticleSelectors.avatar)
src = self.get_attribute(ArticleSelectors.avatar, 'src')
self.assertIn('/pic/', src)
self.assertGreater(len(src), len('/pic/'))
@parameterized.expand(articles)
def test_article_author_verified(self, tweet_id, title, username, fullname):
self.open_nitter(f'i/article/{tweet_id}')
self.assert_element_visible(ArticleSelectors.verified)
def test_article_author_verified_business(self):
self.open_nitter('i/article/2064755789391110154')
self.assert_element_visible('.article-author .verified-icon.business')
class ArticleContentTest(BaseTestCase):
def test_article_has_paragraphs(self):
self.open_nitter('i/article/2064689664213041529')
paragraphs = self.find_elements('.article-body p')
self.assertGreater(len(paragraphs), 10)
def test_article_has_headers(self):
self.open_nitter('i/article/2064689664213041529')
headers = self.find_elements('.article-body h1, .article-body h2')
self.assertGreater(len(headers), 5)
def test_article_has_bold_text(self):
self.open_nitter('i/article/2064166507438059759')
bold = self.find_elements('.article-body strong')
self.assertGreater(len(bold), 0)
def test_article_has_italic_text(self):
self.open_nitter('i/article/2064166507438059759')
italic = self.find_elements('.article-body em')
self.assertGreater(len(italic), 0)
def test_article_has_blockquotes(self):
self.open_nitter('i/article/2064166507438059759')
self.assert_element_visible('.article-body blockquote')
def test_article_has_lists(self):
self.open_nitter('i/article/2064166507438059759')
self.assert_element_visible('.article-body ul')
def test_article_has_emoji_text(self):
self.open_nitter('i/article/2064166507438059759')
body = self.get_text(ArticleSelectors.body)
self.assertTrue(any(ord(c) > 0x1F000 for c in body))
def test_article_has_links(self):
self.open_nitter('i/article/2064691088636424322')
links = self.find_elements('.article-body a[href]')
self.assertGreater(len(links), 0)
def test_article_twitter_links_localized(self):
self.open_nitter('i/article/2064755789391110154')
links = self.find_elements('.article-body a[href^="https://x.com"]')
self.assertEqual(len(links), 0, 'x.com links should be converted to local paths')
@parameterized.expand(articles_with_media)
def test_article_media_count(self, tweet_id, expected_count):
self.open_nitter(f'i/article/{tweet_id}')
media = self.find_elements(ArticleSelectors.media)
self.assertEqual(len(media), expected_count)
@parameterized.expand(articles_with_dividers)
def test_article_divider_count(self, tweet_id, expected_count):
self.open_nitter(f'i/article/{tweet_id}')
dividers = self.find_elements(ArticleSelectors.divider)
self.assertEqual(len(dividers), expected_count)
class ArticleMediaTest(BaseTestCase):
def test_media_images_proxied(self):
self.open_nitter('i/article/2064689664213041529')
self.assert_element_visible(ArticleSelectors.media)
img = self.find_element(f'{ArticleSelectors.media} img')
src = img.get_attribute('src')
self.assertIn('/pic/', src)
self.assertFalse(src.startswith('https://pbs.twimg.com'))
def test_cover_image_proxied(self):
self.open_nitter('i/article/2064689664213041529')
self.assert_element_visible(ArticleSelectors.cover)
src = self.get_attribute(ArticleSelectors.cover, 'src')
self.assertIn('/pic/', src)
self.assertFalse(src.startswith('https://pbs.twimg.com'))
def test_embedded_tweet(self):
self.open_nitter('i/article/2064755789391110154')
self.assert_element_visible('.article-body .timeline-item')
def test_multiple_embedded_tweets(self):
self.open_nitter('i/article/2064755231901319527')
tweets = self.find_elements('.article-body .timeline-item')
self.assertGreaterEqual(len(tweets), 3)
def test_media_caption_displayed(self):
self.open_nitter('i/article/2064689664213041529')
self.assert_element_visible(ArticleSelectors.caption)
captions = self.find_elements(ArticleSelectors.caption)
self.assertGreaterEqual(len(captions), 5)
def test_media_caption_text(self):
self.open_nitter('i/article/2064689664213041529')
self.assert_text_visible('FIGURE 1', ArticleSelectors.caption)
def test_media_caption_alt_attribute(self):
self.open_nitter('i/article/2064689664213041529')
img = self.find_element(f'{ArticleSelectors.media} img')
alt = img.get_attribute('alt')
self.assertGreater(len(alt), 0)
def test_no_caption_when_absent(self):
self.open_nitter('i/article/2062858677149675788')
captions = self.find_elements(ArticleSelectors.caption)
self.assertEqual(len(captions), 0)
class ArticleMentionTest(BaseTestCase):
def test_mention_linkified(self):
self.open_nitter('i/article/2064755231901319527')
link = self.find_element('.article-body a[href="/ZachXBT"]')
self.assertEqual(link.text, '@ZachXBT')
def test_multiple_mentions_linkified(self):
self.open_nitter('i/article/2064755231901319527')
links = self.find_elements('.article-body a[href^="/"]')
mention_hrefs = [l.get_attribute('href') for l in links
if l.text.startswith('@')]
usernames = [h.split('/')[-1] for h in mention_hrefs]
self.assertIn('ZachXBT', usernames)
self.assertIn('River', usernames)
def test_mention_in_different_article(self):
self.open_nitter('i/article/2064689664213041529')
link = self.find_element('.article-body a[href="/FutureJurvetson"]')
self.assertEqual(link.text, '@FutureJurvetson')
def test_no_spurious_whitespace_in_styled_paragraph(self):
"""Styled paragraphs should not have extra whitespace from VNode serialization."""
self.open_nitter('i/article/2064166507438059759')
source = self.get_page_source()
self.assertNotIn('white-space: pre-wrap', source)
self.assertNotIn('white-space:pre-wrap', source)
class ArticleCardTest(BaseTestCase):
@parameterized.expand(articles)
def test_status_page_shows_article_card(self, tweet_id, title, username, fullname):
self.open_nitter(f'{username}/status/{tweet_id}')
self.assert_element_visible('.article-card')
self.assert_text(title, '.article-card .card-title')
def test_article_card_has_cover_image(self):
self.open_nitter('Dialectic_Group/status/2064689664213041529')
self.assert_element_visible('.article-card .card-image img')
src = self.get_attribute('.article-card .card-image img', 'src')
self.assertIn('/pic/', src)
def test_article_card_has_badge(self):
self.open_nitter('Dialectic_Group/status/2064689664213041529')
self.assert_element_visible('.article-card-badge')
self.assert_text('Article', '.article-card-badge')
def test_article_card_has_preview_text(self):
self.open_nitter('CosmicOrFun/status/2064691088636424322')
self.assert_element_visible('.article-card .card-description')
def test_article_card_links_to_article(self):
self.open_nitter('punk2898/status/2064166507438059759')
href = self.get_attribute('.article-card .card-container', 'href')
self.assertIn('/article/', href)
def test_article_url_stripped_from_tweet_text(self):
self.open_nitter('punk2898/status/2064166507438059759')
self.assert_element_visible('.article-card')
source = self.get_page_source()
# Main tweet text should not contain article URL
import re
main = re.search(r'id="m".*?tweet-content[^>]*>(.*?)</div>', source, re.DOTALL)
self.assertIsNotNone(main)
self.assertNotIn('/article/', main.group(1))
class ArticleQuotedCardTest(BaseTestCase):
"""Article cards inside quoted tweets (1914ad quoting own article)."""
quoted_tweet = '1914ad/status/2064789532071891085'
quoted_article_id = '2063677483548102688'
def test_quoted_card_visible(self):
self.open_nitter(self.quoted_tweet)
self.assert_element_visible('.quote .article-card')
def test_quoted_card_has_title(self):
self.open_nitter(self.quoted_tweet)
self.assert_text('David Bailey Already Won', '.quote .article-card .card-title')
def test_quoted_card_has_badge(self):
self.open_nitter(self.quoted_tweet)
self.assert_element_visible('.quote .article-card-badge')
self.assert_text('Article', '.quote .article-card-badge')
def test_quoted_card_has_cover_image(self):
self.open_nitter(self.quoted_tweet)
self.assert_element_visible('.quote .article-card .card-image img')
src = self.get_attribute('.quote .article-card .card-image img', 'src')
self.assertIn('/pic/', src)
def test_quoted_card_has_description(self):
self.open_nitter(self.quoted_tweet)
self.assert_element_visible('.quote .article-card .card-description')
def test_quoted_card_links_to_article(self):
self.open_nitter(self.quoted_tweet)
href = self.get_attribute('.quote .article-card .card-container', 'href')
self.assertIn(f'/article/{self.quoted_article_id}', href)
class ArticleRoutingTest(BaseTestCase):
def test_username_article_route_redirects(self):
self.open_nitter('punk2898/article/2064166507438059759')
self.assert_element_visible(ArticleSelectors.page)
self.assert_text('1s', ArticleSelectors.title)
def test_status_article_route_redirects(self):
self.open_nitter('punk2898/status/2064166507438059759/article')
self.assert_element_visible(ArticleSelectors.page)
self.assert_text('1s', ArticleSelectors.title)
def test_invalid_id_returns_404(self):
self.open_nitter('i/article/notanumber')
self.assert_element_not_visible(ArticleSelectors.page)
def test_nonexistent_article(self):
self.open_nitter('i/article/1')
self.assert_element_visible('.error-panel')

29
tests/test_card.py
View file

@ -11,18 +11,18 @@ card = [
['voidtarget/status/1094632512926605312',
'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too)',
'Basic OBS Studio plugin, written in nim, supporting C++ (C fine too) - obsplugin.nim',
'gist.github.com', True]
'gist.github.com', True],
['NASA/status/2061872347477418301',
'Nancy Grace Roman Space Telescope - NASA Science',
'The Nancy Grace Roman Space Telescope will settle essential questions in the areas of dark energy, exoplanets, and astrophysics.',
'science.nasa.gov', True]
]
no_thumb = [
['FluentAI/status/1116417904831029248',
'LinkedIn',
'This link will take you to a page that’s not on LinkedIn',
'lnkd.in'],
['Thom_Wolf/status/1122466524860702729',
'GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in',
'',
'GitHub - facebookresearch/XLM: PyTorch original implementation of Cross-lingual Language Model',
'PyTorch original implementation of Cross-lingual Language Model Pretraining.',
'github.com'],
['brent_p/status/1088857328680488961',
@ -37,14 +37,9 @@ no_thumb = [
]
playable = [
['nim_lang/status/1118234460904919042',
'Nim development blog 2019-03',
'Arne (aka Krux02)* debugging: * improved nim-gdb, $ works, framefilter * alias for --debugger:native: -g* bugs: * forwarding of .pure. * sizeof union* fe...',
'youtube.com'],
['nim_lang/status/1121090879823986688',
'Nim - First natively compiled language w/ hot code-reloading at...',
'#nim #c++ #ACCUConfNim is a statically typed systems and applications programming language which offers perhaps some of the most powerful metaprogramming cap...',
['NASA/status/2047048645845897398',
'NASA\'s Artemis II News Conference with Moon Astronauts',
'Live from NASA\'s Johnson Space Center in Houston',
'youtube.com']
]
@ -72,7 +67,7 @@ class CardTest(BaseTestCase):
if len(description) > 0:
self.assert_text(description, c.description)
@parameterized.expand(playable)
@parameterized.expand(playable, skip_on_empty=True)
def test_card_playable(self, tweet, title, description, destination):
self.open_nitter(tweet)
c = Card(Conversation.main + " ")

211
tests/test_community.py Normal file
View file

@ -0,0 +1,211 @@
from base import BaseTestCase
from parameterized import parameterized
COMMUNITY_ID = '1493446837214187523'
COMMUNITY_PATH = f'i/communities/{COMMUNITY_ID}'
class CommunityTest(BaseTestCase):
def test_top_page_loads(self):
self.open_nitter(COMMUNITY_PATH)
self.assert_element_visible('.community-header')
self.assert_text('Build in Public', '.community-name')
def test_banner_visible(self):
self.open_nitter(COMMUNITY_PATH)
self.assert_element_visible('.timeline-banner img')
def test_member_count(self):
self.open_nitter(COMMUNITY_PATH)
self.assert_element_visible('.community-member-count')
self.assert_text('Members', '.community-member-count')
def test_description_visible(self):
self.open_nitter(COMMUNITY_PATH)
self.assert_element_visible('.community-description')
def test_tabs_present(self):
self.open_nitter(COMMUNITY_PATH)
tabs = self.find_elements('.tab a')
labels = [t.text for t in tabs]
self.assertEqual(labels, ['Top', 'Latest', 'Media', 'About'])
def test_top_tab_active(self):
self.open_nitter(COMMUNITY_PATH)
self.assert_element_visible('.tab .active a[href$="/' + COMMUNITY_ID + '"]')
def test_top_has_tweets(self):
self.open_nitter(COMMUNITY_PATH)
self.assert_element_visible('.timeline-item .tweet-body')
def test_top_has_pagination(self):
self.open_nitter(COMMUNITY_PATH)
self.assert_element_visible('.show-more')
self.assert_text('Load more', '.show-more')
def test_latest_has_tweets(self):
self.open_nitter(f'{COMMUNITY_PATH}/latest')
self.assert_element_visible('.timeline-item .tweet-body')
def test_latest_tab_active(self):
self.open_nitter(f'{COMMUNITY_PATH}/latest')
self.assert_element_visible('.tab .active a[href$="/latest"]')
def test_media_has_tweets(self):
self.open_nitter(f'{COMMUNITY_PATH}/media')
self.assert_element_visible('.timeline-item .tweet-body')
def test_media_tab_active(self):
self.open_nitter(f'{COMMUNITY_PATH}/media')
self.assert_element_visible('.tab .active a[href$="/media"]')
def test_about_page(self):
self.open_nitter(f'{COMMUNITY_PATH}/about')
self.assert_element_visible('.community-about')
self.assert_text('Community Info', '.community-info h2')
def test_about_rules(self):
self.open_nitter(f'{COMMUNITY_PATH}/about')
self.assert_element_visible('.community-rules')
self.assert_text('Rules', '.community-rules h2')
rules = self.find_elements('.community-rule')
self.assertGreater(len(rules), 0)
def test_about_creator(self):
self.open_nitter(f'{COMMUNITY_PATH}/about')
self.assert_text('Created', '.community-info')
link = self.find_element('.community-info-item a')
self.assertTrue(link.text.startswith('@'))
self.assertGreater(len(link.text), 1)
def test_about_tab_active(self):
self.open_nitter(f'{COMMUNITY_PATH}/about')
self.assert_element_visible('.tab .active a[href$="/about"]')
def test_about_moderators(self):
self.open_nitter(f'{COMMUNITY_PATH}/about')
self.assert_element_visible('.community-moderators')
self.assert_text('Moderators', '.community-moderators h2')
mods = self.find_elements('.community-moderator')
self.assertGreater(len(mods), 0)
def test_about_moderators_have_avatars(self):
self.open_nitter(f'{COMMUNITY_PATH}/about')
avatars = self.find_elements('.community-mod-avatar')
self.assertGreater(len(avatars), 0)
def test_about_moderators_link_to_profiles(self):
self.open_nitter(f'{COMMUNITY_PATH}/about')
links = self.find_elements('.community-mod-username')
self.assertGreater(len(links), 0)
for link in links:
self.assertTrue(link.text.startswith('@'))
self.assertTrue(link.get_attribute('href').startswith('http'))
def test_about_see_all_link(self):
self.open_nitter(f'{COMMUNITY_PATH}/about')
link = self.find_element('.community-mods-link')
self.assertEqual(link.text, 'See all')
self.assertIn('/moderators', link.get_attribute('href'))
def test_members_page(self):
self.open_nitter(f'{COMMUNITY_PATH}/members')
self.assert_element_visible('.timeline-item')
users = self.find_elements('.timeline-item .username')
self.assertGreater(len(users), 0)
def test_members_has_member_tabs(self):
self.open_nitter(f'{COMMUNITY_PATH}/members')
tabs = self.find_elements('.tab a')
labels = [t.text for t in tabs]
self.assertEqual(labels, ['All', 'Moderators'])
def test_members_all_tab_active(self):
self.open_nitter(f'{COMMUNITY_PATH}/members')
self.assert_element_visible('.tab .active a[href$="/members"]')
def test_members_count_is_link(self):
self.open_nitter(COMMUNITY_PATH)
link = self.find_element('.community-member-count')
self.assertIn('Members', link.text)
self.assertIn('/members', link.get_attribute('href'))
def test_moderators_page(self):
self.open_nitter(f'{COMMUNITY_PATH}/moderators')
self.assert_element_visible('.timeline-item')
users = self.find_elements('.timeline-item .username')
self.assertGreater(len(users), 0)
def test_moderators_tab_active(self):
self.open_nitter(f'{COMMUNITY_PATH}/moderators')
self.assert_element_visible('.tab .active a[href$="/moderators"]')
def test_moderators_has_member_tabs(self):
self.open_nitter(f'{COMMUNITY_PATH}/moderators')
tabs = self.find_elements('.tab a')
labels = [t.text for t in tabs]
self.assertEqual(labels, ['All', 'Moderators'])
def test_pinned_tweet_label(self):
self.open_nitter(COMMUNITY_PATH)
self.assert_element_visible('.pinned')
self.assert_text('Pinned by Community mods', '.pinned')
def test_hashtags_visible(self):
self.open_nitter(COMMUNITY_PATH)
self.assert_element_visible('.community-tags')
tags = self.find_elements('.community-tag')
self.assertGreater(len(tags), 0)
def test_hashtags_are_links(self):
self.open_nitter(COMMUNITY_PATH)
tags = self.find_elements('.community-tag')
for tag in tags:
href = tag.get_attribute('href')
self.assertIn('/hashtag/', href)
self.assertTrue(tag.text.startswith('#'))
def test_hashtag_page(self):
self.open_nitter(f'{COMMUNITY_PATH}/hashtag/buildinpublic')
self.assert_element_visible('.timeline-item .tweet-body')
def test_hashtag_shows_header(self):
self.open_nitter(f'{COMMUNITY_PATH}/hashtag/buildinpublic')
self.assert_element_visible('.community-header')
self.assert_text('Build in Public', '.community-name')
def test_hashtag_shows_tag_title(self):
self.open_nitter(f'{COMMUNITY_PATH}/hashtag/buildinpublic')
self.assert_element_visible('.community-hashtag-header')
self.assert_text('#buildinpublic', '.community-hashtag-title')
def test_hashtag_no_main_tabs(self):
self.open_nitter(f'{COMMUNITY_PATH}/hashtag/buildinpublic')
tabs = self.find_elements('.tab a')
tab_labels = [t.text for t in tabs]
self.assertNotIn('Top', tab_labels)
self.assertNotIn('About', tab_labels)
def test_category_visible(self):
self.open_nitter(COMMUNITY_PATH)
self.assert_element_visible('.community-category')
def test_about_join_policy(self):
self.open_nitter(f'{COMMUNITY_PATH}/about')
self.assert_text('Anyone can join', '.community-info')
def test_about_visibility_note(self):
self.open_nitter(f'{COMMUNITY_PATH}/about')
self.assert_text('publicly visible', '.community-info')
def test_404_invalid_id(self):
self.open_nitter('i/communities/999')
self.assert_element_visible('.error-panel')
self.assert_text('not found', '.error-panel')
@parameterized.expand(['', '/latest', '/media', '/about',
'/members', '/moderators'])
def test_page_no_error(self, suffix):
self.open_nitter(f'{COMMUNITY_PATH}{suffix}')
self.assert_element_not_visible('.error-panel')

43
tests/test_followers.py Normal file
View file

@ -0,0 +1,43 @@
# SPDX-License-Identifier: AGPL-3.0-only
from base import BaseTestCase, Timeline
class FollowersTest(BaseTestCase):
"""Tests for followers and following pages"""
def test_followers_page_loads(self):
"""Test that followers page loads and shows users"""
self.open_nitter('jack/followers')
self.assert_title_contains('following @jack')
# Check for user list
self.assert_element('.timeline-item')
def test_following_page_loads(self):
"""Test that following page loads and shows users"""
self.open_nitter('jack/following')
self.assert_title_contains('followed by @jack')
# Check for user list
self.assert_element('.timeline-item')
def test_followers_has_navigation_tabs(self):
"""Test that followers page has following/followers tabs"""
self.open_nitter('jack/followers')
self.assert_element('.profile-statlist')
# Check for Following and Followers links
self.assert_element('a[href="/jack/following"]')
self.assert_element('a[href="/jack/followers"]')
def test_following_pagination(self):
"""Test that following page has load more button or scroll-to-top"""
self.open_nitter('jack/following')
# Should have either more button or top-ref (scroll to top arrow)
try:
self.assert_element('.show-more')
except:
self.assert_element('.top-ref')
def test_nonexistent_user_followers(self):
"""Test 404 for non-existent user"""
self.open_nitter('this_user_definitely_does_not_exist_12345/followers')
self.assert_element('.error-panel')
self.assert_text_visible('not found')

60
tests/test_security.py Normal file
View file

@ -0,0 +1,60 @@
import subprocess
from parameterized import parameterized
BASE_URL = 'http://localhost:8080'
def curl_status(url):
"""Get HTTP status code using curl to avoid URL normalization by Python libs."""
result = subprocess.run(
['curl', '-s', '-o', '/dev/null', '-w', '%{http_code}', url],
capture_output=True, text=True, timeout=10
)
return int(result.stdout)
class TestMalformedPaths:
"""Test that malformed paths don't crash the server.
URLs like //foo are parsed as having 'foo' as the authority (host),
resulting in an empty path. Empty paths previously crashed jester's
static file handler. Now they return 400.
URLs like //foo/bar are parsed as authority='foo', path='/bar',
so they route normally (not empty path).
"""
@parameterized.expand([
# These parse to empty paths -> 400
('//lefty_rae', 400),
('//test', 400),
('//anyuser', 400),
])
def test_empty_path_returns_400(self, path, expected_status):
"""URLs that parse to empty paths should return 400, not crash."""
status = curl_status(f'{BASE_URL}{path}')
assert status == expected_status, \
f'Expected {expected_status} for {path}, got {status}'
@parameterized.expand([
('/jack', 200),
('/about', 200),
('/', 200),
])
def test_normal_paths_work(self, path, expected_status):
"""Normal paths should still work."""
status = curl_status(f'{BASE_URL}{path}')
assert status == expected_status, \
f'Expected {expected_status} for {path}, got {status}'
def test_server_survives_malformed_requests(self):
"""Server should handle malformed requests without crashing."""
# These all parse to empty paths
malformed_paths = ['//a', '//b', '//c', '//user', '//test']
for path in malformed_paths:
status = curl_status(f'{BASE_URL}{path}')
assert status == 400, f'Expected 400 for {path}, got {status}'
# Verify server is still responding after malformed requests
status = curl_status(f'{BASE_URL}/')
assert status == 200, 'Server should still be alive'

119
tests/test_space.py Normal file
View file

@ -0,0 +1,119 @@
# SPDX-License-Identifier: AGPL-3.0-only
"""Integration tests for Twitter Spaces support."""
import pytest
from seleniumbase import BaseCase
class TestSpacePage(BaseCase):
"""Tests for /i/spaces/@id route."""
SPACE_ID = "1mxPaaRAwYjKN"
SPACE_URL = f"http://localhost:8080/i/spaces/{SPACE_ID}"
def test_space_page_loads(self):
"""Space page should load with title."""
self.open(self.SPACE_URL)
self.assert_element(".space-page")
self.assert_element(".space-panel")
self.assert_text_visible("INTEL WILL MOON NEXT WEEK", ".space-title")
def test_space_host_info(self):
"""Space should display host in participants."""
self.open(self.SPACE_URL)
self.assert_element(".space-participants")
self.assert_text_visible("bubble boi")
self.assert_element(".host-badge")
def test_space_metadata(self):
"""Space should display listener count and state."""
self.open(self.SPACE_URL)
self.assert_element(".space-meta")
# Should show listener count (number format)
meta_text = self.get_text(".space-meta")
assert any(c.isdigit() for c in meta_text), "Should show listener count"
# Should show ended state
assert "Ended" in meta_text or "Jun" in meta_text, "Should show ended state"
def test_space_participants(self):
"""Space should display host and speakers."""
self.open(self.SPACE_URL)
self.assert_element(".space-participants")
# Host should have badge
self.assert_element(".host-badge")
self.assert_text_visible("Host", ".host-badge")
# Should show speakers
self.assert_text_visible("CANTELOPEPEEL")
self.assert_text_visible("Based Burner Account")
self.assert_text_visible("anon invests")
def test_space_participant_avatars(self):
"""Participant avatars should load correctly."""
self.open(self.SPACE_URL)
# Check avatars in participants section
avatars = self.find_elements(".space-participant img")
assert len(avatars) >= 4, "Should have at least 4 participant avatars"
for avatar in avatars:
src = avatar.get_attribute("src")
# Should NOT be double-encoded
assert "%2Fpic%2F" not in src, f"Avatar URL double-encoded: {src}"
# Should have valid path
assert "/pic/" in src, f"Avatar URL missing /pic/: {src}"
def test_space_player_hls_disabled(self):
"""Without HLS, should show enable button with video-overlay style."""
self.open(self.SPACE_URL)
self.assert_element(".space-player")
self.assert_element(".video-overlay")
# Should show duration in overlay-duration
self.assert_element(".overlay-duration")
# Should have enable button
source = self.get_page_source()
assert "Enable hls playback" in source
def test_space_player_hls_enabled(self):
"""With HLS enabled, should have audio element in DOM."""
self.open(self.SPACE_URL)
# Set HLS preference via cookie
self.add_cookie({"name": "hlsPlayback", "value": "on"})
self.refresh()
# Check page source for audio element (hidden until play clicked)
source = self.get_page_source()
assert '<audio data-url="' in source, "Should have audio element"
assert "playAudio(this)" in source, "Should have play handler"
# Overlay should be visible (uses video-overlay class)
self.assert_element(".video-overlay")
def test_space_stream_endpoint(self):
"""Stream endpoint should return HLS manifest."""
import requests
resp = requests.get(f"{self.SPACE_URL}/stream")
assert resp.status_code == 200
assert "#EXTM3U" in resp.text
assert "#EXT-X-TARGETDURATION" in resp.text
class TestSpaceCard(BaseCase):
"""Tests for Space cards in tweets."""
def test_space_card_in_tweet(self):
"""Tweet with Space should show card linking to Space page."""
self.open("http://localhost:8080/bubbleboi/status/2065299704808960482")
self.assert_element(".card")
# Card should link to Space
card_link = self.find_element(".card-container")
href = card_link.get_attribute("href")
assert "/i/spaces/" in href
def test_space_card_title(self):
"""Space card should have title."""
self.open("http://localhost:8080/bubbleboi/status/2065299704808960482")
self.assert_text_visible("Twitter Space", ".card-title")
class TestSpaceNotFound(BaseCase):
"""Tests for error handling."""
def test_invalid_space_id(self):
"""Invalid Space ID should show error."""
self.open("http://localhost:8080/i/spaces/invalid123")
self.assert_text_visible("Space not found")

38
tests/test_ssrf_1411.nim Normal file
View file

@ -0,0 +1,38 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Reproduction + regression test for issue #1411:
# SSRF via /video proxy with default HMAC key and missing host validation.
import std/[unittest, uri]
import ".."/src/utils
suite "issue #1411 SSRF via /video proxy":
setup:
# The default key shipped in nitter.example.conf / config.nim.
setHmacKey("secretkey")
test "HMAC for arbitrary SSRF URLs is forgeable with the default key":
# These signatures were independently computed (Python hmac-sha256, uppercase
# hex, first 13 chars) and observed live in the issue report.
check getHmac("http://172.17.0.1:19999/secret_data.m3u8") == "BBD19ACC6C012"
check getHmac("http://172.17.0.1:19999/secret_data.mp4") == "0780F00DDF3E7"
test "isTwitterUrl rejects SSRF targets (the guard /video is missing)":
# Internal / metadata hosts an attacker would target.
check isTwitterUrl(parseUri("http://172.17.0.1:19999/secret_data.m3u8")) == false
check isTwitterUrl(parseUri("http://169.254.169.254/latest/meta-data/x.m3u8")) == false
check isTwitterUrl(parseUri("http://localhost/x.mp4")) == false
check isTwitterUrl(parseUri("http://[::1]/x.mp4")) == false
test "isTwitterUrl rejects userinfo / look-alike host bypass attempts":
check isTwitterUrl(parseUri("http://video.twimg.com@169.254.169.254/x.mp4")) == false
check isTwitterUrl(parseUri("http://video.twimg.com.evil.com/x.mp4")) == false
check isTwitterUrl(parseUri("http://evilvideo.twimg.com.attacker/x.mp4")) == false
test "isTwitterUrl rejects non-http schemes even on a Twitter host":
check isTwitterUrl(parseUri("gopher://video.twimg.com/x.mp4")) == false
check isTwitterUrl(parseUri("file:///etc/passwd")) == false
check isTwitterUrl(parseUri("ftp://video.twimg.com/x.mp4")) == false
test "isTwitterUrl still allows legitimate Twitter video hosts":
check isTwitterUrl(parseUri("https://video.twimg.com/ext_tw_video/1/pu/pl/x.m3u8")) == true
check isTwitterUrl(parseUri("https://video.twimg.com/amplify_video/1/vid/x.mp4")) == true
check isTwitterUrl(parseUri("https://prod-fastly-us-east-1.video.pscp.tv/x.m3u8")) == true

11
tests/test_thread.py
View file

@ -1,4 +1,5 @@
from parameterized import parameterized
import pytest
from base import BaseTestCase, Conversation
@ -8,17 +9,10 @@ thread = [
[],
"Based",
["Crystal", "Julia"],
[["yeah,"]],
[["For", "Then"], ["yeah,"]],
],
["octonion/status/975254452625002496", ["Based"], "Crystal", ["Julia"], []],
["octonion/status/975256058384887808", ["Based", "Crystal"], "Julia", [], []],
[
"gauravssnl/status/975364889039417344",
["Based", "For", "Then", "Okay,", "Python"],
"Speed",
[],
[["Java", "Coding", "I", "You"], ["JAVA!"]],
],
[
"d0m96/status/1141811379407425537",
[],
@ -48,6 +42,7 @@ class ThreadTest(BaseTestCase):
self.assert_equal(tweets[i], text)
@parameterized.expand(thread)
@pytest.mark.flaky(reruns=3)
def test_thread(self, tweet, before, main, after, replies):
self.open_nitter(tweet)
self.assert_element_visible(Conversation.main)

6
tests/test_tweet.py
View file

@ -71,8 +71,8 @@ emoji = [
]
retweet = [
[7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test', '1234'],
[3, 'mobile_test_8', 'mobile test 8', 'jack', '@jack', 'twttr']
[7, 'mobile_test_2', 'mobile test 2', 'Test account', '@mobile_test',
'Testing. 1234.']
]
@ -120,7 +120,7 @@ class TweetTest(BaseTestCase):
link = self.find_link_text(f'@{un}')
self.assertIn(f'/{un}', link.get_property('href'))
@parameterized.expand(retweet)
@parameterized.expand(retweet, skip_on_empty=True)
def test_retweet(self, index, url, retweet_by, fullname, username, text):
self.open_nitter(url)
tweet = get_timeline_tweet(index)

28
tests/test_tweet_media.py
View file

@ -1,6 +1,7 @@
from base import BaseTestCase, Poll, Media
from parameterized import parameterized
from selenium.webdriver.common.by import By
import pytest
poll = [
['nim_lang/status/1064219801499955200', 'Style insensitivity', '91', 1, [
@ -28,14 +29,14 @@ video_m3u8 = [
]
gallery = [
# ['mobile_test/status/451108446603980803', [
# ['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
# ]],
['mobile_test/status/451108446603980803', [
['BkKovdrCUAAEz79', 'BkKovdcCEAAfoBO']
]],
# ['mobile_test/status/471539824713691137', [
# ['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
# ['Bos--IqIQAAav23']
# ]],
['mobile_test/status/471539824713691137', [
['Bos--KNIQAAA7Li', 'Bos--FAIAAAWpah'],
['Bos--IqIQAAav23']
]],
['mobile_test/status/469530783384743936', [
['BoQbwJAIUAA0QCY', 'BoQbwN1IMAAuTiP'],
@ -81,19 +82,20 @@ class MediaTest(BaseTestCase):
self.assert_element_visible(Media.container)
self.assert_element_visible(Media.gif)
url = self.get_attribute('source', 'src')
thumb = self.get_attribute('video', 'poster')
url = self.get_attribute('.main-tweet source', 'src')
thumb = self.get_attribute('.main-tweet video', 'poster')
self.assertIn(gif_id + '.mp4', url)
self.assertIn(gif_id + '.jpg', thumb)
@parameterized.expand(video_m3u8)
def test_video_m3u8(self, tweet, thumb):
# no url because video playback isn't supported yet
self.open_nitter(tweet)
self.assert_element_visible(Media.container)
self.assert_element_visible(Media.video)
self.driver.delete_cookie("hlsPlayback")
self.refresh()
self.assert_element_visible('.main-tweet ' + Media.container)
self.assert_element_visible('.main-tweet ' + Media.video)
video_thumb = self.get_attribute(Media.video + ' img', 'src')
video_thumb = self.get_attribute('.main-tweet ' + Media.video + ' img', 'src')
self.assertIn(thumb, video_thumb)
@parameterized.expand(gallery)