Watch
1
0
Fork
You've already forked nitter
0
mirror of https://github.com/zedeus/nitter synced 2026-11-22 12:57:19 +00:00

Merge branch 'zedeus:master' into master

This commit is contained in:
Salastil 2026-07-02 17:53:12 -04:00 • committed by GitHub
commit 106df59291
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
71 changed files with 3652 additions and 481 deletions

View file

@ -7,55 +7,105 @@ on:
branches:
- master
concurrency:
group: docker-publish-${{ github.ref }}
cancel-in-progress: true
env:
IMAGE: zedeus/nitter
jobs:
tests:
uses: ./.github/workflows/run-tests.yml
secrets: inherit
build-docker-amd64:
build:
needs: [tests]
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
include:
- runner: ubuntu-24.04
platform: linux/amd64
- runner: ubuntu-24.04-arm
platform: linux/arm64
runs-on: ${{ matrix.runner }}
steps:
- name: Prepare platform name
run: echo "PLATFORM_PAIR=${platform//\//-}" >> "$GITHUB_ENV"
env:
platform: ${{ matrix.platform }}
- uses: actions/checkout@v6
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
version: latest
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push AMD64 Docker image
uses: docker/build-push-action@v3
- name: Build and push by digest
id: build
uses: docker/build-push-action@v6
with:
context: .
file: ./Dockerfile
platforms: linux/amd64
push: true
tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}
platforms: ${{ matrix.platform }}
outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
provenance: false
sbom: false
build-docker-arm64:
needs: [tests]
runs-on: ubuntu-24.04-arm
- name: Export digest
run: |
mkdir -p "${{ runner.temp }}/digests"
digest="${{ steps.build.outputs.digest }}"
touch "${{ runner.temp }}/digests/${digest#sha256:}"
- name: Upload digest
uses: actions/upload-artifact@v4
with:
name: digests-${{ env.PLATFORM_PAIR }}
path: ${{ runner.temp }}/digests/*
if-no-files-found: error
retention-days: 1
# Combine the per-arch digests into one multi-arch manifest so that
# `docker pull zedeus/nitter:latest` serves the right image on any CPU.
merge:
needs: [build]
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v6
- name: Download digests
uses: actions/download-artifact@v4
with:
path: ${{ runner.temp }}/digests
pattern: digests-*
merge-multiple: true
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
version: latest
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push ARM64 Docker image
uses: docker/build-push-action@v3
with:
context: .
file: ./Dockerfile.arm64
platforms: linux/arm64
push: true
tags: zedeus/nitter:latest-arm64,zedeus/nitter:${{ github.sha }}-arm64
- name: Create manifest list and push
working-directory: ${{ runner.temp }}/digests
run: |
docker buildx imagetools create \
-t ${{ env.IMAGE }}:latest \
-t ${{ env.IMAGE }}:latest-arm64 \
-t ${{ env.IMAGE }}:${{ github.sha }} \
$(printf '${{ env.IMAGE }}@sha256:%s ' *)
- name: Inspect image
run: docker buildx imagetools inspect ${{ env.IMAGE }}:${{ github.sha }}

View file

@ -32,10 +32,12 @@ jobs:
id: cache-nimble
uses: actions/cache@v5
with:
path: ~/.nimble
key: ${{ matrix.nim }}-nimble-v2-${{ hashFiles('*.nimble') }}
path: |
~/.nimble/pkgcache
~/.nimble/packages_official.json
key: ${{ matrix.nim }}-nimble-v6-${{ hashFiles('*.nimble') }}
restore-keys: |
${{ matrix.nim }}-nimble-v2-
${{ matrix.nim }}-nimble-v6-
- name: Setup Nim
uses: jiro4989/setup-nim-action@v2
@ -103,10 +105,12 @@ jobs:
- name: Cache Nimble Dependencies
uses: actions/cache@v5
with:
path: ~/.nimble
key: 2.2.x-nimble-v2-${{ hashFiles('*.nimble') }}
path: |
~/.nimble/pkgcache
~/.nimble/packages_official.json
key: 2.2.x-nimble-v6-${{ hashFiles('*.nimble') }}
restore-keys: |
2.2.x-nimble-v2-
2.2.x-nimble-v6-
- name: Setup Nim
uses: jiro4989/setup-nim-action@v2
@ -115,6 +119,9 @@ jobs:
use-nightlies: true
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Install Nimble dependencies
run: nimble install -y --depsOnly
- name: Download 2.2.x build artifact
uses: actions/download-artifact@v4
with:
@ -130,10 +137,8 @@ jobs:
sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf
sed -i 's/maxRetries = 1/maxRetries = 10/g' nitter.conf
# Run both Nimble tasks concurrently
nim r tools/rendermd.nim &
nim r tools/gencss.nim &
wait
nim r tools/rendermd.nim
nim r tools/gencss.nim
echo '${{ secrets.SESSIONS }}' | head -n1
echo '${{ secrets.SESSIONS }}' > ./sessions.jsonl

3
.gitignore vendored
View file

@ -15,3 +15,6 @@ sessions.json*
dump.rdb
*.bak
/tools/*.json*
nimbledeps/
nimble.paths
nimble.develop

View file

@ -1,4 +1,4 @@
FROM nimlang/nim:2.2.0-alpine-regular as nim
FROM nimlang/nim:2.2.6-alpine-regular as nim
LABEL maintainer="setenforce@protonmail.com"
RUN apk --no-cache add libsass-dev pcre
@ -15,7 +15,7 @@ RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
FROM alpine:latest
WORKDIR /src/
RUN apk --no-cache add pcre ca-certificates
RUN apk --no-cache add pcre ca-certificates openssl
COPY --from=nim /src/nitter/nitter ./
COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
COPY --from=nim /src/nitter/public ./public

View file

@ -1,25 +0,0 @@
FROM alpine:3.20.6 as nim
LABEL maintainer="setenforce@protonmail.com"
RUN apk --no-cache add libsass-dev pcre gcc git libc-dev nim nimble
WORKDIR /src/nitter
COPY nitter.nimble .
RUN nimble install -y --depsOnly
COPY . .
RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
&& nimble scss \
&& nimble md
FROM alpine:3.20.6
WORKDIR /src/
RUN apk --no-cache add pcre ca-certificates openssl
COPY --from=nim /src/nitter/nitter ./
COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
COPY --from=nim /src/nitter/public ./public
EXPOSE 8080
RUN adduser -h /src/ -D -s /bin/sh nitter
USER nitter
CMD ./nitter

View file

@ -104,9 +104,9 @@ along with the scss and md files.
# su nitter
$ git clone https://github.com/zedeus/nitter
$ cd nitter
$ nimble build -d:danger --mm:refc
$ nimble scss
$ nimble md
$ nimble -l build -d:danger --mm:refc
$ nimble -l scss
$ nimble -l md
$ cp nitter.example.conf nitter.conf
```
@ -123,12 +123,23 @@ performance reasons.
Page for the Docker image: https://hub.docker.com/r/zedeus/nitter
#### NOTE: For ARM64 support, please use the separate ARM64 docker image: [`zedeus/nitter:latest-arm64`](https://hub.docker.com/r/zedeus/nitter/tags).
#### NOTE: The published image is multi-arch — `zedeus/nitter:latest` runs natively on both `amd64` and `arm64`.
To run Nitter with Docker, you'll need to install and run Redis separately
before you can run the container. See below for how to also run Redis using
Docker.
First create your config file. The Docker commands mount it into the container,
so it has to exist on the host beforehand. If you've cloned the repo:
```bash
cp nitter.example.conf nitter.conf
```
If you're using the prebuilt image without a local clone, download
[`nitter.example.conf`](https://raw.githubusercontent.com/zedeus/nitter/master/nitter.example.conf)
and save it as `nitter.conf` instead.
To build and run Nitter in Docker:
```bash
@ -136,8 +147,6 @@ docker build -t nitter:latest .
docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest
```
Note: For ARM64, use this Dockerfile: [`Dockerfile.arm64`](https://github.com/zedeus/nitter/blob/master/Dockerfile.arm64).
A prebuilt Docker image is provided as well:
```bash
@ -151,8 +160,11 @@ Change `redisHost` from `localhost` to `nitter-redis` in `nitter.conf`, then run
docker-compose up -d
```
Note the Docker commands expect a `nitter.conf` file in the directory you run
them.
Note the Docker commands mount `nitter.conf` (and `sessions.jsonl` for
docker-compose) from the directory you run them in. If a mounted file doesn't
exist, Docker silently creates a directory in its place and the container fails
with `not a directory: Are you trying to mount a directory onto a file`. Remove
that directory and create the file as shown above.
### systemd

View file

@ -1,5 +1,3 @@
version: "3"
services:
nitter:

View file

@ -11,3 +11,7 @@ warning("HoleEnumConv", off)
hint("XDeclaredButNotUsed", off)
hint("XCannotRaiseY", off)
hint("User", off)
# begin Nimble config (version 2)
when withDir(thisDir(), system.fileExists("nimble.paths")):
include "nimble.paths"
# end Nimble config

View file

@ -20,7 +20,7 @@ redisMaxConnections = 30
# you receive tons of requests per second
[Config]
hmacKey = "secretkey" # random key for cryptographic signing of video urls
hmacKey = "secretkey" # CHANGE THIS to a unique random value (e.g. `openssl rand -hex 32`); signs media urls
base64Media = false # use base64 encoding for proxied media urls
enableRSS = true # master switch, set to false to disable all RSS feeds
enableRSSUserTweets = true # /@user/rss

View file

@ -11,19 +11,18 @@ bin = @["nitter"]
# Dependencies
requires "nim >= 2.0.0"
requires "jester#baca3f"
requires "karax#5cf360c"
requires "sass#7dfdd03"
requires "nimcrypto#a079df9"
requires "markdown#158efe3"
requires "jester == 0.6.0"
requires "karax == 1.5.0"
requires "sass == 0.2.0"
requires "nimcrypto == 0.7.3"
requires "markdown == 0.8.8"
requires "packedjson#9e6fbb6"
requires "supersnappy#6c94198"
requires "redpool#8b7c1db"
requires "https://github.com/zedeus/redis#d0a0e6f"
requires "zippy#ca5989a"
requires "flatty#e668085"
requires "jsony#1de1f08"
requires "oauth#b8c163b"
requires "supersnappy == 2.1.4"
requires "redpool == 0.2.2"
requires "zippy == 0.10.19"
requires "flatty == 0.4.0"
requires "jsony == 1.1.6"
requires "oauth == 0.11"
# Tasks

View file

@ -1,12 +1,12 @@
@font-face {
font-family: "fontello";
src: url("/fonts/fontello.eot?49059696");
src: url("/fonts/fontello.eot?59696369");
src:
url("/fonts/fontello.eot?49059696#iefix") format("embedded-opentype"),
url("/fonts/fontello.woff2?49059696") format("woff2"),
url("/fonts/fontello.woff?49059696") format("woff"),
url("/fonts/fontello.ttf?49059696") format("truetype"),
url("/fonts/fontello.svg?49059696#fontello") format("svg");
url("/fonts/fontello.eot?59696369#iefix") format("embedded-opentype"),
url("/fonts/fontello.woff2?59696369") format("woff2"),
url("/fonts/fontello.woff?59696369") format("woff"),
url("/fonts/fontello.ttf?59696369") format("truetype"),
url("/fonts/fontello.svg?59696369#fontello") format("svg");
font-weight: normal;
font-style: normal;
}
@ -36,113 +36,118 @@
-moz-osx-font-smoothing: grayscale;
}
.icon-group:before {
content: "\e0c3";
}
/* '' */
.icon-views:before {
content: "\e800";
}
/* '' */
/* '' */
.icon-heart:before {
content: "\e801";
}
/* '' */
/* '' */
.icon-quote:before {
content: "\e802";
}
/* '' */
/* '' */
.icon-comment:before {
content: "\e803";
}
/* '' */
.icon-group:before {
content: "\e804";
}
/* '' */
/* '' */
.icon-play:before {
content: "\e805";
}
/* '' */
/* '' */
.icon-link:before {
content: "\e806";
}
/* '' */
/* '' */
.icon-calendar:before {
content: "\e807";
}
/* '' */
/* '' */
.icon-location:before {
content: "\e808";
}
/* '' */
/* '' */
.icon-picture:before {
content: "\e809";
}
/* '' */
/* '' */
.icon-lock:before {
content: "\e80a";
}
/* '' */
/* '' */
.icon-down:before {
content: "\e80b";
}
/* '' */
/* '' */
.icon-retweet:before {
content: "\e80c";
}
/* '' */
/* '' */
.icon-search:before {
content: "\e80d";
}
/* '' */
/* '' */
.icon-pin:before {
content: "\e80e";
}
/* '' */
/* '' */
.icon-cog:before {
content: "\e80f";
}
/* '' */
/* '' */
.icon-rss:before {
content: "\e810";
}
/* '' */
/* '' */
.icon-ok:before {
content: "\e811";
}
/* '' */
.icon-attention:before {
/* '' */
.icon-attention-circled:before {
content: "\e812";
}
/* '' */
/* '' */
.icon-download-alt:before {
content: "\e813";
}
/* '' */
.icon-circle:before {
content: "\f111";
}
/* '' */
/* '' */
.icon-info:before {
content: "\f128";
}
/* '' */
/* '' */
.icon-bird:before {
content: "\f309";
}
/* '' */
/* '' */

Binary file not shown.

View file

@ -6,6 +6,8 @@
<font id="fontello" horiz-adv-x="1000" >
<font-face font-family="fontello" font-weight="400" font-stretch="normal" units-per-em="1000" ascent="850" descent="-150" />
<missing-glyph horiz-adv-x="1000" />
<glyph glyph-name="group" unicode="&#xe0c3;" d="M0 106l0 134q0 26 18 32l171 80q-66 39-68 131 0 56 35 103 37 41 90 43 31 0 63-19-49-125 23-237-12-11-25-19l-114-55q-48-23-52-84l0-143-114 0q-25 0-27 34z m193-59l0 168q0 27 22 37l152 70 57 28q-37 23-60 66t-22 94q0 76 46 130t110 54 109-54 45-130q0-105-78-158l61-30 146-70q24-10 24-37l0-168q-2-37-37-41l-541 0q-14 2-24 14t-10 27z m473 330q68 106 22 231 31 19 66 21 49 0 90-43 35-41 35-103 0-82-65-131l168-80q18-10 18-32l0-134q0-32-27-34l-118 0 0 143q0 57-50 84l-110 53q-15 8-29 25z" horiz-adv-x="1000" />
<glyph glyph-name="views" unicode="&#xe800;" d="M180 516l0-538-180 0 0 538 180 0z m250-138l0-400-180 0 0 400 180 0z m250 344l0-744-180 0 0 744 180 0z" horiz-adv-x="680" />
<glyph glyph-name="heart" unicode="&#xe801;" d="M790 644q70-64 70-156t-70-158l-360-330-360 330q-70 66-70 158t70 156q62 58 151 58t153-58l56-52 58 52q62 58 150 58t152-58z" horiz-adv-x="860" />
@ -14,8 +16,6 @@
<glyph glyph-name="comment" unicode="&#xe803;" d="M1000 350q0-97-67-179t-182-130-251-48q-39 0-81 4-110-97-257-135-27-8-63-12-10-1-17 5t-10 16v1q-2 2 0 6t1 6 2 5l4 5t4 5 4 5q4 5 17 19t20 22 17 22 18 28 15 33 15 42q-88 50-138 123t-51 157q0 73 40 139t106 114 160 76 194 28q136 0 251-48t182-130 67-179z" horiz-adv-x="1000" />
<glyph glyph-name="group" unicode="&#xe804;" d="M0 106l0 134q0 26 18 32l171 80q-66 39-68 131 0 56 35 103 37 41 90 43 31 0 63-19-49-125 23-237-12-11-25-19l-114-55q-48-23-52-84l0-143-114 0q-25 0-27 34z m193-59l0 168q0 27 22 37l152 70 57 28q-37 23-60 66t-22 94q0 76 46 130t110 54 109-54 45-130q0-105-78-158l61-30 146-70q24-10 24-37l0-168q-2-37-37-41l-541 0q-14 2-24 14t-10 27z m473 330q68 106 22 231 31 19 66 21 49 0 90-43 35-41 35-103 0-82-65-131l168-80q18-10 18-32l0-134q0-32-27-34l-118 0 0 143q0 57-50 84l-110 53q-15 8-29 25z" horiz-adv-x="1000" />
<glyph glyph-name="play" unicode="&#xe805;" d="M772 333l-741-412q-13-7-22-2t-9 20v822q0 14 9 20t22-2l741-412q13-7 13-17t-13-17z" horiz-adv-x="785.7" />
<glyph glyph-name="link" unicode="&#xe806;" d="M294 116q14 14 34 14t36-14q32-34 0-70l-42-40q-56-56-132-56-78 0-134 56t-56 132q0 78 56 134l148 148q70 68 144 77t128-43q16-16 16-36t-16-36q-36-32-70 0-50 48-132-34l-148-146q-26-26-26-64t26-62q26-26 63-26t63 26z m450 574q56-56 56-132 0-78-56-134l-158-158q-74-72-150-72-62 0-112 50-14 14-14 34t14 36q14 14 35 14t35-14q50-48 122 24l158 156q28 28 28 64 0 38-28 62-24 26-56 31t-60-21l-50-50q-16-14-36-14t-34 14q-34 34 0 70l50 50q54 54 127 51t129-61z" horiz-adv-x="800" />
@ -44,6 +44,8 @@
<glyph glyph-name="attention-circled" unicode="&#xe812;" d="M429 779q116 0 215-58t156-156 57-215-57-215-156-156-215-58-216 58-155 156-58 215 58 215 155 156 216 58z m71-696v106q0 8-5 13t-12 5h-107q-8 0-13-5t-6-13v-106q0-8 6-13t13-6h107q7 0 12 6t5 13z m-1 192l10 346q0 7-6 10-5 5-13 5h-123q-8 0-13-5-6-3-6-10l10-346q0-6 5-10t14-4h103q8 0 13 4t6 10z" horiz-adv-x="857.1" />
<glyph glyph-name="download-alt" unicode="&#xe813;" d="M0-150l0 135 1000 0 0-135-1000 0z m88 586l228 0 0 414 370 0 0-414 228 0-414-385z" horiz-adv-x="1000" />
<glyph glyph-name="circle" unicode="&#xf111;" d="M857 350q0-117-57-215t-156-156-215-58-216 58-155 156-58 215 58 215 155 156 216 58 215-58 156-156 57-215z" horiz-adv-x="857.1" />
<glyph glyph-name="info" unicode="&#xf128;" d="M393 149v-134q0-9-7-15t-15-7h-134q-9 0-16 7t-7 15v134q0 9 7 16t16 6h134q9 0 15-6t7-16z m176 335q0-30-8-56t-20-43-31-33-32-25-34-19q-23-13-38-37t-15-37q0-10-7-18t-16-9h-134q-8 0-14 11t-6 20v26q0 46 37 87t79 60q33 16 47 32t14 42q0 24-26 41t-60 18q-36 0-60-16-20-14-60-64-7-9-17-9-7 0-14 4l-91 70q-8 6-9 14t3 16q89 148 259 148 45 0 90-17t81-46 59-72 23-88z" horiz-adv-x="571.4" />

Before

Width:  |  Height:  |  Size: 7.3 KiB

After

Width:  |  Height:  |  Size: 7.4 KiB

Before After
Before After

Binary file not shown.

Binary file not shown.

Binary file not shown.

View file

@ -1,27 +1,30 @@
// @license http://www.gnu.org/licenses/agpl-3.0.html AGPL-3.0
// SPDX-License-Identifier: AGPL-3.0-only
function playVideo(overlay) {
const video = overlay.parentElement.querySelector('video');
const url = video.getAttribute("data-url");
const startTime = parseFloat(video.getAttribute("data-start") || "0");
video.setAttribute("controls", "");
function playMedia(overlay, tagName) {
const media = overlay.parentElement.querySelector(tagName);
const url = media.getAttribute("data-url");
const startTime = parseFloat(media.getAttribute("data-start") || "0");
media.setAttribute("controls", "");
overlay.style.display = "none";
if (Hls.isSupported()) {
var hls = new Hls({autoStartLoad: false});
hls.loadSource(url);
hls.attachMedia(video);
hls.attachMedia(media);
hls.on(Hls.Events.MANIFEST_PARSED, function () {
hls.loadLevel = hls.levels.length - 1;
hls.startLoad(startTime);
video.play();
media.play();
});
} else if (video.canPlayType('application/vnd.apple.mpegurl')) {
video.src = url;
video.addEventListener('canplay', function() {
if (startTime > 0) video.currentTime = startTime;
video.play();
} else if (media.canPlayType('application/vnd.apple.mpegurl')) {
media.src = url;
media.addEventListener('canplay', function() {
if (startTime > 0) media.currentTime = startTime;
media.play();
});
}
}
function playVideo(overlay) { playMedia(overlay, 'video'); }
function playAudio(overlay) { playMedia(overlay, 'audio'); }
// @license-end

View file

@ -2,7 +2,7 @@
import asyncdispatch, httpclient, strutils, sequtils, sugar
import packedjson
import types, query, formatters, consts, apiutils, parser, utils
import experimental/parser as newParser
import experimental/parser
# Helper to generate params object for GraphQL requests
proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
@ -11,13 +11,18 @@ proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
if fieldToggles.len > 0:
result.add ("fieldToggles", fieldToggles)
proc apiUrl(endpoint, variables: string; fieldToggles = ""): ApiUrl =
return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles))
proc apiUrl(endpoint, variables: string; fieldToggles = ""; skipTid = false): ApiUrl =
return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles), skipTid: skipTid)
proc apiReq(endpoint, variables: string; fieldToggles = ""): ApiReq =
let url = apiUrl(endpoint, variables, fieldToggles)
proc apiReq(endpoint, variables: string; fieldToggles = ""; skipTid = false): ApiReq =
let url = apiUrl(endpoint, variables, fieldToggles, skipTid)
return ApiReq(cookie: url, oauth: url)
proc cursorParam(after: string): string =
## JSON-escape the user-supplied cursor so it cannot break out of the GraphQL
## variables object (same input-validation class as the #1411 media SSRF).
if after.len > 0: "\"cursor\":" & $(%after) & "," else: ""
proc mediaUrl(id, cursor: string; count=20): ApiReq =
result = ApiReq(
cookie: apiUrl(graphUserMedia, userMediaVars % [id, cursor, $count]),
@ -25,33 +30,24 @@ proc mediaUrl(id, cursor: string; count=20): ApiReq =
)
proc userTweetsUrl(id: string; cursor: string): ApiReq =
result = ApiReq(
# cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"])
)
# might change this in the future pending testing
result.cookie = result.oauth
return apiReq(graphUserTweetsV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles)
proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq =
let cookieVars = userTweetsAndRepliesVars % [id, cursor]
result = ApiReq(
cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"])
)
return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles, skipTid=true)
proc tweetDetailUrl(id: string; cursor: string): ApiReq =
let cookieVars = tweetDetailVars % [id, cursor]
result = ApiReq(
# cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
cookie: apiUrl(graphTweet, tweetVars % [id, cursor]),
oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
)
return apiReq(graphTweet, tweetVars % [id, cursor])
# let cookieVars = tweetDetailVars % [id, cursor]
# result = ApiReq(
# cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
# oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
# )
proc userUrl(username: string): ApiReq =
let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
let cookieVars = $(%*{"screen_name": username, "withGrokTranslatedBio": false})
result = ApiReq(
cookie: apiUrl(graphUser, cookieVars, tweetDetailFieldToggles),
oauth: apiUrl(graphUserV2, """{"screen_name": "$1"}""" % username)
oauth: apiUrl(graphUserV2, $(%*{"screen_name": username}))
)
proc getGraphUser*(username: string): Future[User] {.async.} =
@ -69,7 +65,7 @@ proc getGraphUserById*(id: string): Future[User] {.async.} =
proc getAboutAccount*(username: string): Future[AccountInfo] {.async.} =
if username.len == 0: return
let
url = apiReq(graphAboutAccount, """{"screenName":"$1"}""" % username)
url = apiReq(graphAboutAccount, $(%*{"screenName": username}))
js = await fetch(url)
result = parseAboutAccount(js)
@ -80,7 +76,7 @@ proc restReq(endpoint: string; params: seq[(string, string)] = @[]): ApiReq =
proc getBroadcastInfo*(id: string): Future[Broadcast] {.async.} =
if id.len == 0: return
let
req = apiReq(graphBroadcast, """{"id":"$1"}""" % id)
req = apiReq(graphBroadcast, $(%*{"id": id}))
js = await fetch(req)
result = parseBroadcastInfo(js)
@ -92,10 +88,23 @@ proc fetchBroadcastStream*(mediaKey: string): Future[string] {.async.} =
result = streamJs{"source", "noRedirectPlaybackUrl"}.getStr(
streamJs{"source", "location"}.getStr)
proc getAudioSpace*(id: string): Future[AudioSpace] {.async.} =
if id.len == 0: return
let
variables = %*{
"id": id,
"isMetatagsQuery": false,
"withReplays": true,
"withListeners": true
}
req = apiReq(graphAudioSpace, $variables)
js = await fetch(req)
result = parseAudioSpace(js)
proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
cursor = cursorParam(after)
url = case kind
of TimelineKind.tweets: userTweetsUrl(id, cursor)
of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor)
@ -103,10 +112,61 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi
js = await fetch(url)
result = parseGraphTimeline(js, after)
proc getGraphCommunity*(id: string): Future[Community] {.async.} =
if id.len == 0: return
let
url = apiReq(graphCommunity, $(%*{"communityId": id}))
js = await fetch(url)
result = parseGraphCommunity(js)
proc getGraphCommunityTweets*(id: string; rankingMode: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = cursorParam(after)
url = apiReq(graphCommunityTweets, communityTweetsVars % [id, cursor, rankingMode])
js = await fetch(url)
result = parseGraphCommunityTimeline(js, after)
proc getGraphCommunityMedia*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = cursorParam(after)
url = apiReq(graphCommunityMedia, communityMediaVars % [id, cursor])
js = await fetch(url)
result = parseGraphCommunityTimeline(js, after)
proc communitySliceReq(endpoint, variables: string): ApiReq =
let url = ApiUrl(endpoint: endpoint, params: @[("variables", variables)])
ApiReq(cookie: url, oauth: url)
proc getGraphCommunityMembers*(id: string; after=""): Future[Result[User]] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: $(%after) else: "null"
url = communitySliceReq(graphCommunityMembers, communityMembersVars % [id, cursor])
js = await fetch(url)
result = parseGraphCommunityMembers(js, after)
proc getGraphCommunityModerators*(id: string): Future[Result[User]] {.async.} =
if id.len == 0: return
let
url = communitySliceReq(graphCommunityModerators, communityMembersVars % [id, "null"])
js = await fetch(url)
result = parseGraphCommunityMembers(js)
proc getGraphCommunityHashtags*(id, hashtag: string; after=""): Future[Timeline] {.async.} =
if id.len == 0 or hashtag.len == 0: return
let
safeTag = multiReplace(hashtag, ("\"", ""), ("\\", ""))
cursor = cursorParam(after)
url = apiReq(graphCommunityHashtags, communityHashtagsVars % [id, cursor, safeTag])
js = await fetch(url)
result = parseGraphCommunityTimeline(js, after)
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
cursor = cursorParam(after)
url = apiReq(graphListTweets, restIdVars % [id, cursor, "20"])
js = await fetch(url)
result = parseGraphTimeline(js, after).tweets
@ -120,7 +180,7 @@ proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
proc getGraphList*(id: string): Future[List] {.async.} =
let
url = apiReq(graphListById, """{"listId": "$1"}""" % id)
url = apiReq(graphListById, $(%*{"listId": id}))
js = await fetch(url)
result = parseGraphList(js)
@ -141,17 +201,39 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
js = await fetchRaw(url)
result = parseGraphListMembers(js, after)
proc getGraphUserConnections(userId: string; endpoint: string; kind: QueryKind;
after=""): Future[Result[User]] {.async.} =
if userId.len == 0: return
var variables = %*{
"userId": userId,
"count": 20,
"includePromotedContent": false,
"withGrokTranslatedBio": true
}
if after.len > 0:
variables["cursor"] = %after
let
url = apiReq(endpoint, $variables)
js = await fetchRaw(url)
result = parseGraphFollowers(js, after, kind)
proc getGraphFollowers*(userId: string; after=""): Future[Result[User]] {.async.} =
result = await getGraphUserConnections(userId, graphFollowers, followers, after)
proc getGraphFollowing*(userId: string; after=""): Future[Result[User]] {.async.} =
result = await getGraphUserConnections(userId, graphFollowing, following, after)
proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
if id.len == 0: return
let
url = apiReq(graphTweetResult, """{"rest_id": "$1"}""" % id)
url = apiReq(graphTweetResult, $(%*{"rest_id": id}))
js = await fetch(url)
result = parseGraphTweetResult(js)
proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
cursor = cursorParam(after)
js = await fetch(tweetDetailUrl(id, cursor))
result = parseGraphConversation(js, id)
@ -184,13 +266,13 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
var
variables = %*{
"rawQuery": q,
"query_source": "typedQuery",
"count": 20,
"querySource": "typed_query",
"product": "Latest",
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false
"withGrokTranslatedBio":true,
"withQuickPromoteEligibilityTweetFields":false
}
if after.len > 0 and maxId.len == 0:
variables["cursor"] = % after
let
@ -212,12 +294,11 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
var
variables = %*{
"rawQuery": query.text,
"query_source": "typedQuery",
"count": 20,
"querySource": "typed_query",
"product": "People",
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false
"withGrokTranslatedBio":true,
"withQuickPromoteEligibilityTweetFields":false
}
if after.len > 0:
variables["cursor"] = % after
@ -234,6 +315,21 @@ proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
let js = await fetch(mediaUrl(id, "", 30))
result = parseGraphPhotoRail(js)
proc getGraphArticle*(id: string): Future[Article] {.async.} =
if id.len == 0: return
let
url = apiReq(graphTweetResultByRestId, articleVars % id, articleFieldToggles)
json = await fetchRaw(url)
result = parseGraphArticle(json)
proc getGraphTweetResults*(ids: seq[string]): Future[seq[Tweet]] {.async.} =
if ids.len == 0: return
let
idsJson = "[" & ids.mapIt("\"" & it & "\"").join(",") & "]"
url = apiReq(graphTweetResultsByRestIds, articleBatchVars % idsJson, articleFieldToggles)
js = await fetch(url)
result = parseGraphTweetResults(js)
proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
let client = newAsyncHttpClient(maxRedirects=0)
try:

View file

@ -1,6 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-only
import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
import jsony, packedjson, zippy, oauth1
import jsony, packedjson, zippy, oauth/oauth1
import types, auth, consts, parserutils, http_pool, tid
import experimental/types/common
@ -8,6 +8,7 @@ const
rlRemaining = "x-rate-limit-remaining"
rlReset = "x-rate-limit-reset"
rlLimit = "x-rate-limit-limit"
npCache = "x-np-cache"
errorsToSkip = {null, doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
var
@ -63,7 +64,7 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
proc getCookieHeader(authToken, ct0: string): string =
"auth_token=" & authToken & "; ct0=" & ct0
proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
proc genHeaders*(session: Session, url: Uri, skipTid: bool): Future[HttpHeaders] {.async.} =
result = newHttpHeaders({
"accept": "*/*",
"accept-encoding": "gzip",
@ -84,13 +85,14 @@ proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
result["x-twitter-auth-type"] = "OAuth2Session"
result["x-csrf-token"] = session.ct0
result["cookie"] = getCookieHeader(session.authToken, session.ct0)
result["referer"] = "https://x.com/"
result["sec-ch-ua"] = """"Google Chrome";v="142", "Chromium";v="142", "Not A(Brand";v="24""""
result["sec-ch-ua-mobile"] = "?0"
result["sec-ch-ua-platform"] = "Windows"
result["sec-fetch-dest"] = "empty"
result["sec-fetch-mode"] = "cors"
result["sec-fetch-site"] = "same-site"
if disableTid or "/1.1/" in url.path:
result["sec-fetch-site"] = "same-origin"
if disableTid or skipTid or "/1.1/" in url.path:
result["authorization"] = bearerToken2
else:
result["authorization"] = bearerToken
@ -114,7 +116,12 @@ template fetchImpl(result, fetchBody) {.dirty.} =
try:
var resp: AsyncResponse
pool.use(await genHeaders(session, url)):
let skipTid = case session.kind
of oauth: req.oauth.skipTid
of cookie: req.cookie.skipTid
let headers = await genHeaders(session, url, skipTid)
pool.use(headers):
template getContent =
# TODO: this is a temporary simple implementation
if apiProxy.len > 0 and "/1.1/" notin url.path:
@ -130,10 +137,11 @@ template fetchImpl(result, fetchBody) {.dirty.} =
raise newException(BadClientError, "Bad client")
if resp.status == $Http404 and result.len == 0:
echo "[sessions] transient 404 (empty body), retrying: ", url.path
echo "[sessions] transient 404 (empty body), retrying: ", url.path, ", session: ", session.pretty
raise rateLimitError()
if resp.headers.hasKey(rlRemaining):
let cacheStatus = resp.headers.getOrDefault(npCache)
if cacheStatus notin ["HIT", "STALE"] and resp.headers.hasKey(rlRemaining):
let
remaining = parseInt(resp.headers[rlRemaining])
reset = parseInt(resp.headers[rlReset])
@ -147,7 +155,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
if result.startsWith("{\"errors"):
let errors = result.fromJson(Errors)
if errors notin errorsToSkip:
echo "Fetch error, API: ", url.path, ", errors: ", errors
echo "Fetch error, API: ", url.path, ", errors: ", errors, ", session: ", session.pretty
if errors in {expiredToken, badToken, locked}:
invalidate(session)
raise rateLimitError()
@ -162,7 +170,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
fetchBody
if resp.status == $Http400:
echo "ERROR 400, ", url.path, ": ", result
echo "ERROR 400, ", url.path, ": ", result, ", session: ", session.pretty
raise newException(InternalError, $url)
except InternalError as e:
raise e
@ -177,22 +185,34 @@ template fetchImpl(result, fetchBody) {.dirty.} =
finally:
release(session)
template retry(bod) =
template retry(bod) {.dirty.} =
var session: Session
var retrySuccess = false
for i in 0 ..< maxRetries:
try:
session = nil
bod
retrySuccess = true
break
except RateLimitError:
echo "[sessions] Rate limited, retrying ", req.cookie.endpoint,
" request (", i, "/", maxRetries, ")..."
let api = if session.isNil: req.cookie.endpoint
else: req.endpoint(session)
if session.isNil:
echo "[sessions] Rate limited, retrying ", api,
" request (", i, "/", maxRetries, ")..."
else:
echo "[sessions] Rate limited, retrying ", api,
" request (", i, "/", maxRetries, ")..., session: ", session.pretty
session = nil
if retryDelayMs > 0:
await sleepAsync(retryDelayMs)
if not retrySuccess:
raise rateLimitError()
proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
retry:
var
body: string
session = await getAndValidateSession(req)
var body: string
session = await getAndValidateSession(req)
let url = req.toUrl(session.kind)
@ -200,22 +220,22 @@ proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
if body.startsWith('{') or body.startsWith('['):
result = parseJson(body)
else:
echo resp.status, ": ", body, " --- url: ", url
echo resp.status, ": ", body, " --- url: ", url, ", session: ", session.pretty
result = newJNull()
let error = result.getError
if error != null and error notin errorsToSkip:
echo "Fetch error, API: ", url.path, ", error: ", error
echo "Fetch error, API: ", url.path, ", error: ", error, ", session: ", session.pretty
if error in {expiredToken, badToken, locked}:
invalidate(session)
raise rateLimitError()
proc fetchRaw*(req: ApiReq): Future[string] {.async.} =
retry:
var session = await getAndValidateSession(req)
session = await getAndValidateSession(req)
let url = req.toUrl(session.kind)
fetchImpl result:
if not (result.startsWith('{') or result.startsWith('[')):
echo resp.status, ": ", result, " --- url: ", url
echo resp.status, ": ", result, " --- url: ", url, ", session: ", session.pretty
result.setLen(0)

View file

@ -18,7 +18,7 @@ proc setMaxConcurrentReqs*(reqs: int) =
template log(str: varargs[string, `$`]) =
echo "[sessions] ", str.join("")
proc endpoint(req: ApiReq; session: Session): string =
proc endpoint*(req: ApiReq; session: Session): string =
case session.kind
of oauth: req.oauth.endpoint
of cookie: req.cookie.endpoint
@ -50,6 +50,8 @@ proc getSessionPoolHealth*(): JsonNode =
oldest = now.int64
newest = 0'i64
average = 0'i64
oauthTotal, cookieTotal = 0
oauthLimited, cookieLimited = 0
for session in sessionPool:
let created = snowflakeToEpoch(session.id)
@ -59,8 +61,15 @@ proc getSessionPoolHealth*(): JsonNode =
oldest = created
average += created
case session.kind
of oauth: inc oauthTotal
of cookie: inc cookieTotal
if session.limited:
limited.incl session.id
case session.kind
of oauth: inc oauthLimited
of cookie: inc cookieLimited
for api in session.apis.keys:
let
@ -84,6 +93,8 @@ proc getSessionPoolHealth*(): JsonNode =
"sessions": %*{
"total": sessionPool.len,
"limited": limited.card,
"oauth": %*{"total": oauthTotal, "limited": oauthLimited},
"cookie": %*{"total": cookieTotal, "limited": cookieLimited},
"oldest": $fromUnix(oldest),
"newest": $fromUnix(newest),
"average": $fromUnix(average)
@ -100,6 +111,7 @@ proc getSessionPoolDebug*(): JsonNode =
for session in sessionPool:
let sessionJson = %*{
"kind": $session.kind,
"apis": newJObject(),
"pending": session.pending,
}
@ -173,7 +185,10 @@ proc getSession*(req: ApiReq): Future[Session] {.async.} =
if not result.isNil and result.isReady(req):
inc result.pending
else:
log "no sessions available for API: ", req.cookie.endpoint
if result.isNil:
log "no sessions available for API: ", req.cookie.endpoint
else:
log "no sessions available for API: ", req.endpoint(result), ", last tried: ", result.pretty
raise noSessionsError()
proc setLimited*(session: Session; req: ApiReq) =

Some files were not shown because too many files have changed in this diff Show more