mirror of
https://github.com/zedeus/nitter
synced 2026-11-22 12:57:19 +00:00
Merge branch 'zedeus:master' into master
This commit is contained in:
commit
106df59291
71 changed files with 3652 additions and 481 deletions
92
.github/workflows/build-docker.yml
vendored
92
.github/workflows/build-docker.yml
vendored
|
|
@ -7,55 +7,105 @@ on:
|
|||
branches:
|
||||
- master
|
||||
|
||||
concurrency:
|
||||
group: docker-publish-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
IMAGE: zedeus/nitter
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
uses: ./.github/workflows/run-tests.yml
|
||||
secrets: inherit
|
||||
|
||||
build-docker-amd64:
|
||||
build:
|
||||
needs: [tests]
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- runner: ubuntu-24.04
|
||||
platform: linux/amd64
|
||||
- runner: ubuntu-24.04-arm
|
||||
platform: linux/arm64
|
||||
runs-on: ${{ matrix.runner }}
|
||||
steps:
|
||||
- name: Prepare platform name
|
||||
run: echo "PLATFORM_PAIR=${platform//\//-}" >> "$GITHUB_ENV"
|
||||
env:
|
||||
platform: ${{ matrix.platform }}
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
version: latest
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Build and push AMD64 Docker image
|
||||
uses: docker/build-push-action@v3
|
||||
|
||||
- name: Build and push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}
|
||||
platforms: ${{ matrix.platform }}
|
||||
outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
|
||||
provenance: false
|
||||
sbom: false
|
||||
|
||||
build-docker-arm64:
|
||||
needs: [tests]
|
||||
runs-on: ubuntu-24.04-arm
|
||||
- name: Export digest
|
||||
run: |
|
||||
mkdir -p "${{ runner.temp }}/digests"
|
||||
digest="${{ steps.build.outputs.digest }}"
|
||||
touch "${{ runner.temp }}/digests/${digest#sha256:}"
|
||||
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: digests-${{ env.PLATFORM_PAIR }}
|
||||
path: ${{ runner.temp }}/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Combine the per-arch digests into one multi-arch manifest so that
|
||||
# `docker pull zedeus/nitter:latest` serves the right image on any CPU.
|
||||
merge:
|
||||
needs: [build]
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: ${{ runner.temp }}/digests
|
||||
pattern: digests-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
version: latest
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Build and push ARM64 Docker image
|
||||
uses: docker/build-push-action@v3
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.arm64
|
||||
platforms: linux/arm64
|
||||
push: true
|
||||
tags: zedeus/nitter:latest-arm64,zedeus/nitter:${{ github.sha }}-arm64
|
||||
|
||||
- name: Create manifest list and push
|
||||
working-directory: ${{ runner.temp }}/digests
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
-t ${{ env.IMAGE }}:latest \
|
||||
-t ${{ env.IMAGE }}:latest-arm64 \
|
||||
-t ${{ env.IMAGE }}:${{ github.sha }} \
|
||||
$(printf '${{ env.IMAGE }}@sha256:%s ' *)
|
||||
|
||||
- name: Inspect image
|
||||
run: docker buildx imagetools inspect ${{ env.IMAGE }}:${{ github.sha }}
|
||||
|
|
|
|||
25
.github/workflows/run-tests.yml
vendored
25
.github/workflows/run-tests.yml
vendored
|
|
@ -32,10 +32,12 @@ jobs:
|
|||
id: cache-nimble
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ~/.nimble
|
||||
key: ${{ matrix.nim }}-nimble-v2-${{ hashFiles('*.nimble') }}
|
||||
path: |
|
||||
~/.nimble/pkgcache
|
||||
~/.nimble/packages_official.json
|
||||
key: ${{ matrix.nim }}-nimble-v6-${{ hashFiles('*.nimble') }}
|
||||
restore-keys: |
|
||||
${{ matrix.nim }}-nimble-v2-
|
||||
${{ matrix.nim }}-nimble-v6-
|
||||
|
||||
- name: Setup Nim
|
||||
uses: jiro4989/setup-nim-action@v2
|
||||
|
|
@ -103,10 +105,12 @@ jobs:
|
|||
- name: Cache Nimble Dependencies
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ~/.nimble
|
||||
key: 2.2.x-nimble-v2-${{ hashFiles('*.nimble') }}
|
||||
path: |
|
||||
~/.nimble/pkgcache
|
||||
~/.nimble/packages_official.json
|
||||
key: 2.2.x-nimble-v6-${{ hashFiles('*.nimble') }}
|
||||
restore-keys: |
|
||||
2.2.x-nimble-v2-
|
||||
2.2.x-nimble-v6-
|
||||
|
||||
- name: Setup Nim
|
||||
uses: jiro4989/setup-nim-action@v2
|
||||
|
|
@ -115,6 +119,9 @@ jobs:
|
|||
use-nightlies: true
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Install Nimble dependencies
|
||||
run: nimble install -y --depsOnly
|
||||
|
||||
- name: Download 2.2.x build artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
|
|
@ -130,10 +137,8 @@ jobs:
|
|||
sed -i 's/enableDebug = false/enableDebug = true/g' nitter.conf
|
||||
sed -i 's/maxRetries = 1/maxRetries = 10/g' nitter.conf
|
||||
|
||||
# Run both Nimble tasks concurrently
|
||||
nim r tools/rendermd.nim &
|
||||
nim r tools/gencss.nim &
|
||||
wait
|
||||
nim r tools/rendermd.nim
|
||||
nim r tools/gencss.nim
|
||||
|
||||
echo '${{ secrets.SESSIONS }}' | head -n1
|
||||
echo '${{ secrets.SESSIONS }}' > ./sessions.jsonl
|
||||
|
|
|
|||
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -15,3 +15,6 @@ sessions.json*
|
|||
dump.rdb
|
||||
*.bak
|
||||
/tools/*.json*
|
||||
nimbledeps/
|
||||
nimble.paths
|
||||
nimble.develop
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
FROM nimlang/nim:2.2.0-alpine-regular as nim
|
||||
FROM nimlang/nim:2.2.6-alpine-regular as nim
|
||||
LABEL maintainer="setenforce@protonmail.com"
|
||||
|
||||
RUN apk --no-cache add libsass-dev pcre
|
||||
|
|
@ -15,7 +15,7 @@ RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
|
|||
|
||||
FROM alpine:latest
|
||||
WORKDIR /src/
|
||||
RUN apk --no-cache add pcre ca-certificates
|
||||
RUN apk --no-cache add pcre ca-certificates openssl
|
||||
COPY --from=nim /src/nitter/nitter ./
|
||||
COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
|
||||
COPY --from=nim /src/nitter/public ./public
|
||||
|
|
|
|||
|
|
@ -1,25 +0,0 @@
|
|||
FROM alpine:3.20.6 as nim
|
||||
LABEL maintainer="setenforce@protonmail.com"
|
||||
|
||||
RUN apk --no-cache add libsass-dev pcre gcc git libc-dev nim nimble
|
||||
|
||||
WORKDIR /src/nitter
|
||||
|
||||
COPY nitter.nimble .
|
||||
RUN nimble install -y --depsOnly
|
||||
|
||||
COPY . .
|
||||
RUN nimble build -d:danger -d:lto -d:strip --mm:refc \
|
||||
&& nimble scss \
|
||||
&& nimble md
|
||||
|
||||
FROM alpine:3.20.6
|
||||
WORKDIR /src/
|
||||
RUN apk --no-cache add pcre ca-certificates openssl
|
||||
COPY --from=nim /src/nitter/nitter ./
|
||||
COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
|
||||
COPY --from=nim /src/nitter/public ./public
|
||||
EXPOSE 8080
|
||||
RUN adduser -h /src/ -D -s /bin/sh nitter
|
||||
USER nitter
|
||||
CMD ./nitter
|
||||
28
README.md
28
README.md
|
|
@ -104,9 +104,9 @@ along with the scss and md files.
|
|||
# su nitter
|
||||
$ git clone https://github.com/zedeus/nitter
|
||||
$ cd nitter
|
||||
$ nimble build -d:danger --mm:refc
|
||||
$ nimble scss
|
||||
$ nimble md
|
||||
$ nimble -l build -d:danger --mm:refc
|
||||
$ nimble -l scss
|
||||
$ nimble -l md
|
||||
$ cp nitter.example.conf nitter.conf
|
||||
```
|
||||
|
||||
|
|
@ -123,12 +123,23 @@ performance reasons.
|
|||
|
||||
Page for the Docker image: https://hub.docker.com/r/zedeus/nitter
|
||||
|
||||
#### NOTE: For ARM64 support, please use the separate ARM64 docker image: [`zedeus/nitter:latest-arm64`](https://hub.docker.com/r/zedeus/nitter/tags).
|
||||
#### NOTE: The published image is multi-arch — `zedeus/nitter:latest` runs natively on both `amd64` and `arm64`.
|
||||
|
||||
To run Nitter with Docker, you'll need to install and run Redis separately
|
||||
before you can run the container. See below for how to also run Redis using
|
||||
Docker.
|
||||
|
||||
First create your config file. The Docker commands mount it into the container,
|
||||
so it has to exist on the host beforehand. If you've cloned the repo:
|
||||
|
||||
```bash
|
||||
cp nitter.example.conf nitter.conf
|
||||
```
|
||||
|
||||
If you're using the prebuilt image without a local clone, download
|
||||
[`nitter.example.conf`](https://raw.githubusercontent.com/zedeus/nitter/master/nitter.example.conf)
|
||||
and save it as `nitter.conf` instead.
|
||||
|
||||
To build and run Nitter in Docker:
|
||||
|
||||
```bash
|
||||
|
|
@ -136,8 +147,6 @@ docker build -t nitter:latest .
|
|||
docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest
|
||||
```
|
||||
|
||||
Note: For ARM64, use this Dockerfile: [`Dockerfile.arm64`](https://github.com/zedeus/nitter/blob/master/Dockerfile.arm64).
|
||||
|
||||
A prebuilt Docker image is provided as well:
|
||||
|
||||
```bash
|
||||
|
|
@ -151,8 +160,11 @@ Change `redisHost` from `localhost` to `nitter-redis` in `nitter.conf`, then run
|
|||
docker-compose up -d
|
||||
```
|
||||
|
||||
Note the Docker commands expect a `nitter.conf` file in the directory you run
|
||||
them.
|
||||
Note the Docker commands mount `nitter.conf` (and `sessions.jsonl` for
|
||||
docker-compose) from the directory you run them in. If a mounted file doesn't
|
||||
exist, Docker silently creates a directory in its place and the container fails
|
||||
with `not a directory: Are you trying to mount a directory onto a file`. Remove
|
||||
that directory and create the file as shown above.
|
||||
|
||||
### systemd
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
version: "3"
|
||||
|
||||
services:
|
||||
|
||||
nitter:
|
||||
|
|
@ -11,3 +11,7 @@ warning("HoleEnumConv", off)
|
|||
hint("XDeclaredButNotUsed", off)
|
||||
hint("XCannotRaiseY", off)
|
||||
hint("User", off)
|
||||
# begin Nimble config (version 2)
|
||||
when withDir(thisDir(), system.fileExists("nimble.paths")):
|
||||
include "nimble.paths"
|
||||
# end Nimble config
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ redisMaxConnections = 30
|
|||
# you receive tons of requests per second
|
||||
|
||||
[Config]
|
||||
hmacKey = "secretkey" # random key for cryptographic signing of video urls
|
||||
hmacKey = "secretkey" # CHANGE THIS to a unique random value (e.g. `openssl rand -hex 32`); signs media urls
|
||||
base64Media = false # use base64 encoding for proxied media urls
|
||||
enableRSS = true # master switch, set to false to disable all RSS feeds
|
||||
enableRSSUserTweets = true # /@user/rss
|
||||
|
|
|
|||
|
|
@ -11,19 +11,18 @@ bin = @["nitter"]
|
|||
# Dependencies
|
||||
|
||||
requires "nim >= 2.0.0"
|
||||
requires "jester#baca3f"
|
||||
requires "karax#5cf360c"
|
||||
requires "sass#7dfdd03"
|
||||
requires "nimcrypto#a079df9"
|
||||
requires "markdown#158efe3"
|
||||
requires "jester == 0.6.0"
|
||||
requires "karax == 1.5.0"
|
||||
requires "sass == 0.2.0"
|
||||
requires "nimcrypto == 0.7.3"
|
||||
requires "markdown == 0.8.8"
|
||||
requires "packedjson#9e6fbb6"
|
||||
requires "supersnappy#6c94198"
|
||||
requires "redpool#8b7c1db"
|
||||
requires "https://github.com/zedeus/redis#d0a0e6f"
|
||||
requires "zippy#ca5989a"
|
||||
requires "flatty#e668085"
|
||||
requires "jsony#1de1f08"
|
||||
requires "oauth#b8c163b"
|
||||
requires "supersnappy == 2.1.4"
|
||||
requires "redpool == 0.2.2"
|
||||
requires "zippy == 0.10.19"
|
||||
requires "flatty == 0.4.0"
|
||||
requires "jsony == 1.1.6"
|
||||
requires "oauth == 0.11"
|
||||
|
||||
# Tasks
|
||||
|
||||
|
|
|
|||
71
public/css/fontello.css
vendored
71
public/css/fontello.css
vendored
|
|
@ -1,12 +1,12 @@
|
|||
@font-face {
|
||||
font-family: "fontello";
|
||||
src: url("/fonts/fontello.eot?49059696");
|
||||
src: url("/fonts/fontello.eot?59696369");
|
||||
src:
|
||||
url("/fonts/fontello.eot?49059696#iefix") format("embedded-opentype"),
|
||||
url("/fonts/fontello.woff2?49059696") format("woff2"),
|
||||
url("/fonts/fontello.woff?49059696") format("woff"),
|
||||
url("/fonts/fontello.ttf?49059696") format("truetype"),
|
||||
url("/fonts/fontello.svg?49059696#fontello") format("svg");
|
||||
url("/fonts/fontello.eot?59696369#iefix") format("embedded-opentype"),
|
||||
url("/fonts/fontello.woff2?59696369") format("woff2"),
|
||||
url("/fonts/fontello.woff?59696369") format("woff"),
|
||||
url("/fonts/fontello.ttf?59696369") format("truetype"),
|
||||
url("/fonts/fontello.svg?59696369#fontello") format("svg");
|
||||
font-weight: normal;
|
||||
font-style: normal;
|
||||
}
|
||||
|
|
@ -36,113 +36,118 @@
|
|||
-moz-osx-font-smoothing: grayscale;
|
||||
}
|
||||
|
||||
.icon-group:before {
|
||||
content: "\e0c3";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
.icon-views:before {
|
||||
content: "\e800";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-heart:before {
|
||||
content: "\e801";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-quote:before {
|
||||
content: "\e802";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-comment:before {
|
||||
content: "\e803";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
.icon-group:before {
|
||||
content: "\e804";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-play:before {
|
||||
content: "\e805";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-link:before {
|
||||
content: "\e806";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-calendar:before {
|
||||
content: "\e807";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-location:before {
|
||||
content: "\e808";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-picture:before {
|
||||
content: "\e809";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-lock:before {
|
||||
content: "\e80a";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-down:before {
|
||||
content: "\e80b";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-retweet:before {
|
||||
content: "\e80c";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-search:before {
|
||||
content: "\e80d";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-pin:before {
|
||||
content: "\e80e";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-cog:before {
|
||||
content: "\e80f";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-rss:before {
|
||||
content: "\e810";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-ok:before {
|
||||
content: "\e811";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
.icon-attention:before {
|
||||
/* '' */
|
||||
.icon-attention-circled:before {
|
||||
content: "\e812";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-download-alt:before {
|
||||
content: "\e813";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
.icon-circle:before {
|
||||
content: "\f111";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-info:before {
|
||||
content: "\f128";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
.icon-bird:before {
|
||||
content: "\f309";
|
||||
}
|
||||
|
||||
/* '' */
|
||||
/* '' */
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -6,6 +6,8 @@
|
|||
<font id="fontello" horiz-adv-x="1000" >
|
||||
<font-face font-family="fontello" font-weight="400" font-stretch="normal" units-per-em="1000" ascent="850" descent="-150" />
|
||||
<missing-glyph horiz-adv-x="1000" />
|
||||
<glyph glyph-name="group" unicode="" d="M0 106l0 134q0 26 18 32l171 80q-66 39-68 131 0 56 35 103 37 41 90 43 31 0 63-19-49-125 23-237-12-11-25-19l-114-55q-48-23-52-84l0-143-114 0q-25 0-27 34z m193-59l0 168q0 27 22 37l152 70 57 28q-37 23-60 66t-22 94q0 76 46 130t110 54 109-54 45-130q0-105-78-158l61-30 146-70q24-10 24-37l0-168q-2-37-37-41l-541 0q-14 2-24 14t-10 27z m473 330q68 106 22 231 31 19 66 21 49 0 90-43 35-41 35-103 0-82-65-131l168-80q18-10 18-32l0-134q0-32-27-34l-118 0 0 143q0 57-50 84l-110 53q-15 8-29 25z" horiz-adv-x="1000" />
|
||||
|
||||
<glyph glyph-name="views" unicode="" d="M180 516l0-538-180 0 0 538 180 0z m250-138l0-400-180 0 0 400 180 0z m250 344l0-744-180 0 0 744 180 0z" horiz-adv-x="680" />
|
||||
|
||||
<glyph glyph-name="heart" unicode="" d="M790 644q70-64 70-156t-70-158l-360-330-360 330q-70 66-70 158t70 156q62 58 151 58t153-58l56-52 58 52q62 58 150 58t152-58z" horiz-adv-x="860" />
|
||||
|
|
@ -14,8 +16,6 @@
|
|||
|
||||
<glyph glyph-name="comment" unicode="" d="M1000 350q0-97-67-179t-182-130-251-48q-39 0-81 4-110-97-257-135-27-8-63-12-10-1-17 5t-10 16v1q-2 2 0 6t1 6 2 5l4 5t4 5 4 5q4 5 17 19t20 22 17 22 18 28 15 33 15 42q-88 50-138 123t-51 157q0 73 40 139t106 114 160 76 194 28q136 0 251-48t182-130 67-179z" horiz-adv-x="1000" />
|
||||
|
||||
<glyph glyph-name="group" unicode="" d="M0 106l0 134q0 26 18 32l171 80q-66 39-68 131 0 56 35 103 37 41 90 43 31 0 63-19-49-125 23-237-12-11-25-19l-114-55q-48-23-52-84l0-143-114 0q-25 0-27 34z m193-59l0 168q0 27 22 37l152 70 57 28q-37 23-60 66t-22 94q0 76 46 130t110 54 109-54 45-130q0-105-78-158l61-30 146-70q24-10 24-37l0-168q-2-37-37-41l-541 0q-14 2-24 14t-10 27z m473 330q68 106 22 231 31 19 66 21 49 0 90-43 35-41 35-103 0-82-65-131l168-80q18-10 18-32l0-134q0-32-27-34l-118 0 0 143q0 57-50 84l-110 53q-15 8-29 25z" horiz-adv-x="1000" />
|
||||
|
||||
<glyph glyph-name="play" unicode="" d="M772 333l-741-412q-13-7-22-2t-9 20v822q0 14 9 20t22-2l741-412q13-7 13-17t-13-17z" horiz-adv-x="785.7" />
|
||||
|
||||
<glyph glyph-name="link" unicode="" d="M294 116q14 14 34 14t36-14q32-34 0-70l-42-40q-56-56-132-56-78 0-134 56t-56 132q0 78 56 134l148 148q70 68 144 77t128-43q16-16 16-36t-16-36q-36-32-70 0-50 48-132-34l-148-146q-26-26-26-64t26-62q26-26 63-26t63 26z m450 574q56-56 56-132 0-78-56-134l-158-158q-74-72-150-72-62 0-112 50-14 14-14 34t14 36q14 14 35 14t35-14q50-48 122 24l158 156q28 28 28 64 0 38-28 62-24 26-56 31t-60-21l-50-50q-16-14-36-14t-34 14q-34 34 0 70l50 50q54 54 127 51t129-61z" horiz-adv-x="800" />
|
||||
|
|
@ -44,6 +44,8 @@
|
|||
|
||||
<glyph glyph-name="attention-circled" unicode="" d="M429 779q116 0 215-58t156-156 57-215-57-215-156-156-215-58-216 58-155 156-58 215 58 215 155 156 216 58z m71-696v106q0 8-5 13t-12 5h-107q-8 0-13-5t-6-13v-106q0-8 6-13t13-6h107q7 0 12 6t5 13z m-1 192l10 346q0 7-6 10-5 5-13 5h-123q-8 0-13-5-6-3-6-10l10-346q0-6 5-10t14-4h103q8 0 13 4t6 10z" horiz-adv-x="857.1" />
|
||||
|
||||
<glyph glyph-name="download-alt" unicode="" d="M0-150l0 135 1000 0 0-135-1000 0z m88 586l228 0 0 414 370 0 0-414 228 0-414-385z" horiz-adv-x="1000" />
|
||||
|
||||
<glyph glyph-name="circle" unicode="" d="M857 350q0-117-57-215t-156-156-215-58-216 58-155 156-58 215 58 215 155 156 216 58 215-58 156-156 57-215z" horiz-adv-x="857.1" />
|
||||
|
||||
<glyph glyph-name="info" unicode="" d="M393 149v-134q0-9-7-15t-15-7h-134q-9 0-16 7t-7 15v134q0 9 7 16t16 6h134q9 0 15-6t7-16z m176 335q0-30-8-56t-20-43-31-33-32-25-34-19q-23-13-38-37t-15-37q0-10-7-18t-16-9h-134q-8 0-14 11t-6 20v26q0 46 37 87t79 60q33 16 47 32t14 42q0 24-26 41t-60 18q-36 0-60-16-20-14-60-64-7-9-17-9-7 0-14 4l-91 70q-8 6-9 14t3 16q89 148 259 148 45 0 90-17t81-46 59-72 23-88z" horiz-adv-x="571.4" />
|
||||
|
|
|
|||
|
Before Width: | Height: | Size: 7.3 KiB After Width: | Height: | Size: 7.4 KiB |
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -1,27 +1,30 @@
|
|||
// @license http://www.gnu.org/licenses/agpl-3.0.html AGPL-3.0
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
function playVideo(overlay) {
|
||||
const video = overlay.parentElement.querySelector('video');
|
||||
const url = video.getAttribute("data-url");
|
||||
const startTime = parseFloat(video.getAttribute("data-start") || "0");
|
||||
video.setAttribute("controls", "");
|
||||
function playMedia(overlay, tagName) {
|
||||
const media = overlay.parentElement.querySelector(tagName);
|
||||
const url = media.getAttribute("data-url");
|
||||
const startTime = parseFloat(media.getAttribute("data-start") || "0");
|
||||
media.setAttribute("controls", "");
|
||||
overlay.style.display = "none";
|
||||
|
||||
if (Hls.isSupported()) {
|
||||
var hls = new Hls({autoStartLoad: false});
|
||||
hls.loadSource(url);
|
||||
hls.attachMedia(video);
|
||||
hls.attachMedia(media);
|
||||
hls.on(Hls.Events.MANIFEST_PARSED, function () {
|
||||
hls.loadLevel = hls.levels.length - 1;
|
||||
hls.startLoad(startTime);
|
||||
video.play();
|
||||
media.play();
|
||||
});
|
||||
} else if (video.canPlayType('application/vnd.apple.mpegurl')) {
|
||||
video.src = url;
|
||||
video.addEventListener('canplay', function() {
|
||||
if (startTime > 0) video.currentTime = startTime;
|
||||
video.play();
|
||||
} else if (media.canPlayType('application/vnd.apple.mpegurl')) {
|
||||
media.src = url;
|
||||
media.addEventListener('canplay', function() {
|
||||
if (startTime > 0) media.currentTime = startTime;
|
||||
media.play();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function playVideo(overlay) { playMedia(overlay, 'video'); }
|
||||
function playAudio(overlay) { playMedia(overlay, 'audio'); }
|
||||
// @license-end
|
||||
|
|
|
|||
174
src/api.nim
174
src/api.nim
|
|
@ -2,7 +2,7 @@
|
|||
import asyncdispatch, httpclient, strutils, sequtils, sugar
|
||||
import packedjson
|
||||
import types, query, formatters, consts, apiutils, parser, utils
|
||||
import experimental/parser as newParser
|
||||
import experimental/parser
|
||||
|
||||
# Helper to generate params object for GraphQL requests
|
||||
proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
|
||||
|
|
@ -11,13 +11,18 @@ proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
|
|||
if fieldToggles.len > 0:
|
||||
result.add ("fieldToggles", fieldToggles)
|
||||
|
||||
proc apiUrl(endpoint, variables: string; fieldToggles = ""): ApiUrl =
|
||||
return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles))
|
||||
proc apiUrl(endpoint, variables: string; fieldToggles = ""; skipTid = false): ApiUrl =
|
||||
return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles), skipTid: skipTid)
|
||||
|
||||
proc apiReq(endpoint, variables: string; fieldToggles = ""): ApiReq =
|
||||
let url = apiUrl(endpoint, variables, fieldToggles)
|
||||
proc apiReq(endpoint, variables: string; fieldToggles = ""; skipTid = false): ApiReq =
|
||||
let url = apiUrl(endpoint, variables, fieldToggles, skipTid)
|
||||
return ApiReq(cookie: url, oauth: url)
|
||||
|
||||
proc cursorParam(after: string): string =
|
||||
## JSON-escape the user-supplied cursor so it cannot break out of the GraphQL
|
||||
## variables object (same input-validation class as the #1411 media SSRF).
|
||||
if after.len > 0: "\"cursor\":" & $(%after) & "," else: ""
|
||||
|
||||
proc mediaUrl(id, cursor: string; count=20): ApiReq =
|
||||
result = ApiReq(
|
||||
cookie: apiUrl(graphUserMedia, userMediaVars % [id, cursor, $count]),
|
||||
|
|
@ -25,33 +30,24 @@ proc mediaUrl(id, cursor: string; count=20): ApiReq =
|
|||
)
|
||||
|
||||
proc userTweetsUrl(id: string; cursor: string): ApiReq =
|
||||
result = ApiReq(
|
||||
# cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
|
||||
oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"])
|
||||
)
|
||||
# might change this in the future pending testing
|
||||
result.cookie = result.oauth
|
||||
return apiReq(graphUserTweetsV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles)
|
||||
|
||||
proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq =
|
||||
let cookieVars = userTweetsAndRepliesVars % [id, cursor]
|
||||
result = ApiReq(
|
||||
cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
|
||||
oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"])
|
||||
)
|
||||
return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"], userTweetsFieldToggles, skipTid=true)
|
||||
|
||||
proc tweetDetailUrl(id: string; cursor: string): ApiReq =
|
||||
let cookieVars = tweetDetailVars % [id, cursor]
|
||||
result = ApiReq(
|
||||
# cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
|
||||
cookie: apiUrl(graphTweet, tweetVars % [id, cursor]),
|
||||
oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
|
||||
)
|
||||
return apiReq(graphTweet, tweetVars % [id, cursor])
|
||||
# let cookieVars = tweetDetailVars % [id, cursor]
|
||||
# result = ApiReq(
|
||||
# cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
|
||||
# oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
|
||||
# )
|
||||
|
||||
proc userUrl(username: string): ApiReq =
|
||||
let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
|
||||
let cookieVars = $(%*{"screen_name": username, "withGrokTranslatedBio": false})
|
||||
result = ApiReq(
|
||||
cookie: apiUrl(graphUser, cookieVars, tweetDetailFieldToggles),
|
||||
oauth: apiUrl(graphUserV2, """{"screen_name": "$1"}""" % username)
|
||||
oauth: apiUrl(graphUserV2, $(%*{"screen_name": username}))
|
||||
)
|
||||
|
||||
proc getGraphUser*(username: string): Future[User] {.async.} =
|
||||
|
|
@ -69,7 +65,7 @@ proc getGraphUserById*(id: string): Future[User] {.async.} =
|
|||
proc getAboutAccount*(username: string): Future[AccountInfo] {.async.} =
|
||||
if username.len == 0: return
|
||||
let
|
||||
url = apiReq(graphAboutAccount, """{"screenName":"$1"}""" % username)
|
||||
url = apiReq(graphAboutAccount, $(%*{"screenName": username}))
|
||||
js = await fetch(url)
|
||||
result = parseAboutAccount(js)
|
||||
|
||||
|
|
@ -80,7 +76,7 @@ proc restReq(endpoint: string; params: seq[(string, string)] = @[]): ApiReq =
|
|||
proc getBroadcastInfo*(id: string): Future[Broadcast] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
req = apiReq(graphBroadcast, """{"id":"$1"}""" % id)
|
||||
req = apiReq(graphBroadcast, $(%*{"id": id}))
|
||||
js = await fetch(req)
|
||||
result = parseBroadcastInfo(js)
|
||||
|
||||
|
|
@ -92,10 +88,23 @@ proc fetchBroadcastStream*(mediaKey: string): Future[string] {.async.} =
|
|||
result = streamJs{"source", "noRedirectPlaybackUrl"}.getStr(
|
||||
streamJs{"source", "location"}.getStr)
|
||||
|
||||
proc getAudioSpace*(id: string): Future[AudioSpace] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
variables = %*{
|
||||
"id": id,
|
||||
"isMetatagsQuery": false,
|
||||
"withReplays": true,
|
||||
"withListeners": true
|
||||
}
|
||||
req = apiReq(graphAudioSpace, $variables)
|
||||
js = await fetch(req)
|
||||
result = parseAudioSpace(js)
|
||||
|
||||
proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
cursor = cursorParam(after)
|
||||
url = case kind
|
||||
of TimelineKind.tweets: userTweetsUrl(id, cursor)
|
||||
of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor)
|
||||
|
|
@ -103,10 +112,61 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi
|
|||
js = await fetch(url)
|
||||
result = parseGraphTimeline(js, after)
|
||||
|
||||
proc getGraphCommunity*(id: string): Future[Community] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
url = apiReq(graphCommunity, $(%*{"communityId": id}))
|
||||
js = await fetch(url)
|
||||
result = parseGraphCommunity(js)
|
||||
|
||||
proc getGraphCommunityTweets*(id: string; rankingMode: string; after=""): Future[Timeline] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = cursorParam(after)
|
||||
url = apiReq(graphCommunityTweets, communityTweetsVars % [id, cursor, rankingMode])
|
||||
js = await fetch(url)
|
||||
result = parseGraphCommunityTimeline(js, after)
|
||||
|
||||
proc getGraphCommunityMedia*(id: string; after=""): Future[Timeline] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = cursorParam(after)
|
||||
url = apiReq(graphCommunityMedia, communityMediaVars % [id, cursor])
|
||||
js = await fetch(url)
|
||||
result = parseGraphCommunityTimeline(js, after)
|
||||
|
||||
proc communitySliceReq(endpoint, variables: string): ApiReq =
|
||||
let url = ApiUrl(endpoint: endpoint, params: @[("variables", variables)])
|
||||
ApiReq(cookie: url, oauth: url)
|
||||
|
||||
proc getGraphCommunityMembers*(id: string; after=""): Future[Result[User]] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: $(%after) else: "null"
|
||||
url = communitySliceReq(graphCommunityMembers, communityMembersVars % [id, cursor])
|
||||
js = await fetch(url)
|
||||
result = parseGraphCommunityMembers(js, after)
|
||||
|
||||
proc getGraphCommunityModerators*(id: string): Future[Result[User]] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
url = communitySliceReq(graphCommunityModerators, communityMembersVars % [id, "null"])
|
||||
js = await fetch(url)
|
||||
result = parseGraphCommunityMembers(js)
|
||||
|
||||
proc getGraphCommunityHashtags*(id, hashtag: string; after=""): Future[Timeline] {.async.} =
|
||||
if id.len == 0 or hashtag.len == 0: return
|
||||
let
|
||||
safeTag = multiReplace(hashtag, ("\"", ""), ("\\", ""))
|
||||
cursor = cursorParam(after)
|
||||
url = apiReq(graphCommunityHashtags, communityHashtagsVars % [id, cursor, safeTag])
|
||||
js = await fetch(url)
|
||||
result = parseGraphCommunityTimeline(js, after)
|
||||
|
||||
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
cursor = cursorParam(after)
|
||||
url = apiReq(graphListTweets, restIdVars % [id, cursor, "20"])
|
||||
js = await fetch(url)
|
||||
result = parseGraphTimeline(js, after).tweets
|
||||
|
|
@ -120,7 +180,7 @@ proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
|
|||
|
||||
proc getGraphList*(id: string): Future[List] {.async.} =
|
||||
let
|
||||
url = apiReq(graphListById, """{"listId": "$1"}""" % id)
|
||||
url = apiReq(graphListById, $(%*{"listId": id}))
|
||||
js = await fetch(url)
|
||||
result = parseGraphList(js)
|
||||
|
||||
|
|
@ -141,17 +201,39 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
|
|||
js = await fetchRaw(url)
|
||||
result = parseGraphListMembers(js, after)
|
||||
|
||||
proc getGraphUserConnections(userId: string; endpoint: string; kind: QueryKind;
|
||||
after=""): Future[Result[User]] {.async.} =
|
||||
if userId.len == 0: return
|
||||
var variables = %*{
|
||||
"userId": userId,
|
||||
"count": 20,
|
||||
"includePromotedContent": false,
|
||||
"withGrokTranslatedBio": true
|
||||
}
|
||||
if after.len > 0:
|
||||
variables["cursor"] = %after
|
||||
let
|
||||
url = apiReq(endpoint, $variables)
|
||||
js = await fetchRaw(url)
|
||||
result = parseGraphFollowers(js, after, kind)
|
||||
|
||||
proc getGraphFollowers*(userId: string; after=""): Future[Result[User]] {.async.} =
|
||||
result = await getGraphUserConnections(userId, graphFollowers, followers, after)
|
||||
|
||||
proc getGraphFollowing*(userId: string; after=""): Future[Result[User]] {.async.} =
|
||||
result = await getGraphUserConnections(userId, graphFollowing, following, after)
|
||||
|
||||
proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
url = apiReq(graphTweetResult, """{"rest_id": "$1"}""" % id)
|
||||
url = apiReq(graphTweetResult, $(%*{"rest_id": id}))
|
||||
js = await fetch(url)
|
||||
result = parseGraphTweetResult(js)
|
||||
|
||||
proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
cursor = cursorParam(after)
|
||||
js = await fetch(tweetDetailUrl(id, cursor))
|
||||
result = parseGraphConversation(js, id)
|
||||
|
||||
|
|
@ -184,13 +266,13 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
|
|||
var
|
||||
variables = %*{
|
||||
"rawQuery": q,
|
||||
"query_source": "typedQuery",
|
||||
"count": 20,
|
||||
"querySource": "typed_query",
|
||||
"product": "Latest",
|
||||
"withDownvotePerspective": false,
|
||||
"withReactionsMetadata": false,
|
||||
"withReactionsPerspective": false
|
||||
"withGrokTranslatedBio":true,
|
||||
"withQuickPromoteEligibilityTweetFields":false
|
||||
}
|
||||
|
||||
if after.len > 0 and maxId.len == 0:
|
||||
variables["cursor"] = % after
|
||||
let
|
||||
|
|
@ -212,12 +294,11 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
|
|||
var
|
||||
variables = %*{
|
||||
"rawQuery": query.text,
|
||||
"query_source": "typedQuery",
|
||||
"count": 20,
|
||||
"querySource": "typed_query",
|
||||
"product": "People",
|
||||
"withDownvotePerspective": false,
|
||||
"withReactionsMetadata": false,
|
||||
"withReactionsPerspective": false
|
||||
"withGrokTranslatedBio":true,
|
||||
"withQuickPromoteEligibilityTweetFields":false
|
||||
}
|
||||
if after.len > 0:
|
||||
variables["cursor"] = % after
|
||||
|
|
@ -234,6 +315,21 @@ proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
|
|||
let js = await fetch(mediaUrl(id, "", 30))
|
||||
result = parseGraphPhotoRail(js)
|
||||
|
||||
proc getGraphArticle*(id: string): Future[Article] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
url = apiReq(graphTweetResultByRestId, articleVars % id, articleFieldToggles)
|
||||
json = await fetchRaw(url)
|
||||
result = parseGraphArticle(json)
|
||||
|
||||
proc getGraphTweetResults*(ids: seq[string]): Future[seq[Tweet]] {.async.} =
|
||||
if ids.len == 0: return
|
||||
let
|
||||
idsJson = "[" & ids.mapIt("\"" & it & "\"").join(",") & "]"
|
||||
url = apiReq(graphTweetResultsByRestIds, articleBatchVars % idsJson, articleFieldToggles)
|
||||
js = await fetch(url)
|
||||
result = parseGraphTweetResults(js)
|
||||
|
||||
proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
|
||||
let client = newAsyncHttpClient(maxRedirects=0)
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
|
||||
import jsony, packedjson, zippy, oauth1
|
||||
import jsony, packedjson, zippy, oauth/oauth1
|
||||
import types, auth, consts, parserutils, http_pool, tid
|
||||
import experimental/types/common
|
||||
|
||||
|
|
@ -8,6 +8,7 @@ const
|
|||
rlRemaining = "x-rate-limit-remaining"
|
||||
rlReset = "x-rate-limit-reset"
|
||||
rlLimit = "x-rate-limit-limit"
|
||||
npCache = "x-np-cache"
|
||||
errorsToSkip = {null, doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
|
||||
|
||||
var
|
||||
|
|
@ -63,7 +64,7 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
|
|||
proc getCookieHeader(authToken, ct0: string): string =
|
||||
"auth_token=" & authToken & "; ct0=" & ct0
|
||||
|
||||
proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
|
||||
proc genHeaders*(session: Session, url: Uri, skipTid: bool): Future[HttpHeaders] {.async.} =
|
||||
result = newHttpHeaders({
|
||||
"accept": "*/*",
|
||||
"accept-encoding": "gzip",
|
||||
|
|
@ -84,13 +85,14 @@ proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
|
|||
result["x-twitter-auth-type"] = "OAuth2Session"
|
||||
result["x-csrf-token"] = session.ct0
|
||||
result["cookie"] = getCookieHeader(session.authToken, session.ct0)
|
||||
result["referer"] = "https://x.com/"
|
||||
result["sec-ch-ua"] = """"Google Chrome";v="142", "Chromium";v="142", "Not A(Brand";v="24""""
|
||||
result["sec-ch-ua-mobile"] = "?0"
|
||||
result["sec-ch-ua-platform"] = "Windows"
|
||||
result["sec-fetch-dest"] = "empty"
|
||||
result["sec-fetch-mode"] = "cors"
|
||||
result["sec-fetch-site"] = "same-site"
|
||||
if disableTid or "/1.1/" in url.path:
|
||||
result["sec-fetch-site"] = "same-origin"
|
||||
if disableTid or skipTid or "/1.1/" in url.path:
|
||||
result["authorization"] = bearerToken2
|
||||
else:
|
||||
result["authorization"] = bearerToken
|
||||
|
|
@ -114,7 +116,12 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
|||
|
||||
try:
|
||||
var resp: AsyncResponse
|
||||
pool.use(await genHeaders(session, url)):
|
||||
let skipTid = case session.kind
|
||||
of oauth: req.oauth.skipTid
|
||||
of cookie: req.cookie.skipTid
|
||||
let headers = await genHeaders(session, url, skipTid)
|
||||
|
||||
pool.use(headers):
|
||||
template getContent =
|
||||
# TODO: this is a temporary simple implementation
|
||||
if apiProxy.len > 0 and "/1.1/" notin url.path:
|
||||
|
|
@ -130,10 +137,11 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
|||
raise newException(BadClientError, "Bad client")
|
||||
|
||||
if resp.status == $Http404 and result.len == 0:
|
||||
echo "[sessions] transient 404 (empty body), retrying: ", url.path
|
||||
echo "[sessions] transient 404 (empty body), retrying: ", url.path, ", session: ", session.pretty
|
||||
raise rateLimitError()
|
||||
|
||||
if resp.headers.hasKey(rlRemaining):
|
||||
let cacheStatus = resp.headers.getOrDefault(npCache)
|
||||
if cacheStatus notin ["HIT", "STALE"] and resp.headers.hasKey(rlRemaining):
|
||||
let
|
||||
remaining = parseInt(resp.headers[rlRemaining])
|
||||
reset = parseInt(resp.headers[rlReset])
|
||||
|
|
@ -147,7 +155,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
|||
if result.startsWith("{\"errors"):
|
||||
let errors = result.fromJson(Errors)
|
||||
if errors notin errorsToSkip:
|
||||
echo "Fetch error, API: ", url.path, ", errors: ", errors
|
||||
echo "Fetch error, API: ", url.path, ", errors: ", errors, ", session: ", session.pretty
|
||||
if errors in {expiredToken, badToken, locked}:
|
||||
invalidate(session)
|
||||
raise rateLimitError()
|
||||
|
|
@ -162,7 +170,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
|||
fetchBody
|
||||
|
||||
if resp.status == $Http400:
|
||||
echo "ERROR 400, ", url.path, ": ", result
|
||||
echo "ERROR 400, ", url.path, ": ", result, ", session: ", session.pretty
|
||||
raise newException(InternalError, $url)
|
||||
except InternalError as e:
|
||||
raise e
|
||||
|
|
@ -177,22 +185,34 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
|||
finally:
|
||||
release(session)
|
||||
|
||||
template retry(bod) =
|
||||
template retry(bod) {.dirty.} =
|
||||
var session: Session
|
||||
var retrySuccess = false
|
||||
for i in 0 ..< maxRetries:
|
||||
try:
|
||||
session = nil
|
||||
bod
|
||||
retrySuccess = true
|
||||
break
|
||||
except RateLimitError:
|
||||
echo "[sessions] Rate limited, retrying ", req.cookie.endpoint,
|
||||
" request (", i, "/", maxRetries, ")..."
|
||||
let api = if session.isNil: req.cookie.endpoint
|
||||
else: req.endpoint(session)
|
||||
if session.isNil:
|
||||
echo "[sessions] Rate limited, retrying ", api,
|
||||
" request (", i, "/", maxRetries, ")..."
|
||||
else:
|
||||
echo "[sessions] Rate limited, retrying ", api,
|
||||
" request (", i, "/", maxRetries, ")..., session: ", session.pretty
|
||||
session = nil
|
||||
if retryDelayMs > 0:
|
||||
await sleepAsync(retryDelayMs)
|
||||
if not retrySuccess:
|
||||
raise rateLimitError()
|
||||
|
||||
proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
|
||||
retry:
|
||||
var
|
||||
body: string
|
||||
session = await getAndValidateSession(req)
|
||||
var body: string
|
||||
session = await getAndValidateSession(req)
|
||||
|
||||
let url = req.toUrl(session.kind)
|
||||
|
||||
|
|
@ -200,22 +220,22 @@ proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
|
|||
if body.startsWith('{') or body.startsWith('['):
|
||||
result = parseJson(body)
|
||||
else:
|
||||
echo resp.status, ": ", body, " --- url: ", url
|
||||
echo resp.status, ": ", body, " --- url: ", url, ", session: ", session.pretty
|
||||
result = newJNull()
|
||||
|
||||
let error = result.getError
|
||||
if error != null and error notin errorsToSkip:
|
||||
echo "Fetch error, API: ", url.path, ", error: ", error
|
||||
echo "Fetch error, API: ", url.path, ", error: ", error, ", session: ", session.pretty
|
||||
if error in {expiredToken, badToken, locked}:
|
||||
invalidate(session)
|
||||
raise rateLimitError()
|
||||
|
||||
proc fetchRaw*(req: ApiReq): Future[string] {.async.} =
|
||||
retry:
|
||||
var session = await getAndValidateSession(req)
|
||||
session = await getAndValidateSession(req)
|
||||
let url = req.toUrl(session.kind)
|
||||
|
||||
fetchImpl result:
|
||||
if not (result.startsWith('{') or result.startsWith('[')):
|
||||
echo resp.status, ": ", result, " --- url: ", url
|
||||
echo resp.status, ": ", result, " --- url: ", url, ", session: ", session.pretty
|
||||
result.setLen(0)
|
||||
|
|
|
|||
19
src/auth.nim
19
src/auth.nim
|
|
@ -18,7 +18,7 @@ proc setMaxConcurrentReqs*(reqs: int) =
|
|||
template log(str: varargs[string, `$`]) =
|
||||
echo "[sessions] ", str.join("")
|
||||
|
||||
proc endpoint(req: ApiReq; session: Session): string =
|
||||
proc endpoint*(req: ApiReq; session: Session): string =
|
||||
case session.kind
|
||||
of oauth: req.oauth.endpoint
|
||||
of cookie: req.cookie.endpoint
|
||||
|
|
@ -50,6 +50,8 @@ proc getSessionPoolHealth*(): JsonNode =
|
|||
oldest = now.int64
|
||||
newest = 0'i64
|
||||
average = 0'i64
|
||||
oauthTotal, cookieTotal = 0
|
||||
oauthLimited, cookieLimited = 0
|
||||
|
||||
for session in sessionPool:
|
||||
let created = snowflakeToEpoch(session.id)
|
||||
|
|
@ -59,8 +61,15 @@ proc getSessionPoolHealth*(): JsonNode =
|
|||
oldest = created
|
||||
average += created
|
||||
|
||||
case session.kind
|
||||
of oauth: inc oauthTotal
|
||||
of cookie: inc cookieTotal
|
||||
|
||||
if session.limited:
|
||||
limited.incl session.id
|
||||
case session.kind
|
||||
of oauth: inc oauthLimited
|
||||
of cookie: inc cookieLimited
|
||||
|
||||
for api in session.apis.keys:
|
||||
let
|
||||
|
|
@ -84,6 +93,8 @@ proc getSessionPoolHealth*(): JsonNode =
|
|||
"sessions": %*{
|
||||
"total": sessionPool.len,
|
||||
"limited": limited.card,
|
||||
"oauth": %*{"total": oauthTotal, "limited": oauthLimited},
|
||||
"cookie": %*{"total": cookieTotal, "limited": cookieLimited},
|
||||
"oldest": $fromUnix(oldest),
|
||||
"newest": $fromUnix(newest),
|
||||
"average": $fromUnix(average)
|
||||
|
|
@ -100,6 +111,7 @@ proc getSessionPoolDebug*(): JsonNode =
|
|||
|
||||
for session in sessionPool:
|
||||
let sessionJson = %*{
|
||||
"kind": $session.kind,
|
||||
"apis": newJObject(),
|
||||
"pending": session.pending,
|
||||
}
|
||||
|
|
@ -173,7 +185,10 @@ proc getSession*(req: ApiReq): Future[Session] {.async.} =
|
|||
if not result.isNil and result.isReady(req):
|
||||
inc result.pending
|
||||
else:
|
||||
log "no sessions available for API: ", req.cookie.endpoint
|
||||
if result.isNil:
|
||||
log "no sessions available for API: ", req.cookie.endpoint
|
||||
else:
|
||||
log "no sessions available for API: ", req.endpoint(result), ", last tried: ", result.pretty
|
||||
raise noSessionsError()
|
||||
|
||||
proc setLimited*(session: Session; req: ApiReq) =
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue