mirror of
https://github.com/zedeus/nitter
synced 2026-11-22 12:57:19 +00:00
Merge branch 'master' into feature/mp4-streaming
This commit is contained in:
commit
93208908e6
50 changed files with 842 additions and 492 deletions
48
.github/workflows/build-docker.yml
vendored
48
.github/workflows/build-docker.yml
vendored
|
|
@ -1,4 +1,4 @@
|
|||
name: CI/CD
|
||||
name: Docker
|
||||
|
||||
on:
|
||||
push:
|
||||
|
|
@ -8,31 +8,55 @@ on:
|
|||
- master
|
||||
|
||||
jobs:
|
||||
build-docker:
|
||||
runs-on: ubuntu-latest
|
||||
tests:
|
||||
uses: ./.github/workflows/run-tests.yml
|
||||
build-docker-amd64:
|
||||
needs: [tests]
|
||||
runs-on: buildjet-2vcpu-ubuntu-2204
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
with:
|
||||
platforms: all
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v1
|
||||
uses: docker/setup-buildx-action@v2
|
||||
with:
|
||||
version: latest
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v1
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v2
|
||||
- name: Build and push AMD64 Docker image
|
||||
uses: docker/build-push-action@v3
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}
|
||||
build-docker-arm64:
|
||||
needs: [tests]
|
||||
runs-on: buildjet-2vcpu-ubuntu-2204-arm
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
with:
|
||||
version: latest
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Build and push ARM64 Docker image
|
||||
uses: docker/build-push-action@v3
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.arm64
|
||||
platforms: linux/arm64
|
||||
push: true
|
||||
tags: zedeus/nitter:latest-arm64,zedeus/nitter:${{ github.sha }}-arm64
|
||||
|
|
|
|||
45
.github/workflows/run-tests.yml
vendored
Normal file
45
.github/workflows/run-tests.yml
vendored
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
name: Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
paths-ignore:
|
||||
- "*.md"
|
||||
branches-ignore:
|
||||
- master
|
||||
workflow_call:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Cache nimble
|
||||
id: cache-nimble
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.nimble
|
||||
key: nimble-${{ hashFiles('*.nimble') }}
|
||||
restore-keys: "nimble-"
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.10"
|
||||
cache: "pip"
|
||||
- uses: jiro4989/setup-nim-action@v1
|
||||
with:
|
||||
nim-version: "1.x"
|
||||
- run: nimble build -d:release -Y
|
||||
- run: pip install seleniumbase
|
||||
- run: seleniumbase install chromedriver
|
||||
- uses: supercharge/redis-github-action@1.5.0
|
||||
- name: Prepare Nitter
|
||||
run: |
|
||||
sudo apt install libsass-dev -y
|
||||
cp nitter.example.conf nitter.conf
|
||||
nimble md
|
||||
nimble scss
|
||||
- name: Run tests
|
||||
run: |
|
||||
./nitter &
|
||||
pytest -n4 tests
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
FROM nimlang/nim:1.6.2-alpine-regular as nim
|
||||
FROM nimlang/nim:1.6.10-alpine-regular as nim
|
||||
LABEL maintainer="setenforce@protonmail.com"
|
||||
|
||||
RUN apk --no-cache add libsass-dev pcre
|
||||
|
|
@ -20,4 +20,6 @@ COPY --from=nim /src/nitter/nitter ./
|
|||
COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
|
||||
COPY --from=nim /src/nitter/public ./public
|
||||
EXPOSE 8080
|
||||
RUN adduser -h /src/ -D -s /bin/sh nitter
|
||||
USER nitter
|
||||
CMD ./nitter
|
||||
|
|
|
|||
23
Dockerfile.arm64
Normal file
23
Dockerfile.arm64
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
FROM alpine:3.17 as nim
|
||||
LABEL maintainer="setenforce@protonmail.com"
|
||||
|
||||
RUN apk --no-cache add gcc git libc-dev libsass-dev "nim=1.6.8-r0" nimble pcre
|
||||
|
||||
WORKDIR /src/nitter
|
||||
|
||||
COPY nitter.nimble .
|
||||
RUN nimble install -y --depsOnly
|
||||
|
||||
COPY . .
|
||||
RUN nimble build -d:danger -d:lto -d:strip \
|
||||
&& nimble scss \
|
||||
&& nimble md
|
||||
|
||||
FROM alpine:3.17
|
||||
WORKDIR /src/
|
||||
RUN apk --no-cache add ca-certificates pcre openssl1.1-compat
|
||||
COPY --from=nim /src/nitter/nitter ./
|
||||
COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
|
||||
COPY --from=nim /src/nitter/public ./public
|
||||
EXPOSE 8080
|
||||
CMD ./nitter
|
||||
21
README.md
21
README.md
|
|
@ -1,6 +1,7 @@
|
|||
# Nitter
|
||||
|
||||
[](https://github.com/zedeus/nitter/actions?query=workflow%3ACI/CD)
|
||||
[](https://github.com/zedeus/nitter/actions/workflows/run-tests.yml)
|
||||
[](https://github.com/zedeus/nitter/actions/workflows/build-docker.yml)
|
||||
[](#license)
|
||||
|
||||
A free and open source alternative Twitter front-end focused on privacy and
|
||||
|
|
@ -34,7 +35,7 @@ XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscW
|
|||
|
||||
## Resources
|
||||
|
||||
The wiki contains
|
||||
The wiki contains
|
||||
[a list of instances](https://github.com/zedeus/nitter/wiki/Instances) and
|
||||
[browser extensions](https://github.com/zedeus/nitter/wiki/Extensions)
|
||||
maintained by the community.
|
||||
|
|
@ -67,9 +68,10 @@ Twitter account.
|
|||
## Installation
|
||||
|
||||
### Dependencies
|
||||
* libpcre
|
||||
* libsass
|
||||
* redis
|
||||
|
||||
- libpcre
|
||||
- libsass
|
||||
- redis
|
||||
|
||||
To compile Nitter you need a Nim installation, see
|
||||
[nim-lang.org](https://nim-lang.org/install.html) for details. It is possible to
|
||||
|
|
@ -108,25 +110,32 @@ performance reasons.
|
|||
|
||||
### Docker
|
||||
|
||||
#### NOTE: For ARM64/ARM support, please use [unixfox's image](https://quay.io/repository/unixfox/nitter?tab=tags), more info [here](https://github.com/zedeus/nitter/issues/399#issuecomment-997263495)
|
||||
Page for the Docker image: https://hub.docker.com/r/zedeus/nitter
|
||||
|
||||
#### NOTE: For ARM64 support, please use the separate ARM64 docker image: [`zedeus/nitter:latest-arm64`](https://hub.docker.com/r/zedeus/nitter/tags).
|
||||
|
||||
To run Nitter with Docker, you'll need to install and run Redis separately
|
||||
before you can run the container. See below for how to also run Redis using
|
||||
Docker.
|
||||
|
||||
To build and run Nitter in Docker:
|
||||
|
||||
```bash
|
||||
docker build -t nitter:latest .
|
||||
docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest
|
||||
```
|
||||
|
||||
Note: For ARM64, use this Dockerfile: [`Dockerfile.arm64`](https://github.com/zedeus/nitter/blob/master/Dockerfile.arm64).
|
||||
|
||||
A prebuilt Docker image is provided as well:
|
||||
|
||||
```bash
|
||||
docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host zedeus/nitter:latest
|
||||
```
|
||||
|
||||
Using docker-compose to run both Nitter and Redis as different containers:
|
||||
Change `redisHost` from `localhost` to `nitter-redis` in `nitter.conf`, then run:
|
||||
|
||||
```bash
|
||||
docker-compose up -d
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
--define:ssl
|
||||
--define:useStdLib
|
||||
--threads:off
|
||||
|
||||
# workaround httpbeast file upload bug
|
||||
--assertions:off
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ services:
|
|||
ports:
|
||||
- "127.0.0.1:8080:8080" # Replace with "8080:8080" if you don't use a reverse proxy
|
||||
volumes:
|
||||
- ./nitter.conf:/src/nitter.conf:ro
|
||||
- ./nitter.conf:/src/nitter.conf:Z,ro
|
||||
depends_on:
|
||||
- nitter-redis
|
||||
restart: unless-stopped
|
||||
|
|
@ -17,6 +17,12 @@ services:
|
|||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 2
|
||||
user: "998:998"
|
||||
read_only: true
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
cap_drop:
|
||||
- ALL
|
||||
|
||||
nitter-redis:
|
||||
image: redis:6-alpine
|
||||
|
|
@ -30,6 +36,12 @@ services:
|
|||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 2
|
||||
user: "999:1000"
|
||||
read_only: true
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
cap_drop:
|
||||
- ALL
|
||||
|
||||
volumes:
|
||||
nitter-redis:
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
[Server]
|
||||
hostname = "nitter.net" # for generating links, change this to your own domain/ip
|
||||
title = "nitter"
|
||||
address = "0.0.0.0"
|
||||
port = 8080
|
||||
https = false # disable to enable cookies when not using https
|
||||
httpMaxConnections = 100
|
||||
staticDir = "./public"
|
||||
title = "nitter"
|
||||
hostname = "nitter.net"
|
||||
|
||||
[Cache]
|
||||
listMinutes = 240 # how long to cache list info (not the tweets, so keep it high)
|
||||
|
|
@ -13,9 +13,9 @@ rssMinutes = 10 # how long to cache rss queries
|
|||
redisHost = "localhost" # Change to "nitter-redis" if using docker-compose
|
||||
redisPort = 6379
|
||||
redisPassword = ""
|
||||
redisConnections = 20 # connection pool size
|
||||
redisConnections = 20 # minimum open connections in pool
|
||||
redisMaxConnections = 30
|
||||
# max, new connections are opened when none are available, but if the pool size
|
||||
# new connections are opened when none are available, but if the pool size
|
||||
# goes above this, they're closed when released. don't worry about this unless
|
||||
# you receive tons of requests per second
|
||||
|
||||
|
|
@ -23,23 +23,22 @@ redisMaxConnections = 30
|
|||
hmacKey = "secretkey" # random key for cryptographic signing of video urls
|
||||
base64Media = false # use base64 encoding for proxied media urls
|
||||
enableRSS = true # set this to false to disable RSS feeds
|
||||
enableDebug = false # enable request logs and debug endpoints
|
||||
enableDebug = false # enable request logs and debug endpoints (/.tokens)
|
||||
proxy = "" # http/https url, SOCKS proxies are not supported
|
||||
proxyAuth = ""
|
||||
tokenCount = 10
|
||||
# minimum amount of usable tokens. tokens are used to authorize API requests,
|
||||
# but they expire after ~1 hour, and have a limit of 187 requests.
|
||||
# the limit gets reset every 15 minutes, and the pool is filled up so there's
|
||||
# always at least $tokenCount usable tokens. again, only increase this if
|
||||
# you receive major bursts all the time
|
||||
# but they expire after ~1 hour, and have a limit of 500 requests per endpoint.
|
||||
# the limits reset every 15 minutes, and the pool is filled up so there's
|
||||
# always at least `tokenCount` usable tokens. only increase this if you receive
|
||||
# major bursts all the time and don't have a rate limiting setup via e.g. nginx
|
||||
|
||||
# Change default preferences here, see src/prefs_impl.nim for a complete list
|
||||
[Preferences]
|
||||
theme = "Nitter"
|
||||
replaceTwitter = "nitter.net"
|
||||
replaceYouTube = "piped.kavin.rocks"
|
||||
replaceYouTube = "piped.video"
|
||||
replaceReddit = "teddit.net"
|
||||
replaceInstagram = ""
|
||||
proxyVideos = true
|
||||
hlsPlayback = false
|
||||
infiniteScroll = false
|
||||
|
|
|
|||
|
|
@ -11,18 +11,18 @@ bin = @["nitter"]
|
|||
# Dependencies
|
||||
|
||||
requires "nim >= 1.4.8"
|
||||
requires "jester >= 0.5.0"
|
||||
requires "karax#5498909"
|
||||
requires "sass#e683aa1"
|
||||
requires "nimcrypto#a5742a9"
|
||||
requires "markdown#a661c26"
|
||||
requires "packedjson#d11d167"
|
||||
requires "supersnappy#2.1.1"
|
||||
requires "jester#baca3f"
|
||||
requires "karax#5cf360c"
|
||||
requires "sass#7dfdd03"
|
||||
requires "nimcrypto#4014ef9"
|
||||
requires "markdown#158efe3"
|
||||
requires "packedjson#9e6fbb6"
|
||||
requires "supersnappy#6c94198"
|
||||
requires "redpool#8b7c1db"
|
||||
requires "https://github.com/zedeus/redis#d0a0e6f"
|
||||
requires "zippy#0.7.3"
|
||||
requires "flatty#0.2.3"
|
||||
requires "jsony#d0e69bd"
|
||||
requires "zippy#ca5989a"
|
||||
requires "flatty#e668085"
|
||||
requires "jsony#ea811be"
|
||||
|
||||
|
||||
# Tasks
|
||||
|
|
|
|||
6
public/js/hls.light.min.js
vendored
6
public/js/hls.light.min.js
vendored
File diff suppressed because one or more lines are too long
|
|
@ -1,4 +1,5 @@
|
|||
User-agent: *
|
||||
Disallow: /
|
||||
Crawl-delay: 1
|
||||
User-agent: Twitterbot
|
||||
Disallow:
|
||||
|
|
|
|||
180
src/api.nim
180
src/api.nim
|
|
@ -4,123 +4,143 @@ import packedjson
|
|||
import types, query, formatters, consts, apiutils, parser
|
||||
import experimental/parser as newParser
|
||||
|
||||
proc getGraphUser*(id: string): Future[User] {.async.} =
|
||||
proc getGraphUser*(username: string): Future[User] {.async.} =
|
||||
if username.len == 0: return
|
||||
let
|
||||
variables = %*{"screen_name": username}
|
||||
params = {"variables": $variables, "features": gqlFeatures}
|
||||
js = await fetchRaw(graphUser ? params, Api.userScreenName)
|
||||
result = parseGraphUser(js)
|
||||
|
||||
proc getGraphUserById*(id: string): Future[User] {.async.} =
|
||||
if id.len == 0 or id.any(c => not c.isDigit): return
|
||||
let
|
||||
variables = %*{"userId": id, "withSuperFollowsUserFields": true}
|
||||
js = await fetchRaw(graphUser ? {"variables": $variables}, Api.userRestId)
|
||||
variables = %*{"userId": id}
|
||||
params = {"variables": $variables, "features": gqlFeatures}
|
||||
js = await fetchRaw(graphUserById ? params, Api.userRestId)
|
||||
result = parseGraphUser(js)
|
||||
|
||||
proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Timeline] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = userTweetsVariables % [id, cursor]
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
(url, apiId) = case kind
|
||||
of TimelineKind.tweets: (graphUserTweets, Api.userTweets)
|
||||
of TimelineKind.replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
|
||||
of TimelineKind.media: (graphUserMedia, Api.userMedia)
|
||||
js = await fetch(url ? params, apiId)
|
||||
result = parseGraphTimeline(js, "user", after)
|
||||
|
||||
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = listTweetsVariables % [id, cursor]
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
js = await fetch(graphListTweets ? params, Api.listTweets)
|
||||
result = parseGraphTimeline(js, "list", after)
|
||||
|
||||
proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
|
||||
let
|
||||
variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false}
|
||||
url = graphListBySlug ? {"variables": $variables}
|
||||
result = parseGraphList(await fetch(url, Api.listBySlug))
|
||||
variables = %*{"screenName": name, "listSlug": list}
|
||||
params = {"variables": $variables, "features": gqlFeatures}
|
||||
result = parseGraphList(await fetch(graphListBySlug ? params, Api.listBySlug))
|
||||
|
||||
proc getGraphList*(id: string): Future[List] {.async.} =
|
||||
let
|
||||
variables = %*{"listId": id, "withHighlightedLabel": false}
|
||||
url = graphList ? {"variables": $variables}
|
||||
result = parseGraphList(await fetch(url, Api.list))
|
||||
variables = %*{"listId": id}
|
||||
params = {"variables": $variables, "features": gqlFeatures}
|
||||
result = parseGraphList(await fetch(graphListById ? params, Api.list))
|
||||
|
||||
proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
|
||||
if list.id.len == 0: return
|
||||
var
|
||||
variables = %*{
|
||||
"listId": list.id,
|
||||
"withSuperFollowsUserFields": false,
|
||||
"withBirdwatchPivots": false,
|
||||
"withDownvotePerspective": false,
|
||||
"withReactionsMetadata": false,
|
||||
"withReactionsPerspective": false,
|
||||
"withSuperFollowsTweetFields": false
|
||||
"withReactionsPerspective": false
|
||||
}
|
||||
if after.len > 0:
|
||||
variables["cursor"] = % after
|
||||
let url = graphListMembers ? {"variables": $variables}
|
||||
let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
|
||||
result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
|
||||
|
||||
proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
|
||||
proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
ps = genParams({"list_id": id, "ranking_mode": "reverse_chronological"}, after)
|
||||
url = listTimeline ? ps
|
||||
result = parseTimeline(await fetch(url, Api.timeline), after)
|
||||
variables = tweetResultVariables % id
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
js = await fetch(graphTweetResult ? params, Api.tweetResult)
|
||||
result = parseGraphTweetResult(js)
|
||||
|
||||
proc getUser*(username: string): Future[User] {.async.} =
|
||||
if username.len == 0: return
|
||||
let
|
||||
ps = genParams({"screen_name": username})
|
||||
json = await fetchRaw(userShow ? ps, Api.userShow)
|
||||
result = parseUser(json, username)
|
||||
|
||||
proc getUserById*(userId: string): Future[User] {.async.} =
|
||||
if userId.len == 0: return
|
||||
let
|
||||
ps = genParams({"user_id": userId})
|
||||
json = await fetchRaw(userShow ? ps, Api.userShow)
|
||||
result = parseUser(json)
|
||||
|
||||
proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
|
||||
proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
|
||||
url = timeline / (id & ".json") ? ps
|
||||
result = parseTimeline(await fetch(url, Api.timeline), after)
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = tweetVariables % [id, cursor]
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
js = await fetch(graphTweet ? params, Api.tweetDetail)
|
||||
result = parseGraphConversation(js, id)
|
||||
|
||||
proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} =
|
||||
if id.len == 0: return
|
||||
let url = mediaTimeline / (id & ".json") ? genParams(cursor=after)
|
||||
result = parseTimeline(await fetch(url, Api.timeline), after)
|
||||
proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
|
||||
result = (await getGraphTweet(id, after)).replies
|
||||
result.beginning = after.len == 0
|
||||
|
||||
proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
|
||||
result = await getGraphTweet(id)
|
||||
if after.len > 0:
|
||||
result.replies = await getReplies(id, after)
|
||||
|
||||
proc getGraphSearch*(query: Query; after=""): Future[Result[Tweet]] {.async.} =
|
||||
let q = genQueryParam(query)
|
||||
if q.len == 0 or q == emptyQuery:
|
||||
return Result[Tweet](query: query, beginning: true)
|
||||
|
||||
var
|
||||
variables = %*{
|
||||
"rawQuery": q,
|
||||
"count": 20,
|
||||
"product": "Latest",
|
||||
"withDownvotePerspective": false,
|
||||
"withReactionsMetadata": false,
|
||||
"withReactionsPerspective": false
|
||||
}
|
||||
if after.len > 0:
|
||||
variables["cursor"] = % after
|
||||
let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
|
||||
result = parseGraphSearch(await fetch(url, Api.search), after)
|
||||
result.query = query
|
||||
|
||||
proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
|
||||
if query.text.len == 0:
|
||||
return Result[User](query: query, beginning: true)
|
||||
|
||||
var url = userSearch ? {
|
||||
"q": query.text,
|
||||
"skip_status": "1",
|
||||
"count": "20",
|
||||
"page": page
|
||||
}
|
||||
|
||||
result = parseUsers(await fetchRaw(url, Api.userSearch))
|
||||
result.query = query
|
||||
if page.len == 0:
|
||||
result.bottom = "2"
|
||||
elif page.allCharsInSet(Digits):
|
||||
result.bottom = $(parseInt(page) + 1)
|
||||
|
||||
proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
|
||||
if name.len == 0: return
|
||||
let
|
||||
ps = genParams({"screen_name": name, "trim_user": "true"},
|
||||
count="18", ext=false)
|
||||
count="18", ext=false)
|
||||
url = photoRail ? ps
|
||||
result = parsePhotoRail(await fetch(url, Api.timeline))
|
||||
|
||||
proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
|
||||
when T is User:
|
||||
const
|
||||
searchMode = ("result_filter", "user")
|
||||
parse = parseUsers
|
||||
fetchFunc = fetchRaw
|
||||
else:
|
||||
const
|
||||
searchMode = ("tweet_search_mode", "live")
|
||||
parse = parseTimeline
|
||||
fetchFunc = fetch
|
||||
|
||||
let q = genQueryParam(query)
|
||||
if q.len == 0 or q == emptyQuery:
|
||||
return Result[T](beginning: true, query: query)
|
||||
|
||||
let url = search ? genParams(searchParams & @[("q", q), searchMode], after)
|
||||
try:
|
||||
result = parse(await fetchFunc(url, Api.search), after)
|
||||
result.query = query
|
||||
except InternalError:
|
||||
return Result[T](beginning: true, query: query)
|
||||
|
||||
proc getTweetImpl(id: string; after=""): Future[Conversation] {.async.} =
|
||||
let url = tweet / (id & ".json") ? genParams(cursor=after)
|
||||
result = parseConversation(await fetch(url, Api.tweet), id)
|
||||
|
||||
proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
|
||||
result = (await getTweetImpl(id, after)).replies
|
||||
result.beginning = after.len == 0
|
||||
|
||||
proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
|
||||
result = await getTweetImpl(id)
|
||||
if after.len > 0:
|
||||
result.replies = await getReplies(id, after)
|
||||
|
||||
proc getStatus*(id: string): Future[Tweet] {.async.} =
|
||||
let url = status / (id & ".json") ? genParams()
|
||||
result = parseStatus(await fetch(url, Api.status))
|
||||
|
||||
proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
|
||||
let client = newAsyncHttpClient(maxRedirects=0)
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -17,13 +17,13 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
|
|||
result &= p
|
||||
if ext:
|
||||
result &= ("ext", "mediaStats")
|
||||
result &= ("include_ext_alt_text", "true")
|
||||
result &= ("include_ext_media_availability", "true")
|
||||
result &= ("include_ext_alt_text", "1")
|
||||
result &= ("include_ext_media_availability", "1")
|
||||
if count.len > 0:
|
||||
result &= ("count", count)
|
||||
if cursor.len > 0:
|
||||
# The raw cursor often has plus signs, which sometimes get turned into spaces,
|
||||
# so we need to them back into a plus
|
||||
# so we need to turn them back into a plus
|
||||
if " " in cursor:
|
||||
result &= ("cursor", cursor.replace(" ", "+"))
|
||||
else:
|
||||
|
|
@ -44,7 +44,7 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
|
|||
})
|
||||
|
||||
template updateToken() =
|
||||
if api != Api.search and resp.headers.hasKey(rlRemaining):
|
||||
if resp.headers.hasKey(rlRemaining):
|
||||
let
|
||||
remaining = parseInt(resp.headers[rlRemaining])
|
||||
reset = parseInt(resp.headers[rlReset])
|
||||
|
|
@ -61,12 +61,15 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
|||
try:
|
||||
var resp: AsyncResponse
|
||||
pool.use(genHeaders(token)):
|
||||
resp = await c.get($url)
|
||||
result = await resp.body
|
||||
template getContent =
|
||||
resp = await c.get($url)
|
||||
result = await resp.body
|
||||
|
||||
getContent()
|
||||
|
||||
if resp.status == $Http503:
|
||||
badClient = true
|
||||
raise newException(InternalError, result)
|
||||
raise newException(BadClientError, "Bad client")
|
||||
|
||||
if result.len > 0:
|
||||
if resp.headers.getOrDefault("content-encoding") == "gzip":
|
||||
|
|
@ -82,6 +85,9 @@ template fetchImpl(result, fetchBody) {.dirty.} =
|
|||
raise newException(InternalError, $url)
|
||||
except InternalError as e:
|
||||
raise e
|
||||
except BadClientError as e:
|
||||
release(token, used=true)
|
||||
raise e
|
||||
except Exception as e:
|
||||
echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url
|
||||
if "length" notin e.msg and "descriptor" notin e.msg:
|
||||
|
|
@ -100,7 +106,7 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
|
|||
updateToken()
|
||||
|
||||
let error = result.getError
|
||||
if error in {invalidToken, forbidden, badToken}:
|
||||
if error in {invalidToken, badToken}:
|
||||
echo "fetch error: ", result.getError
|
||||
release(token, invalid=true)
|
||||
raise rateLimitError()
|
||||
|
|
@ -115,7 +121,7 @@ proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
|
|||
|
||||
if result.startsWith("{\"errors"):
|
||||
let errors = result.fromJson(Errors)
|
||||
if errors in {invalidToken, forbidden, badToken}:
|
||||
if errors in {invalidToken, badToken}:
|
||||
echo "fetch error: ", errors
|
||||
release(token, invalid=true)
|
||||
raise rateLimitError()
|
||||
|
|
|
|||
126
src/consts.nim
126
src/consts.nim
|
|
@ -1,28 +1,28 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
import uri, sequtils
|
||||
import uri, sequtils, strutils
|
||||
|
||||
const
|
||||
auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
|
||||
auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
|
||||
|
||||
api = parseUri("https://api.twitter.com")
|
||||
activate* = $(api / "1.1/guest/activate.json")
|
||||
|
||||
userShow* = api / "1.1/users/show.json"
|
||||
photoRail* = api / "1.1/statuses/media_timeline.json"
|
||||
status* = api / "1.1/statuses/show"
|
||||
search* = api / "2/search/adaptive.json"
|
||||
|
||||
timelineApi = api / "2/timeline"
|
||||
timeline* = timelineApi / "profile"
|
||||
mediaTimeline* = timelineApi / "media"
|
||||
listTimeline* = timelineApi / "list.json"
|
||||
tweet* = timelineApi / "conversation"
|
||||
userSearch* = api / "1.1/users/search.json"
|
||||
|
||||
graphql = api / "graphql"
|
||||
graphUser* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
|
||||
graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
|
||||
graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
|
||||
graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
|
||||
graphUser* = graphql / "pVrmNaXcxPjisIvKtLDMEA/UserByScreenName"
|
||||
graphUserById* = graphql / "1YAM811Q8Ry4XyPpJclURQ/UserByRestId"
|
||||
graphUserTweets* = graphql / "WzJjibAcDa-oCjCcLOotcg/UserTweets"
|
||||
graphUserTweetsAndReplies* = graphql / "fn9oRltM1N4thkh5CVusPg/UserTweetsAndReplies"
|
||||
graphUserMedia* = graphql / "qQoeS7szGavsi8-ehD2AWg/UserMedia"
|
||||
graphTweet* = graphql / "miKSMGb2R1SewIJv2-ablQ/TweetDetail"
|
||||
graphTweetResult* = graphql / "0kc0a_7TTr3dvweZlMslsQ/TweetResultByRestId"
|
||||
graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline"
|
||||
graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
|
||||
graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug"
|
||||
graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
|
||||
graphListTweets* = graphql / "jZntL0oVJSdjhmPcdbw_eA/ListLatestTweetsTimeline"
|
||||
|
||||
timelineParams* = {
|
||||
"include_profile_interstitial_type": "0",
|
||||
|
|
@ -33,27 +33,89 @@ const
|
|||
"include_mute_edge": "0",
|
||||
"include_can_dm": "0",
|
||||
"include_can_media_tag": "1",
|
||||
"include_ext_is_blue_verified": "1",
|
||||
"skip_status": "1",
|
||||
"cards_platform": "Web-12",
|
||||
"include_cards": "1",
|
||||
"include_composer_source": "false",
|
||||
"include_composer_source": "0",
|
||||
"include_reply_count": "1",
|
||||
"tweet_mode": "extended",
|
||||
"include_entities": "true",
|
||||
"include_user_entities": "true",
|
||||
"include_ext_media_color": "false",
|
||||
"send_error_codes": "true",
|
||||
"simple_quoted_tweet": "true",
|
||||
"include_quote_count": "true"
|
||||
"include_entities": "1",
|
||||
"include_user_entities": "1",
|
||||
"include_ext_media_color": "0",
|
||||
"send_error_codes": "1",
|
||||
"simple_quoted_tweet": "1",
|
||||
"include_quote_count": "1"
|
||||
}.toSeq
|
||||
|
||||
searchParams* = {
|
||||
"query_source": "typed_query",
|
||||
"pc": "1",
|
||||
"spelling_corrections": "1"
|
||||
}.toSeq
|
||||
## top: nothing
|
||||
## latest: "tweet_search_mode: live"
|
||||
## user: "result_filter: user"
|
||||
## photos: "result_filter: photos"
|
||||
## videos: "result_filter: videos"
|
||||
gqlFeatures* = """{
|
||||
"blue_business_profile_image_shape_enabled": false,
|
||||
"creator_subscriptions_tweet_preview_api_enabled": true,
|
||||
"freedom_of_speech_not_reach_fetch_enabled": false,
|
||||
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
|
||||
"highlights_tweets_tab_ui_enabled": false,
|
||||
"interactive_text_enabled": false,
|
||||
"longform_notetweets_consumption_enabled": true,
|
||||
"longform_notetweets_inline_media_enabled": false,
|
||||
"longform_notetweets_richtext_consumption_enabled": true,
|
||||
"longform_notetweets_rich_text_read_enabled": false,
|
||||
"responsive_web_edit_tweet_api_enabled": false,
|
||||
"responsive_web_enhance_cards_enabled": false,
|
||||
"responsive_web_graphql_exclude_directive_enabled": true,
|
||||
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
|
||||
"responsive_web_graphql_timeline_navigation_enabled": false,
|
||||
"responsive_web_text_conversations_enabled": false,
|
||||
"responsive_web_twitter_blue_verified_badge_is_enabled": true,
|
||||
"rweb_lists_timeline_redesign_enabled": true,
|
||||
"spaces_2022_h2_clipping": true,
|
||||
"spaces_2022_h2_spaces_communities": true,
|
||||
"standardized_nudges_misinfo": false,
|
||||
"tweet_awards_web_tipping_enabled": false,
|
||||
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
|
||||
"tweetypie_unmention_optimization_enabled": false,
|
||||
"verified_phone_label_enabled": false,
|
||||
"vibe_api_enabled": false,
|
||||
"view_counts_everywhere_api_enabled": false
|
||||
}""".replace(" ", "").replace("\n", "")
|
||||
|
||||
tweetVariables* = """{
|
||||
"focalTweetId": "$1",
|
||||
$2
|
||||
"withBirdwatchNotes": false,
|
||||
"includePromotedContent": false,
|
||||
"withDownvotePerspective": false,
|
||||
"withReactionsMetadata": false,
|
||||
"withReactionsPerspective": false,
|
||||
"withVoice": false
|
||||
}"""
|
||||
|
||||
tweetResultVariables* = """{
|
||||
"tweetId": "$1",
|
||||
"includePromotedContent": false,
|
||||
"withDownvotePerspective": false,
|
||||
"withReactionsMetadata": false,
|
||||
"withReactionsPerspective": false,
|
||||
"withVoice": false,
|
||||
"withCommunity": false
|
||||
}"""
|
||||
|
||||
userTweetsVariables* = """{
|
||||
"userId": "$1", $2
|
||||
"count": 20,
|
||||
"includePromotedContent": false,
|
||||
"withDownvotePerspective": false,
|
||||
"withReactionsMetadata": false,
|
||||
"withReactionsPerspective": false,
|
||||
"withVoice": false,
|
||||
"withV2Timeline": true
|
||||
}"""
|
||||
|
||||
listTweetsVariables* = """{
|
||||
"listId": "$1", $2
|
||||
"count": 20,
|
||||
"includePromotedContent": false,
|
||||
"withDownvotePerspective": false,
|
||||
"withReactionsMetadata": false,
|
||||
"withReactionsPerspective": false,
|
||||
"withVoice": false
|
||||
}"""
|
||||
|
|
|
|||
|
|
@ -1,2 +1,2 @@
|
|||
import parser/[user, graphql, timeline]
|
||||
export user, graphql, timeline
|
||||
import parser/[user, graphql]
|
||||
export user, graphql
|
||||
|
|
|
|||
|
|
@ -1,11 +1,17 @@
|
|||
import options
|
||||
import jsony
|
||||
import user, ../types/[graphuser, graphlistmembers]
|
||||
from ../../types import User, Result, Query, QueryKind
|
||||
|
||||
proc parseGraphUser*(json: string): User =
|
||||
let raw = json.fromJson(GraphUser)
|
||||
|
||||
if raw.data.user.result.reason.get("") == "Suspended":
|
||||
return User(suspended: true)
|
||||
|
||||
result = toUser raw.data.user.result.legacy
|
||||
result.id = raw.data.user.result.restId
|
||||
result.verified = result.verified or raw.data.user.result.isBlueVerified
|
||||
|
||||
proc parseGraphListMembers*(json, cursor: string): Result[User] =
|
||||
result = Result[User](
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
import std/[strutils, tables]
|
||||
import jsony
|
||||
import user, ../types/timeline
|
||||
from ../../types import Result, User
|
||||
|
||||
proc getId(id: string): string {.inline.} =
|
||||
let start = id.rfind("-")
|
||||
if start < 0: return id
|
||||
id[start + 1 ..< id.len]
|
||||
|
||||
proc parseUsers*(json: string; after=""): Result[User] =
|
||||
result = Result[User](beginning: after.len == 0)
|
||||
|
||||
let raw = json.fromJson(Search)
|
||||
if raw.timeline.instructions.len == 0:
|
||||
return
|
||||
|
||||
for i in raw.timeline.instructions:
|
||||
if i.addEntries.entries.len > 0:
|
||||
for e in i.addEntries.entries:
|
||||
let id = e.entryId.getId
|
||||
if e.entryId.startsWith("user"):
|
||||
if id in raw.globalObjects.users:
|
||||
result.content.add toUser raw.globalObjects.users[id]
|
||||
elif e.entryId.startsWith("cursor"):
|
||||
let cursor = e.content.operation.cursor
|
||||
if cursor.cursorType == "Top":
|
||||
result.top = cursor.value
|
||||
elif cursor.cursorType == "Bottom":
|
||||
result.bottom = cursor.value
|
||||
|
|
@ -66,6 +66,8 @@ proc parseMedia(component: Component; card: UnifiedCard; result: var Card) =
|
|||
durationMs: videoInfo.durationMillis,
|
||||
variants: videoInfo.variants
|
||||
)
|
||||
of model3d:
|
||||
result.title = "Unsupported 3D model ad"
|
||||
|
||||
proc parseUnifiedCard*(json: string): Card =
|
||||
let card = json.fromJson(UnifiedCard)
|
||||
|
|
@ -82,6 +84,10 @@ proc parseUnifiedCard*(json: string): Card =
|
|||
component.parseMedia(card, result)
|
||||
of buttonGroup:
|
||||
discard
|
||||
of ComponentType.hidden:
|
||||
result.kind = CardKind.hidden
|
||||
of ComponentType.unknown:
|
||||
echo "ERROR: Unknown component type: ", json
|
||||
|
||||
case component.kind
|
||||
of twitterListDetails:
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import std/[algorithm, unicode, re, strutils, strformat, options, nre]
|
|||
import jsony
|
||||
import utils, slices
|
||||
import ../types/user as userType
|
||||
from ../../types import User, Error
|
||||
from ../../types import Result, User, Error
|
||||
|
||||
let
|
||||
unRegex = re.re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
|
||||
|
|
@ -76,3 +76,12 @@ proc parseUser*(json: string; username=""): User =
|
|||
else: echo "[error - parseUser]: ", error
|
||||
|
||||
result = toUser json.fromJson(RawUser)
|
||||
|
||||
proc parseUsers*(json: string; after=""): Result[User] =
|
||||
result = Result[User](beginning: after.len == 0)
|
||||
|
||||
# starting with '{' means it's an error
|
||||
if json[0] == '[':
|
||||
let raw = json.fromJson(seq[RawUser])
|
||||
for user in raw:
|
||||
result.content.add user.toUser
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import options
|
||||
import user
|
||||
|
||||
type
|
||||
|
|
@ -10,3 +11,5 @@ type
|
|||
UserResult = object
|
||||
legacy*: RawUser
|
||||
restId*: string
|
||||
isBlueVerified*: bool
|
||||
reason*: Option[string]
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue