Watch
1
0
Fork
You've already forked nitter
0
mirror of https://github.com/zedeus/nitter synced 2026-11-22 12:57:19 +00:00

Merge branch 'master' into feature/mp4-streaming

This commit is contained in:
Zed 2023-05-20 22:23:12 +02:00
commit 93208908e6
50 changed files with 842 additions and 492 deletions

View file

@ -1,4 +1,4 @@
name: CI/CD
name: Docker
on:
push:
@ -8,31 +8,55 @@ on:
- master
jobs:
build-docker:
runs-on: ubuntu-latest
tests:
uses: ./.github/workflows/run-tests.yml
build-docker-amd64:
needs: [tests]
runs-on: buildjet-2vcpu-ubuntu-2204
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
with:
platforms: all
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v1
uses: docker/setup-buildx-action@v2
with:
version: latest
- name: Login to DockerHub
uses: docker/login-action@v1
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push
uses: docker/build-push-action@v2
- name: Build and push AMD64 Docker image
uses: docker/build-push-action@v3
with:
context: .
file: ./Dockerfile
platforms: linux/amd64
push: true
tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}
build-docker-arm64:
needs: [tests]
runs-on: buildjet-2vcpu-ubuntu-2204-arm
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2
with:
version: latest
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push ARM64 Docker image
uses: docker/build-push-action@v3
with:
context: .
file: ./Dockerfile.arm64
platforms: linux/arm64
push: true
tags: zedeus/nitter:latest-arm64,zedeus/nitter:${{ github.sha }}-arm64

45
.github/workflows/run-tests.yml vendored Normal file
View file

@ -0,0 +1,45 @@
name: Tests
on:
push:
paths-ignore:
- "*.md"
branches-ignore:
- master
workflow_call:
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Cache nimble
id: cache-nimble
uses: actions/cache@v3
with:
path: ~/.nimble
key: nimble-${{ hashFiles('*.nimble') }}
restore-keys: "nimble-"
- uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "pip"
- uses: jiro4989/setup-nim-action@v1
with:
nim-version: "1.x"
- run: nimble build -d:release -Y
- run: pip install seleniumbase
- run: seleniumbase install chromedriver
- uses: supercharge/redis-github-action@1.5.0
- name: Prepare Nitter
run: |
sudo apt install libsass-dev -y
cp nitter.example.conf nitter.conf
nimble md
nimble scss
- name: Run tests
run: |
./nitter &
pytest -n4 tests

View file

@ -1,4 +1,4 @@
FROM nimlang/nim:1.6.2-alpine-regular as nim
FROM nimlang/nim:1.6.10-alpine-regular as nim
LABEL maintainer="setenforce@protonmail.com"
RUN apk --no-cache add libsass-dev pcre
@ -20,4 +20,6 @@ COPY --from=nim /src/nitter/nitter ./
COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
COPY --from=nim /src/nitter/public ./public
EXPOSE 8080
RUN adduser -h /src/ -D -s /bin/sh nitter
USER nitter
CMD ./nitter

23
Dockerfile.arm64 Normal file
View file

@ -0,0 +1,23 @@
FROM alpine:3.17 as nim
LABEL maintainer="setenforce@protonmail.com"
RUN apk --no-cache add gcc git libc-dev libsass-dev "nim=1.6.8-r0" nimble pcre
WORKDIR /src/nitter
COPY nitter.nimble .
RUN nimble install -y --depsOnly
COPY . .
RUN nimble build -d:danger -d:lto -d:strip \
&& nimble scss \
&& nimble md
FROM alpine:3.17
WORKDIR /src/
RUN apk --no-cache add ca-certificates pcre openssl1.1-compat
COPY --from=nim /src/nitter/nitter ./
COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
COPY --from=nim /src/nitter/public ./public
EXPOSE 8080
CMD ./nitter

View file

@ -1,6 +1,7 @@
# Nitter
[![Test Matrix](https://github.com/zedeus/nitter/workflows/CI/CD/badge.svg)](https://github.com/zedeus/nitter/actions?query=workflow%3ACI/CD)
[![Test Matrix](https://github.com/zedeus/nitter/workflows/Tests/badge.svg)](https://github.com/zedeus/nitter/actions/workflows/run-tests.yml)
[![Test Matrix](https://github.com/zedeus/nitter/workflows/Docker/badge.svg)](https://github.com/zedeus/nitter/actions/workflows/build-docker.yml)
[![License](https://img.shields.io/github/license/zedeus/nitter?style=flat)](#license)
A free and open source alternative Twitter front-end focused on privacy and
@ -34,7 +35,7 @@ XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscW
## Resources
The wiki contains
The wiki contains
[a list of instances](https://github.com/zedeus/nitter/wiki/Instances) and
[browser extensions](https://github.com/zedeus/nitter/wiki/Extensions)
maintained by the community.
@ -67,9 +68,10 @@ Twitter account.
## Installation
### Dependencies
* libpcre
* libsass
* redis
- libpcre
- libsass
- redis
To compile Nitter you need a Nim installation, see
[nim-lang.org](https://nim-lang.org/install.html) for details. It is possible to
@ -108,25 +110,32 @@ performance reasons.
### Docker
#### NOTE: For ARM64/ARM support, please use [unixfox's image](https://quay.io/repository/unixfox/nitter?tab=tags), more info [here](https://github.com/zedeus/nitter/issues/399#issuecomment-997263495)
Page for the Docker image: https://hub.docker.com/r/zedeus/nitter
#### NOTE: For ARM64 support, please use the separate ARM64 docker image: [`zedeus/nitter:latest-arm64`](https://hub.docker.com/r/zedeus/nitter/tags).
To run Nitter with Docker, you'll need to install and run Redis separately
before you can run the container. See below for how to also run Redis using
Docker.
To build and run Nitter in Docker:
```bash
docker build -t nitter:latest .
docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest
```
Note: For ARM64, use this Dockerfile: [`Dockerfile.arm64`](https://github.com/zedeus/nitter/blob/master/Dockerfile.arm64).
A prebuilt Docker image is provided as well:
```bash
docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host zedeus/nitter:latest
```
Using docker-compose to run both Nitter and Redis as different containers:
Change `redisHost` from `localhost` to `nitter-redis` in `nitter.conf`, then run:
```bash
docker-compose up -d
```

View file

@ -1,5 +1,6 @@
--define:ssl
--define:useStdLib
--threads:off
# workaround httpbeast file upload bug
--assertions:off

View file

@ -8,7 +8,7 @@ services:
ports:
- "127.0.0.1:8080:8080" # Replace with "8080:8080" if you don't use a reverse proxy
volumes:
- ./nitter.conf:/src/nitter.conf:ro
- ./nitter.conf:/src/nitter.conf:Z,ro
depends_on:
- nitter-redis
restart: unless-stopped
@ -17,6 +17,12 @@ services:
interval: 30s
timeout: 5s
retries: 2
user: "998:998"
read_only: true
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
nitter-redis:
image: redis:6-alpine
@ -30,6 +36,12 @@ services:
interval: 30s
timeout: 5s
retries: 2
user: "999:1000"
read_only: true
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
volumes:
nitter-redis:

View file

@ -1,11 +1,11 @@
[Server]
hostname = "nitter.net" # for generating links, change this to your own domain/ip
title = "nitter"
address = "0.0.0.0"
port = 8080
https = false # disable to enable cookies when not using https
httpMaxConnections = 100
staticDir = "./public"
title = "nitter"
hostname = "nitter.net"
[Cache]
listMinutes = 240 # how long to cache list info (not the tweets, so keep it high)
@ -13,9 +13,9 @@ rssMinutes = 10 # how long to cache rss queries
redisHost = "localhost" # Change to "nitter-redis" if using docker-compose
redisPort = 6379
redisPassword = ""
redisConnections = 20 # connection pool size
redisConnections = 20 # minimum open connections in pool
redisMaxConnections = 30
# max, new connections are opened when none are available, but if the pool size
# new connections are opened when none are available, but if the pool size
# goes above this, they're closed when released. don't worry about this unless
# you receive tons of requests per second
@ -23,23 +23,22 @@ redisMaxConnections = 30
hmacKey = "secretkey" # random key for cryptographic signing of video urls
base64Media = false # use base64 encoding for proxied media urls
enableRSS = true # set this to false to disable RSS feeds
enableDebug = false # enable request logs and debug endpoints
enableDebug = false # enable request logs and debug endpoints (/.tokens)
proxy = "" # http/https url, SOCKS proxies are not supported
proxyAuth = ""
tokenCount = 10
# minimum amount of usable tokens. tokens are used to authorize API requests,
# but they expire after ~1 hour, and have a limit of 187 requests.
# the limit gets reset every 15 minutes, and the pool is filled up so there's
# always at least $tokenCount usable tokens. again, only increase this if
# you receive major bursts all the time
# but they expire after ~1 hour, and have a limit of 500 requests per endpoint.
# the limits reset every 15 minutes, and the pool is filled up so there's
# always at least `tokenCount` usable tokens. only increase this if you receive
# major bursts all the time and don't have a rate limiting setup via e.g. nginx
# Change default preferences here, see src/prefs_impl.nim for a complete list
[Preferences]
theme = "Nitter"
replaceTwitter = "nitter.net"
replaceYouTube = "piped.kavin.rocks"
replaceYouTube = "piped.video"
replaceReddit = "teddit.net"
replaceInstagram = ""
proxyVideos = true
hlsPlayback = false
infiniteScroll = false

View file

@ -11,18 +11,18 @@ bin = @["nitter"]
# Dependencies
requires "nim >= 1.4.8"
requires "jester >= 0.5.0"
requires "karax#5498909"
requires "sass#e683aa1"
requires "nimcrypto#a5742a9"
requires "markdown#a661c26"
requires "packedjson#d11d167"
requires "supersnappy#2.1.1"
requires "jester#baca3f"
requires "karax#5cf360c"
requires "sass#7dfdd03"
requires "nimcrypto#4014ef9"
requires "markdown#158efe3"
requires "packedjson#9e6fbb6"
requires "supersnappy#6c94198"
requires "redpool#8b7c1db"
requires "https://github.com/zedeus/redis#d0a0e6f"
requires "zippy#0.7.3"
requires "flatty#0.2.3"
requires "jsony#d0e69bd"
requires "zippy#ca5989a"
requires "flatty#e668085"
requires "jsony#ea811be"
# Tasks

File diff suppressed because one or more lines are too long

View file

@ -1,4 +1,5 @@
User-agent: *
Disallow: /
Crawl-delay: 1
User-agent: Twitterbot
Disallow:

View file

@ -4,123 +4,143 @@ import packedjson
import types, query, formatters, consts, apiutils, parser
import experimental/parser as newParser
proc getGraphUser*(id: string): Future[User] {.async.} =
proc getGraphUser*(username: string): Future[User] {.async.} =
if username.len == 0: return
let
variables = %*{"screen_name": username}
params = {"variables": $variables, "features": gqlFeatures}
js = await fetchRaw(graphUser ? params, Api.userScreenName)
result = parseGraphUser(js)
proc getGraphUserById*(id: string): Future[User] {.async.} =
if id.len == 0 or id.any(c => not c.isDigit): return
let
variables = %*{"userId": id, "withSuperFollowsUserFields": true}
js = await fetchRaw(graphUser ? {"variables": $variables}, Api.userRestId)
variables = %*{"userId": id}
params = {"variables": $variables, "features": gqlFeatures}
js = await fetchRaw(graphUserById ? params, Api.userRestId)
result = parseGraphUser(js)
proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
variables = userTweetsVariables % [id, cursor]
params = {"variables": variables, "features": gqlFeatures}
(url, apiId) = case kind
of TimelineKind.tweets: (graphUserTweets, Api.userTweets)
of TimelineKind.replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
of TimelineKind.media: (graphUserMedia, Api.userMedia)
js = await fetch(url ? params, apiId)
result = parseGraphTimeline(js, "user", after)
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
variables = listTweetsVariables % [id, cursor]
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphListTweets ? params, Api.listTweets)
result = parseGraphTimeline(js, "list", after)
proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
let
variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false}
url = graphListBySlug ? {"variables": $variables}
result = parseGraphList(await fetch(url, Api.listBySlug))
variables = %*{"screenName": name, "listSlug": list}
params = {"variables": $variables, "features": gqlFeatures}
result = parseGraphList(await fetch(graphListBySlug ? params, Api.listBySlug))
proc getGraphList*(id: string): Future[List] {.async.} =
let
variables = %*{"listId": id, "withHighlightedLabel": false}
url = graphList ? {"variables": $variables}
result = parseGraphList(await fetch(url, Api.list))
variables = %*{"listId": id}
params = {"variables": $variables, "features": gqlFeatures}
result = parseGraphList(await fetch(graphListById ? params, Api.list))
proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
if list.id.len == 0: return
var
variables = %*{
"listId": list.id,
"withSuperFollowsUserFields": false,
"withBirdwatchPivots": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withSuperFollowsTweetFields": false
"withReactionsPerspective": false
}
if after.len > 0:
variables["cursor"] = % after
let url = graphListMembers ? {"variables": $variables}
let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
if id.len == 0: return
let
ps = genParams({"list_id": id, "ranking_mode": "reverse_chronological"}, after)
url = listTimeline ? ps
result = parseTimeline(await fetch(url, Api.timeline), after)
variables = tweetResultVariables % id
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphTweetResult ? params, Api.tweetResult)
result = parseGraphTweetResult(js)
proc getUser*(username: string): Future[User] {.async.} =
if username.len == 0: return
let
ps = genParams({"screen_name": username})
json = await fetchRaw(userShow ? ps, Api.userShow)
result = parseUser(json, username)
proc getUserById*(userId: string): Future[User] {.async.} =
if userId.len == 0: return
let
ps = genParams({"user_id": userId})
json = await fetchRaw(userShow ? ps, Api.userShow)
result = parseUser(json)
proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
if id.len == 0: return
let
ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
url = timeline / (id & ".json") ? ps
result = parseTimeline(await fetch(url, Api.timeline), after)
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
variables = tweetVariables % [id, cursor]
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphTweet ? params, Api.tweetDetail)
result = parseGraphConversation(js, id)
proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let url = mediaTimeline / (id & ".json") ? genParams(cursor=after)
result = parseTimeline(await fetch(url, Api.timeline), after)
proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
result = (await getGraphTweet(id, after)).replies
result.beginning = after.len == 0
proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
result = await getGraphTweet(id)
if after.len > 0:
result.replies = await getReplies(id, after)
proc getGraphSearch*(query: Query; after=""): Future[Result[Tweet]] {.async.} =
let q = genQueryParam(query)
if q.len == 0 or q == emptyQuery:
return Result[Tweet](query: query, beginning: true)
var
variables = %*{
"rawQuery": q,
"count": 20,
"product": "Latest",
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false
}
if after.len > 0:
variables["cursor"] = % after
let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
result = parseGraphSearch(await fetch(url, Api.search), after)
result.query = query
proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
if query.text.len == 0:
return Result[User](query: query, beginning: true)
var url = userSearch ? {
"q": query.text,
"skip_status": "1",
"count": "20",
"page": page
}
result = parseUsers(await fetchRaw(url, Api.userSearch))
result.query = query
if page.len == 0:
result.bottom = "2"
elif page.allCharsInSet(Digits):
result.bottom = $(parseInt(page) + 1)
proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
if name.len == 0: return
let
ps = genParams({"screen_name": name, "trim_user": "true"},
count="18", ext=false)
count="18", ext=false)
url = photoRail ? ps
result = parsePhotoRail(await fetch(url, Api.timeline))
proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
when T is User:
const
searchMode = ("result_filter", "user")
parse = parseUsers
fetchFunc = fetchRaw
else:
const
searchMode = ("tweet_search_mode", "live")
parse = parseTimeline
fetchFunc = fetch
let q = genQueryParam(query)
if q.len == 0 or q == emptyQuery:
return Result[T](beginning: true, query: query)
let url = search ? genParams(searchParams & @[("q", q), searchMode], after)
try:
result = parse(await fetchFunc(url, Api.search), after)
result.query = query
except InternalError:
return Result[T](beginning: true, query: query)
proc getTweetImpl(id: string; after=""): Future[Conversation] {.async.} =
let url = tweet / (id & ".json") ? genParams(cursor=after)
result = parseConversation(await fetch(url, Api.tweet), id)
proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
result = (await getTweetImpl(id, after)).replies
result.beginning = after.len == 0
proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
result = await getTweetImpl(id)
if after.len > 0:
result.replies = await getReplies(id, after)
proc getStatus*(id: string): Future[Tweet] {.async.} =
let url = status / (id & ".json") ? genParams()
result = parseStatus(await fetch(url, Api.status))
proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
let client = newAsyncHttpClient(maxRedirects=0)
try:

View file

@ -17,13 +17,13 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
result &= p
if ext:
result &= ("ext", "mediaStats")
result &= ("include_ext_alt_text", "true")
result &= ("include_ext_media_availability", "true")
result &= ("include_ext_alt_text", "1")
result &= ("include_ext_media_availability", "1")
if count.len > 0:
result &= ("count", count)
if cursor.len > 0:
# The raw cursor often has plus signs, which sometimes get turned into spaces,
# so we need to them back into a plus
# so we need to turn them back into a plus
if " " in cursor:
result &= ("cursor", cursor.replace(" ", "+"))
else:
@ -44,7 +44,7 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
})
template updateToken() =
if api != Api.search and resp.headers.hasKey(rlRemaining):
if resp.headers.hasKey(rlRemaining):
let
remaining = parseInt(resp.headers[rlRemaining])
reset = parseInt(resp.headers[rlReset])
@ -61,12 +61,15 @@ template fetchImpl(result, fetchBody) {.dirty.} =
try:
var resp: AsyncResponse
pool.use(genHeaders(token)):
resp = await c.get($url)
result = await resp.body
template getContent =
resp = await c.get($url)
result = await resp.body
getContent()
if resp.status == $Http503:
badClient = true
raise newException(InternalError, result)
raise newException(BadClientError, "Bad client")
if result.len > 0:
if resp.headers.getOrDefault("content-encoding") == "gzip":
@ -82,6 +85,9 @@ template fetchImpl(result, fetchBody) {.dirty.} =
raise newException(InternalError, $url)
except InternalError as e:
raise e
except BadClientError as e:
release(token, used=true)
raise e
except Exception as e:
echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url
if "length" notin e.msg and "descriptor" notin e.msg:
@ -100,7 +106,7 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
updateToken()
let error = result.getError
if error in {invalidToken, forbidden, badToken}:
if error in {invalidToken, badToken}:
echo "fetch error: ", result.getError
release(token, invalid=true)
raise rateLimitError()
@ -115,7 +121,7 @@ proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
if result.startsWith("{\"errors"):
let errors = result.fromJson(Errors)
if errors in {invalidToken, forbidden, badToken}:
if errors in {invalidToken, badToken}:
echo "fetch error: ", errors
release(token, invalid=true)
raise rateLimitError()

View file

@ -1,28 +1,28 @@
# SPDX-License-Identifier: AGPL-3.0-only
import uri, sequtils
import uri, sequtils, strutils
const
auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
api = parseUri("https://api.twitter.com")
activate* = $(api / "1.1/guest/activate.json")
userShow* = api / "1.1/users/show.json"
photoRail* = api / "1.1/statuses/media_timeline.json"
status* = api / "1.1/statuses/show"
search* = api / "2/search/adaptive.json"
timelineApi = api / "2/timeline"
timeline* = timelineApi / "profile"
mediaTimeline* = timelineApi / "media"
listTimeline* = timelineApi / "list.json"
tweet* = timelineApi / "conversation"
userSearch* = api / "1.1/users/search.json"
graphql = api / "graphql"
graphUser* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
graphUser* = graphql / "pVrmNaXcxPjisIvKtLDMEA/UserByScreenName"
graphUserById* = graphql / "1YAM811Q8Ry4XyPpJclURQ/UserByRestId"
graphUserTweets* = graphql / "WzJjibAcDa-oCjCcLOotcg/UserTweets"
graphUserTweetsAndReplies* = graphql / "fn9oRltM1N4thkh5CVusPg/UserTweetsAndReplies"
graphUserMedia* = graphql / "qQoeS7szGavsi8-ehD2AWg/UserMedia"
graphTweet* = graphql / "miKSMGb2R1SewIJv2-ablQ/TweetDetail"
graphTweetResult* = graphql / "0kc0a_7TTr3dvweZlMslsQ/TweetResultByRestId"
graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline"
graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug"
graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
graphListTweets* = graphql / "jZntL0oVJSdjhmPcdbw_eA/ListLatestTweetsTimeline"
timelineParams* = {
"include_profile_interstitial_type": "0",
@ -33,27 +33,89 @@ const
"include_mute_edge": "0",
"include_can_dm": "0",
"include_can_media_tag": "1",
"include_ext_is_blue_verified": "1",
"skip_status": "1",
"cards_platform": "Web-12",
"include_cards": "1",
"include_composer_source": "false",
"include_composer_source": "0",
"include_reply_count": "1",
"tweet_mode": "extended",
"include_entities": "true",
"include_user_entities": "true",
"include_ext_media_color": "false",
"send_error_codes": "true",
"simple_quoted_tweet": "true",
"include_quote_count": "true"
"include_entities": "1",
"include_user_entities": "1",
"include_ext_media_color": "0",
"send_error_codes": "1",
"simple_quoted_tweet": "1",
"include_quote_count": "1"
}.toSeq
searchParams* = {
"query_source": "typed_query",
"pc": "1",
"spelling_corrections": "1"
}.toSeq
## top: nothing
## latest: "tweet_search_mode: live"
## user: "result_filter: user"
## photos: "result_filter: photos"
## videos: "result_filter: videos"
gqlFeatures* = """{
"blue_business_profile_image_shape_enabled": false,
"creator_subscriptions_tweet_preview_api_enabled": true,
"freedom_of_speech_not_reach_fetch_enabled": false,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
"highlights_tweets_tab_ui_enabled": false,
"interactive_text_enabled": false,
"longform_notetweets_consumption_enabled": true,
"longform_notetweets_inline_media_enabled": false,
"longform_notetweets_richtext_consumption_enabled": true,
"longform_notetweets_rich_text_read_enabled": false,
"responsive_web_edit_tweet_api_enabled": false,
"responsive_web_enhance_cards_enabled": false,
"responsive_web_graphql_exclude_directive_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"responsive_web_graphql_timeline_navigation_enabled": false,
"responsive_web_text_conversations_enabled": false,
"responsive_web_twitter_blue_verified_badge_is_enabled": true,
"rweb_lists_timeline_redesign_enabled": true,
"spaces_2022_h2_clipping": true,
"spaces_2022_h2_spaces_communities": true,
"standardized_nudges_misinfo": false,
"tweet_awards_web_tipping_enabled": false,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
"tweetypie_unmention_optimization_enabled": false,
"verified_phone_label_enabled": false,
"vibe_api_enabled": false,
"view_counts_everywhere_api_enabled": false
}""".replace(" ", "").replace("\n", "")
tweetVariables* = """{
"focalTweetId": "$1",
$2
"withBirdwatchNotes": false,
"includePromotedContent": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withVoice": false
}"""
tweetResultVariables* = """{
"tweetId": "$1",
"includePromotedContent": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withVoice": false,
"withCommunity": false
}"""
userTweetsVariables* = """{
"userId": "$1", $2
"count": 20,
"includePromotedContent": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withVoice": false,
"withV2Timeline": true
}"""
listTweetsVariables* = """{
"listId": "$1", $2
"count": 20,
"includePromotedContent": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withVoice": false
}"""

View file

@ -1,2 +1,2 @@
import parser/[user, graphql, timeline]
export user, graphql, timeline
import parser/[user, graphql]
export user, graphql

View file

@ -1,11 +1,17 @@
import options
import jsony
import user, ../types/[graphuser, graphlistmembers]
from ../../types import User, Result, Query, QueryKind
proc parseGraphUser*(json: string): User =
let raw = json.fromJson(GraphUser)
if raw.data.user.result.reason.get("") == "Suspended":
return User(suspended: true)
result = toUser raw.data.user.result.legacy
result.id = raw.data.user.result.restId
result.verified = result.verified or raw.data.user.result.isBlueVerified
proc parseGraphListMembers*(json, cursor: string): Result[User] =
result = Result[User](

View file

@ -1,30 +0,0 @@
import std/[strutils, tables]
import jsony
import user, ../types/timeline
from ../../types import Result, User
proc getId(id: string): string {.inline.} =
let start = id.rfind("-")
if start < 0: return id
id[start + 1 ..< id.len]
proc parseUsers*(json: string; after=""): Result[User] =
result = Result[User](beginning: after.len == 0)
let raw = json.fromJson(Search)
if raw.timeline.instructions.len == 0:
return
for i in raw.timeline.instructions:
if i.addEntries.entries.len > 0:
for e in i.addEntries.entries:
let id = e.entryId.getId
if e.entryId.startsWith("user"):
if id in raw.globalObjects.users:
result.content.add toUser raw.globalObjects.users[id]
elif e.entryId.startsWith("cursor"):
let cursor = e.content.operation.cursor
if cursor.cursorType == "Top":
result.top = cursor.value
elif cursor.cursorType == "Bottom":
result.bottom = cursor.value

View file

@ -66,6 +66,8 @@ proc parseMedia(component: Component; card: UnifiedCard; result: var Card) =
durationMs: videoInfo.durationMillis,
variants: videoInfo.variants
)
of model3d:
result.title = "Unsupported 3D model ad"
proc parseUnifiedCard*(json: string): Card =
let card = json.fromJson(UnifiedCard)
@ -82,6 +84,10 @@ proc parseUnifiedCard*(json: string): Card =
component.parseMedia(card, result)
of buttonGroup:
discard
of ComponentType.hidden:
result.kind = CardKind.hidden
of ComponentType.unknown:
echo "ERROR: Unknown component type: ", json
case component.kind
of twitterListDetails:

View file

@ -2,7 +2,7 @@ import std/[algorithm, unicode, re, strutils, strformat, options, nre]
import jsony
import utils, slices
import ../types/user as userType
from ../../types import User, Error
from ../../types import Result, User, Error
let
unRegex = re.re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
@ -76,3 +76,12 @@ proc parseUser*(json: string; username=""): User =
else: echo "[error - parseUser]: ", error
result = toUser json.fromJson(RawUser)
proc parseUsers*(json: string; after=""): Result[User] =
result = Result[User](beginning: after.len == 0)
# starting with '{' means it's an error
if json[0] == '[':
let raw = json.fromJson(seq[RawUser])
for user in raw:
result.content.add user.toUser

View file

@ -1,3 +1,4 @@
import options
import user
type
@ -10,3 +11,5 @@ type
UserResult = object
legacy*: RawUser
restId*: string
isBlueVerified*: bool
reason*: Option[string]

Some files were not shown because too many files have changed in this diff Show more