add socials tests

subrepo:
  subdir:   "apps/twscrape"
  merged:   "e4dce5d3"
upstream:
  origin:   "https://github.com/vladkens/twscrape"
  branch:   "main"
  commit:   "e4dce5d3"
git-subrepo:
  version:  "0.4.9"
  origin:   "???"
  commit:   "???"

apps/bright/lib/bright/oban_workers/scrape_posts.ex

@@ -0,0 +1,64 @@
+# defmodule Bright.ObanWorkers.ScrapeX do
+#   alias Bright.Vtubers.Vtuber
+#   use Oban.Worker, queue: :default, max_attempts: 3
+
+#   alias Bright.Streams.Vod
+
+#   alias Bright.{
+#     Repo,
+#     Downloader,
+#     B2,
+#     Images
+#   }
+
+#   require Logger
+
+#   @impl Oban.Worker
+#   def perform(%Oban.Job{args: %{"vtuber_id" => vtuber_id}}) do
+#     Logger.info(">>>> Scrape X is performing. with vtuber_id=#{vtuber_id}")
+
+#     # @todo get vtuber from db
+#     # @todo get vtuber's X account
+#     # @todo get nitter URL
+#     # @todo get X posts
+#     # @todo queue posts we haven't yet processed
+#     # @todo parse posts in the queue to find CB/Fansly/OF invite links
+#     # @todo for each post with an invite, create a stream in the db
+
+#     # case Repo.get(Vtuber, vtuber_id) do
+
+#     #   nil ->
+#     #     Logger.error("Vtuber id #{vtuber_id} not found.")
+#     #     {:error, "Vtuber not found"}
+
+#     #   %Vtuber{} = vtuber ->
+#     #     with {:ok, } <-
+
+#     # end
+
+#     case Repo.get(Vod, vod_id) do
+#       nil ->
+#         Logger.error("VOD ID #{vod_id} not found")
+#         {:error, "VOD not found"}
+
+#       %Vod{origin_temp_input_url: origin_temp_input_url} = vod ->
+#         with {:ok, local_filename} <- Downloader.get(origin_temp_input_url),
+#              {:ok, thumbnail_filename} <- Images.create_thumbnail(local_filename),
+#              {:ok, s3Asset} <- B2.put(thumbnail_filename) do
+#           update_vod_with_thumbnail_url(vod, s3Asset.cdn_url)
+#         else
+#           {:error, reason} ->
+#             Logger.error("Failed to create thumbnail for VOD ID #{vod_id}: #{inspect(reason)}")
+#             {:error, reason}
+#         end
+#     end
+#   end
+
+#   # defp generate_thumbnail_url(basename), do: "#{Application.get_env(:bright, :public_s3_endpoint)}/#{basename}"
+#   defp update_vod_with_thumbnail_url(vod, thumbnail_url) do
+#     case Repo.update(vod |> Ecto.Changeset.change(thumbnail_url: thumbnail_url)) do
+#       {:ok, updated_vod} -> {:ok, updated_vod}
+#       {:error, changeset} -> {:error, changeset}
+#     end
+#   end
+# end

apps/bright/lib/bright/platforms/x.ex

@@ -1,22 +0,0 @@
-defmodule Bright.Platforms.XPost do
-  use Ecto.Schema
-  import Ecto.Changeset
-
-  schema "x_posts" do
-    field :raw, :string
-    field :url, :string
-    field :date, :utc_datetime
-    field :is_invitation, :boolean
-
-    belongs_to :vtuber, Bright.Vtubers.Vtuber
-
-    timestamps(type: :utc_datetime)
-  end
-
-  @doc false
-  def changeset(platform, attrs) do
-    platform
-    |> cast(attrs, [:raw, :url, :date])
-    |> validate_required([:raw, :url, :date])
-  end
-end

apps/bright/lib/bright/shopping_cart/cart.ex

@@ -1,19 +0,0 @@
-defmodule Bright.ShoppingCart.Cart do
-  use Ecto.Schema
-  import Ecto.Changeset
-
-  schema "carts" do
-    field :user_uuid, Ecto.UUID
-    has_many :items, Bright.ShoppingCart.CartItem
-
-    timestamps(type: :utc_datetime)
-  end
-
-  @doc false
-  def changeset(cart, attrs) do
-    cart
-    |> cast(attrs, [:user_uuid])
-    |> validate_required([:user_uuid])
-    |> unique_constraint(:user_uuid)
-  end
-end

apps/bright/lib/bright/shopping_cart/cart_item.ex

@@ -1,21 +0,0 @@
-defmodule Bright.ShoppingCart.CartItem do
-  use Ecto.Schema
-  import Ecto.Changeset
-
-  schema "cart_items" do
-    field :price_when_carted, :decimal
-    field :quantity, :integer
-    belongs_to :cart, Bright.ShoppingCart.Cart
-    belongs_to :product, Bright.Catalog.Product
-
-    timestamps(type: :utc_datetime)
-  end
-
-  @doc false
-  def changeset(cart_item, attrs) do
-    cart_item
-    |> cast(attrs, [:price_when_carted, :quantity])
-    |> validate_required([:price_when_carted, :quantity])
-    |> validate_number(:quantity, greater_than_or_equal_to: 0, less_than: 100)
-  end
-end

apps/bright/lib/bright/socials.ex

@@ -0,0 +1,7 @@
+defmodule Bright.Socials do
+  @moduledoc """
+    Socials context, for functions for interacting with social media platforms
+  """
+
+
+end

apps/bright/lib/bright/socials/rss_parser.ex

@@ -0,0 +1,49 @@
+
+defmodule Bright.Socials.RSSParser do
+  @moduledoc """
+  Module to parse X RSS feeds and extract URLs, datestamps, and text content from items.
+  """
+
+  @doc """
+  Extracts URLs, datestamps, and text content from the RSS feed data.
+
+  # Example usage:
+  # Assuming `data` is the parsed RSS feed data you provided.
+
+  ```elixir
+  item_details = RSSParser.extract_item_details(data)
+  IO.inspect(item_details)
+  ```
+
+  """
+  def extract_item_details(data) do
+    data
+    |> find_items()
+    |> Enum.map(fn item ->
+      %{
+        url: extract_value(item, :link),
+        datestamp: extract_value(item, :pubDate),
+        text: extract_value(item, :title)
+      }
+    end)
+  end
+
+  # Helper function to find all :item elements in the nested structure
+  defp find_items(data) do
+    data
+    |> List.wrap()  # Ensure data is treated as a list
+    |> Enum.flat_map(fn
+      %{name: :item} = item -> [item]  # If it's an item, return it
+      %{value: children} -> find_items(children)  # Recursively search children
+      _ -> []  # Skip anything else
+    end)
+  end
+
+  # Helper function to extract the value for a given key from an item
+  defp extract_value(item, key) do
+    case Enum.find(item[:value], fn %{name: name} -> name == key end) do
+      %{value: [value]} -> value  # Extract the value if found
+      _ -> nil  # Return nil if the key is not found
+    end
+  end
+end

apps/bright/lib/bright/socials/x_post.ex

@@ -0,0 +1,84 @@
+defmodule Bright.Socials.XPost do
+  use Ecto.Schema
+  import Ecto.Changeset
+  alias Bright.Vtubers.Vtuber
+  alias Bright.Socials.RSSParser
+  alias Quinn
+  require Logger
+
+  ## @todo this needs to be pulled from the database
+  ## @todo rss.app doesn't scale.
+  ## @todo we need a better way to get data from vtuber's X accounts.
+  @sample_rss_feed "https://rss.app/feeds/FhPetvUY036xiFau.xml"
+
+  @doc """
+  We cache the posts in the db so it's clear which tweets we've read and which ones we haven't.
+  We only parse tweets which haven't been cached.
+  """
+  schema "x_posts" do
+    field :raw, :string
+    field :url, :string
+    field :date, :utc_datetime
+    field :is_invitation, :boolean
+
+    belongs_to :vtuber, Bright.Vtubers.Vtuber
+
+    timestamps(type: :utc_datetime)
+  end
+
+  @doc false
+  def changeset(post, attrs) do
+    post
+    |> cast(attrs, [:raw, :url, :date])
+    |> validate_required([:raw, :url, :date])
+  end
+
+
+  # def get_posts(vtuber_id) do
+  #   case Repo.get(Vtuber, vtuber_id) do
+  #     nil ->
+  #       Logger.error("Vtuber id #{vtuber_id} not found.")
+  #       {:error, "Vtuber not found"}
+
+  #     %Vtuber{} = vtuber ->
+  #       with {:ok, posts} <- get_posts(@sample_rss_feed) # @todo This feed URL needs to be dynamically looked up
+  #       do
+  #         save_posts(posts)
+  #       end
+  #   end
+
+
+  # end
+
+
+  @doc """
+    We read X posts via RSS URL.
+  """
+  def get_posts(feed_url) do
+    case HTTPoison.get(feed_url) do
+      {:ok, %HTTPoison.Response{ body: body }} ->
+
+        data = Quinn.parse(body)
+        extract = RSSParser.extract_item_details(data)
+        Logger.debug("we GETted a rss feed. Parsed data=#{inspect(data)}")
+        Logger.debug("we parsed the rss feed using RSSParser. parsed=#{inspect(extract)}")
+
+
+        {:ok, extract}
+
+      {:error, reason} ->
+        Logger.debug("failed to get_posts. reason=#{inspect(reason)}")
+        {:error, reason}
+    end
+  end
+
+
+  @doc """
+  save the posts to the db
+  """
+  def save_posts(posts) do
+    Logger.debug("@todo implement save_posts()")
+  end
+
+
+end

apps/bright/mix.exs

@@ -73,17 +73,8 @@ defmodule Bright.MixProject do
       {:atomex, "~> 0.3.0"},
       {:bento, "~> 1.0"},
       {:identicon_svg, "~> 0.9"},
-      {:excoveralls, "~> 0.18", only: :test}
-      # {:membrane_core, "~> 1.0"},
-      # {:membrane_mpeg_ts_plugin, "~> 1.0.3"},
-      # {:membrane_file_plugin, "~> 0.17.2"},
-      # {:membrane_mp4_plugin, "~> 0.35.2"},
-      # {:membrane_http_adaptive_stream_plugin, "> 0.0.0"},
-      # {:membrane_h264_ffmpeg_plugin, "~> 0.32.5"},
-      # {:membrane_aac_plugin, "~> 0.11.0"},
-      # {:membrane_hackney_plugin, "~> 0.6.0"}, # incompatible with membrane_core 1.1.2
-      # {:membrane_mpegts_plugin, "~> 0.4.0"}, # official module is 4 years outdated
-      # {:membrane_mpegts_plugin, path: "/home/cj/Documents/membrane_mpegts_plugin"},
+      {:excoveralls, "~> 0.18", only: :test},
+      {:quinn, "~> 1.1.3"},
     ]
   end
 

apps/bright/mix.lock

@@ -31,6 +31,7 @@
   "ex_m3u8": {:hex, :ex_m3u8, "0.14.2", "3eb17f936e2ca2fdcde11664f3a543e75a94814d928098e050bda5b1e149c021", [:mix], [{:nimble_parsec, "~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:typed_struct, "~> 0.3.0", [hex: :typed_struct, repo: "hexpm", optional: false]}], "hexpm", "d2a1fb4382a521cce7f966502ecce6187f286ca2852dbb0dcc25dea72f8ba039"},
   "excoveralls": {:hex, :excoveralls, "0.18.5", "e229d0a65982613332ec30f07940038fe451a2e5b29bce2a5022165f0c9b157e", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "523fe8a15603f86d64852aab2abe8ddbd78e68579c8525ae765facc5eae01562"},
   "expo": {:hex, :expo, "1.1.0", "f7b9ed7fb5745ebe1eeedf3d6f29226c5dd52897ac67c0f8af62a07e661e5c75", [:mix], [], "hexpm", "fbadf93f4700fb44c331362177bdca9eeb8097e8b0ef525c9cc501cb9917c960"},
+  "fast_rss": {:hex, :fast_rss, "0.5.0", "2e79c7142ff7843243635fc3f067999eb26beb0154699f2fa5a39d687629a0c9", [:mix], [{:rustler, "~> 0.29.0", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.6", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}], "hexpm", "dafb3eccc024d7366a0ef78521a26cfaf2075be534cfe1ec70d72903eabdd44d"},
   "ffmpex": {:hex, :ffmpex, "0.11.0", "70d2e211a70e1d8cc1a81d73208d5efedda59d82db4c91160c79e5461529d291", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:rambo, "~> 0.3.0", [hex: :rambo, repo: "hexpm", optional: false]}], "hexpm", "2429d67badc91957ace572b9169615619740904a58791289ba54d99e57a164eb"},
   "file_system": {:hex, :file_system, "1.1.0", "08d232062284546c6c34426997dd7ef6ec9f8bbd090eb91780283c9016840e8f", [:mix], [], "hexpm", "bfcf81244f416871f2a2e15c1b515287faa5db9c6bcf290222206d120b3d43f6"},
   "finch": {:hex, :finch, "0.19.0", "c644641491ea854fc5c1bbaef36bfc764e3f08e7185e1f084e35e0672241b76d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc5324ce209125d1e2fa0fcd2634601c52a787aff1cd33ee833664a5af4ea2b6"},
@@ -95,10 +96,12 @@
   "postgrex": {:hex, :postgrex, "0.19.3", "a0bda6e3bc75ec07fca5b0a89bffd242ca209a4822a9533e7d3e84ee80707e19", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "d31c28053655b78f47f948c85bb1cf86a9c1f8ead346ba1aa0d0df017fa05b61"},
   "pythonx": {:hex, :pythonx, "0.2.5", "05660dc8548a4ab5da5b7f7977c6a5a3fa16eefadbe54077f9a176c9d386be27", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.8", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.3", [hex: :req, repo: "hexpm", optional: false]}], "hexpm", "66d2179e37be527cbecf47097c15aea28a1dbcb2c6e965407c89ca1e1ac74d17"},
   "qex": {:hex, :qex, "0.5.1", "0d82c0f008551d24fffb99d97f8299afcb8ea9cf99582b770bd004ed5af63fd6", [:mix], [], "hexpm", "935a39fdaf2445834b95951456559e9dc2063d0a055742c558a99987b38d6bab"},
+  "quinn": {:hex, :quinn, "1.1.3", "83df8a990d898b7756dce0471439328994396d9cc407763f6c313f632e4571e3", [:mix], [], "hexpm", "685c92ad76f7398ffb86b7a3ba9de244ff66988d497e60b14360878d57a255d1"},
   "rambo": {:hex, :rambo, "0.3.4", "8962ac3bd1a633ee9d0e8b44373c7913e3ce3d875b4151dcd060886092d2dce7", [:mix], [], "hexpm", "0cc54ed089fbbc84b65f4b8a774224ebfe60e5c80186fafc7910b3e379ad58f1"},
   "ratio": {:hex, :ratio, "4.0.1", "3044166f2fc6890aa53d3aef0c336f84b2bebb889dc57d5f95cc540daa1912f8", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:numbers, "~> 5.2.0", [hex: :numbers, repo: "hexpm", optional: false]}], "hexpm", "c60cbb3ccdff9ffa56e7d6d1654b5c70d9f90f4d753ab3a43a6bf40855b881ce"},
   "redirect": {:hex, :redirect, "0.4.0", "98b46053504ee517bc3ad2fd04c064b64b48d339e1e18266355b30c4f8bb52b0", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.3 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "dfa29a8ecbad066ed0b73b34611cf24c78101719737f37bdf750f39197d67b97"},
   "req": {:hex, :req, "0.5.8", "50d8d65279d6e343a5e46980ac2a70e97136182950833a1968b371e753f6a662", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "d7fc5898a566477e174f26887821a3c5082b243885520ee4b45555f5d53f40ef"},
+  "rustler_precompiled": {:hex, :rustler_precompiled, "0.8.2", "5f25cbe220a8fac3e7ad62e6f950fcdca5a5a5f8501835d2823e8c74bf4268d5", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "63d1bd5f8e23096d1ff851839923162096364bac8656a4a3c00d1fff8e83ee0a"},
   "secure_random": {:hex, :secure_random, "0.5.1", "c5532b37c89d175c328f5196a0c2a5680b15ebce3e654da37129a9fe40ebf51b", [:mix], [], "hexpm", "1b9754f15e3940a143baafd19da12293f100044df69ea12db5d72878312ae6ab"},
   "shmex": {:hex, :shmex, "0.5.1", "81dd209093416bf6608e66882cb7e676089307448a1afd4fc906c1f7e5b94cf4", [:mix], [{:bunch_native, "~> 0.5.0", [hex: :bunch_native, repo: "hexpm", optional: false]}, {:bundlex, "~> 1.0", [hex: :bundlex, repo: "hexpm", optional: false]}], "hexpm", "c29f8286891252f64c4e1dac40b217d960f7d58def597c4e606ff8fbe71ceb80"},
   "squircle": {:hex, :squircle, "0.1.1", "2e82410f9cf80de1023e19f5cb75e3e74ffa12ef0d2dabee81b73604447f7d3d", [:mix], [], "hexpm", "fe1214a99c94aea157da106d56c360a7c52861f7608650fbdadf12a7fd60e170"},

apps/bright/test/bright/socials/rss_parser_test.exs

@@ -0,0 +1,60 @@
+defmodule Bright.RSSParserTest do
+  use Bright.DataCase
+
+  alias Bright.Socials.RSSParser
+
+  @sample_data %{
+    name: :rss,
+    value: [
+      %{
+        name: :channel,
+        value: [
+          %{
+            name: :item,
+            value: [
+              %{name: :title, value: ["Test Post 1"], attr: []},
+              %{name: :link, value: ["https://example.com/post/1"], attr: []},
+              %{name: :pubDate, value: ["Wed, 12 Oct 2022 12:00:00 GMT"], attr: []}
+            ],
+            attr: []
+          },
+          %{
+            name: :item,
+            value: [
+              %{name: :title, value: ["Test Post 2"], attr: []},
+              %{name: :link, value: ["https://example.com/post/2"], attr: []},
+              %{name: :pubDate, value: ["Wed, 12 Oct 2022 13:00:00 GMT"], attr: []}
+            ],
+            attr: []
+          }
+        ]
+      }
+    ]
+  }
+
+  describe "extract_item_details/1" do
+    @tag :unit
+    test "extracts URLs, datestamps, and text content from RSS data" do
+      result = RSSParser.extract_item_details(@sample_data)
+
+      assert result == [
+               %{
+                 url: "https://example.com/post/1",
+                 datestamp: "Wed, 12 Oct 2022 12:00:00 GMT",
+                 text: "Test Post 1"
+               },
+               %{
+                 url: "https://example.com/post/2",
+                 datestamp: "Wed, 12 Oct 2022 13:00:00 GMT",
+                 text: "Test Post 2"
+               }
+             ]
+    end
+
+    @tag :unit
+    test "returns an empty list if no items are found" do
+      empty_data = %{name: :rss, value: [%{name: :channel, value: []}]}
+      assert RSSParser.extract_item_details(empty_data) == []
+    end
+  end
+end

apps/bright/test/bright/socials/x_test.exs

@@ -0,0 +1,17 @@
+defmodule Bright.XPostTest do
+  use Bright.DataCase
+
+  alias Bright.Socials.XPost
+
+  @sample_feed "https://rss.app/feeds/FhPetvUY036xiFau.xml"
+
+  describe "x_posts" do
+    # import Bright.SocialsFixtures
+
+    @tag :integration
+    test "get_posts/1" do
+      {:ok, posts} = XPost.get_posts(@sample_feed)
+      assert length(posts) > 0
+    end
+  end
+end

apps/twscrape/.dockerignore

@@ -0,0 +1,10 @@
+.DS_Store
+.ruff_cache/
+.coverage
+accounts/
+results-raw/
+results-parsed/
+accounts.db*
+data/
+data_media/
+dist/

apps/twscrape/.gitattributes

@@ -0,0 +1 @@
+tests/mocked-data/* linguist-generated=true

apps/twscrape/.github/CODEOWNERS

@@ -0,0 +1 @@
+* @vladkens

apps/twscrape/.github/workflows/ci.yml

@@ -0,0 +1,52 @@
+name: build
+
+on:
+  push:
+  pull_request:
+
+env:
+  PIP_ROOT_USER_ACTION: ignore
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      
+      - name: install dependencies
+        run: pip install -e .[dev]
+
+      - run: make lint
+      - run: make test
+
+  release:
+    if: ${{ startsWith(github.ref, 'refs/tags/v') }}
+    runs-on: ubuntu-latest
+    needs: test
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - run: python -m pip install build
+      - run: python -m build --sdist --wheel --outdir dist/ .
+
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
+
+      - name: Create Github Release
+        uses: softprops/action-gh-release@v2
+        with:
+          generate_release_notes: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

apps/twscrape/.gitignore

@@ -0,0 +1,10 @@
+.DS_Store
+.ruff_cache/
+.coverage
+accounts/
+results-raw/
+results-parsed/
+accounts.db*
+data/
+data_media/
+dist/

apps/twscrape/.gitrepo

@@ -0,0 +1,12 @@
+; DO NOT EDIT (unless you know what you are doing)
+;
+; This subdirectory is a git "subrepo", and this file is maintained by the
+; git-subrepo command. See https://github.com/ingydotnet/git-subrepo#readme
+;
+[subrepo]
+	remote = https://github.com/vladkens/twscrape
+	branch = main
+	commit = e4dce5d31f3904481efe4b20f504f50326185147
+	parent = 0afc214ffb1ac7786233d667028b501bb249c868
+	method = merge
+	cmdver = 0.4.9

apps/twscrape/.vscode/settings.json

@@ -0,0 +1,24 @@
+{
+  "[python]": {
+    "editor.formatOnSave": true,
+    "editor.codeActionsOnSave": ["source.organizeImports"],
+    "editor.defaultFormatter": "charliermarsh.ruff"
+  },
+  "[toml]": {
+    "editor.defaultFormatter": "tamasfe.even-better-toml"
+  },
+  "[markdown]": {
+    "editor.wordWrap": "wordWrapColumn",
+    "editor.wordWrapColumn": 99
+  },  
+  "files.exclude": {
+    ".ruff_cache": true,
+    ".pytest_cache": true,
+    ".coverage": true,
+    "htmlcov": true,
+    "dist": true
+  },
+  "files.associations": {
+    "Dockerfile.*": "dockerfile"
+  }
+}

apps/twscrape/Dockerfile.py-matrix

@@ -0,0 +1,11 @@
+ARG VER=3.12
+
+FROM python:${VER}-alpine
+RUN apk add git
+
+WORKDIR /app
+COPY pyproject.toml readme.md /app/
+RUN pip install -e .[dev]
+COPY . /app
+
+CMD ["sh", "-c", "pytest tests/"]

apps/twscrape/Dockerfile.sq-matrix

@@ -0,0 +1,21 @@
+FROM python:3.10-alpine
+RUN apk add build-base
+
+ARG SQLY=2021
+ARG SQLV=3340100
+
+RUN wget https://sqlite.org/${SQLY}/sqlite-autoconf-${SQLV}.tar.gz -O sqlite.tar.gz \
+    && tar xvfz sqlite.tar.gz \
+    && cd sqlite-autoconf-${SQLV} \
+    && ./configure --prefix=/usr/local --build=aarch64-unknown-linux-gnu \
+    && make \
+    && make install \
+    && cd .. \
+    && rm -rf sqlite*
+
+WORKDIR /app
+COPY pyproject.toml readme.md /app/
+RUN pip install -e .[dev]
+COPY . /app
+
+CMD ["sh", "-c", "sqlite3 --version; python -c 'import sqlite3;print(sqlite3.sqlite_version)'; pytest tests/"]

apps/twscrape/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 vladkens
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

apps/twscrape/Makefile

@@ -0,0 +1,69 @@
+check:
+	@make lint
+	@make test
+
+install:
+	pip install -e .[dev]
+	python -m build
+
+build:
+	python -m build --sdist --wheel --outdir dist/ .
+
+lint:
+	@python -m ruff check --select I --fix .
+	@python -m ruff format .
+	@python -m ruff check .
+	@python -m pyright .
+
+test:
+	@python -m pytest -s --cov=twscrape tests/
+
+test-cov:
+	@python -m pytest -s --cov=twscrape tests/
+	@coverage html
+	@open htmlcov/index.html
+
+test-py:
+	$(eval name=twscrape_py$(v))
+	@docker -l warning build -f Dockerfile.py-matrix --build-arg VER=$(v) -t $(name) .
+	@docker run $(name)
+
+test-sq:
+	$(eval name=twscrape_sq$(v))
+	@docker -l warning build -f Dockerfile.sq-matrix --build-arg SQLY=$(y) --build-arg SQLV=$(v) -t $(name) .
+	@docker run $(name)
+
+test-matrix-py:
+	@make test-py v=3.10
+	@make test-py v=3.11
+	@make test-py v=3.12
+	@make test-py v=3.13
+
+test-matrix-sq:
+	@# https://www.sqlite.org/chronology.html
+	@make test-sq y=2018 v=3240000
+	@make test-sq y=2019 v=3270200
+	@make test-sq y=2019 v=3300100
+	@make test-sq y=2020 v=3330000
+	@make test-sq y=2021 v=3340100
+	@make test-sq y=2023 v=3430000
+	@make test-sq y=2023 v=3440000
+	@make test-sq y=2024 v=3450300
+
+update-mocks:
+	@rm -rf ./tests/mocked-data/raw_*.json
+	twscrape user_by_id --raw 2244994945 | jq > ./tests/mocked-data/raw_user_by_id.json
+	twscrape user_by_login --raw xdevelopers | jq > ./tests/mocked-data/raw_user_by_login.json
+	twscrape following --raw --limit 10  2244994945 | jq > ./tests/mocked-data/raw_following.json
+	twscrape followers --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_followers.json
+	twscrape verified_followers --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_verified_followers.json
+	twscrape subscriptions --raw --limit 10 58579942 | jq > ./tests/mocked-data/raw_subscriptions.json
+	twscrape tweet_details --raw 1649191520250245121 | jq > ./tests/mocked-data/raw_tweet_details.json
+	twscrape tweet_replies --limit 1 --raw 1649191520250245121 | jq > ./tests/mocked-data/raw_tweet_replies.json
+	twscrape retweeters --raw --limit 10 1649191520250245121 | jq > ./tests/mocked-data/raw_retweeters.json
+	twscrape user_tweets --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_user_tweets.json
+	twscrape user_tweets_and_replies --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_user_tweets_and_replies.json
+	twscrape user_media --raw --limit 10 2244994945 | jq > ./tests/mocked-data/raw_user_media.json
+	twscrape search --raw --limit 5 "elon musk lang:en" | jq > ./tests/mocked-data/raw_search.json
+	twscrape list_timeline --raw --limit 10 1494877848087187461 | jq > ./tests/mocked-data/raw_list_timeline.json
+	twscrape trends --raw sport | jq > ./tests/mocked-data/raw_trends.json

apps/twscrape/_get_gql_ops.py

@@ -0,0 +1,110 @@
+import json
+import os
+import re
+
+import httpx
+from fake_useragent import UserAgent
+
+"""
+docker run --rm -p "3128:3128/tcp" -p "1080:1080/tcp" -e "PROXY_LOGIN=user" -e "PROXY_PASSWORD=pass" tarampampam/3proxy
+docker run --rm -p "3129:3128/tcp" -p "1081:1080/tcp" tarampampam/3proxy
+"""
+
+client = httpx.Client(headers={"user-agent": UserAgent().chrome}, follow_redirects=True)
+
+with open("./twscrape/api.py") as fp:
+    ops = [x.strip() for x in fp.read().split("\n")]
+    ops = [x.split("=")[0].removeprefix("OP_").strip() for x in ops if x.startswith("OP_")]
+
+
+def script_url(k: str, v: str):
+    return f"https://abs.twimg.com/responsive-web/client-web/{k}.{v}.js"
+
+
+def get_page_text(url: str):
+    rep = client.get(url)
+    rep.raise_for_status()
+    if ">document.location =" not in rep.text:
+        return rep.text
+
+    url = rep.text.split('document.location = "')[1].split('"')[0]
+    rep = client.get(url)
+    rep.raise_for_status()
+    if 'action="https://x.com/x/migrate" method="post"' not in rep.text:
+        return rep.text
+
+    data = {}
+    for x in rep.text.split("<input")[1:]:
+        name = x.split('name="')[1].split('"')[0]
+        value = x.split('value="')[1].split('"')[0]
+        data[name] = value
+
+    rep = client.post("https://x.com/x/migrate", json=data)
+    rep.raise_for_status()
+
+    return rep.text
+
+
+def get_scripts():
+    cache_dir = "/tmp/twscrape-ops"
+    os.makedirs(cache_dir, exist_ok=True)
+
+    text = get_page_text("https://x.com/elonmusk")
+    urls = []
+
+    scripts = text.split('e=>e+"."+')[1].split('[e]+"a.js"')[0]
+    try:
+        for k, v in json.loads(scripts).items():
+            urls.append(script_url(k, f"{v}a"))
+    except json.decoder.JSONDecodeError as e:
+        print(scripts)
+        print(e)
+        exit(1)
+
+    v = text.split("/client-web/main.")[1].split(".")[0]
+    urls.append(script_url("main", v))
+
+    urls = [
+        x
+        for x in urls
+        if "/i18n/" not in x and "/icons/" not in x and "react-syntax-highlighter" not in x
+    ]
+
+    scripts = []
+    for i, x in enumerate(urls, 1):
+        cache_path = os.path.join(cache_dir, x.split("/")[-1].split("?")[0])
+        if os.path.exists(cache_path):
+            with open(cache_path) as fp:
+                scripts.append(fp.read())
+            continue
+
+        print(f"({i:3d} / {len(urls):3d}) {x}")
+        rep = client.get(x)
+        rep.raise_for_status()
+
+        with open(cache_path, "w") as fp:
+            fp.write(rep.text)
+        scripts.append(rep.text)
+
+    return scripts
+
+
+all_pairs = {}
+for txt in get_scripts():
+    pairs = re.findall(r'queryId:"(.+?)".+?operationName:"(.+?)"', txt)
+    pairs = {op_name: op_id for op_id, op_name in pairs}
+
+    for k, v in pairs.items():
+        if k in all_pairs and v != all_pairs[k]:
+            print(f"DIFF: {k} = {v} != {all_pairs[k]}")
+
+        all_pairs[k] = v
+
+
+for k, v in all_pairs.items():
+    print(f'OP_{k} = "{v}/{k}"')
+
+print("-" * 40)
+
+for x in ops:
+    print(f'OP_{x} = "{all_pairs.get(x, "???")}/{x}"')

apps/twscrape/examples/download_media.py

@@ -0,0 +1,43 @@
+import asyncio
+import os
+
+import httpx
+
+from twscrape import API
+
+
+async def download_file(client: httpx.AsyncClient, url: str, outdir: str):
+    filename = url.split("/")[-1].split("?")[0]
+    outpath = os.path.join(outdir, filename)
+
+    async with client.stream("GET", url) as resp:
+        with open(outpath, "wb") as f:
+            async for chunk in resp.aiter_bytes():
+                f.write(chunk)
+
+
+async def load_user_media(api: API, user_id: int, outdir: str):
+    os.makedirs(outdir, exist_ok=True)
+    all_photos = []
+    all_videos = []
+
+    async for doc in api.user_media(user_id):
+        all_photos.extend([x.url for x in doc.media.photos])
+        for video in doc.media.videos:
+            variant = sorted(video.variants, key=lambda x: x.bitrate)[-1]
+            all_videos.append(variant.url)
+
+    async with httpx.AsyncClient() as client:
+        await asyncio.gather(
+            *[download_file(client, url, outdir) for url in all_photos],
+            *[download_file(client, url, outdir) for url in all_videos],
+        )
+
+
+async def main():
+    api = API()
+    await load_user_media(api, 2244994945, "data_media")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

apps/twscrape/examples/parallel_search.py

@@ -0,0 +1,37 @@
+"""
+This example shows how to use twscrape to complete some queries in parallel.
+To limit the number of concurrent requests, see examples/parallel_search_with_limit.py
+"""
+
+import asyncio
+
+import twscrape
+
+
+async def worker(api: twscrape.API, q: str):
+    tweets = []
+    try:
+        async for doc in api.search(q):
+            tweets.append(doc)
+    except Exception as e:
+        print(e)
+    finally:
+        return tweets
+
+
+async def main():
+    api = twscrape.API()
+    # add accounts here or before from cli (see README.md for examples)
+    await api.pool.add_account("u1", "p1", "eu1", "ep1")
+    await api.pool.login_all()
+
+    queries = ["elon musk", "tesla", "spacex", "neuralink", "boring company"]
+    results = await asyncio.gather(*(worker(api, q) for q in queries))
+
+    combined = dict(zip(queries, results))
+    for k, v in combined.items():
+        print(k, len(v))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

apps/twscrape/examples/parallel_search_with_limit.py

@@ -0,0 +1,46 @@
+"""
+This example shows how to use twscrape in parallel with concurrency limit.
+"""
+
+import asyncio
+import time
+
+import twscrape
+
+
+async def worker(queue: asyncio.Queue, api: twscrape.API):
+    while True:
+        query = await queue.get()
+
+        try:
+            tweets = await twscrape.gather(api.search(query))
+            print(f"{query} - {len(tweets)} - {int(time.time())}")
+            # do something with tweets here, eg same to file, etc
+        except Exception as e:
+            print(f"Error on {query} - {type(e)}")
+        finally:
+            queue.task_done()
+
+
+async def main():
+    api = twscrape.API()
+    # add accounts here or before from cli (see README.md for examples)
+    await api.pool.add_account("u1", "p1", "eu1", "ep1")
+    await api.pool.login_all()
+
+    queries = ["elon musk", "tesla", "spacex", "neuralink", "boring company"]
+
+    queue = asyncio.Queue()
+
+    workers_count = 2  # limit concurrency here 2 concurrent requests at time
+    workers = [asyncio.create_task(worker(queue, api)) for _ in range(workers_count)]
+    for q in queries:
+        queue.put_nowait(q)
+
+    await queue.join()
+    for worker_task in workers:
+        worker_task.cancel()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

apps/twscrape/pyproject.toml

@@ -0,0 +1,66 @@
+[build-system]
+requires = ["hatchling>=1.9.1"]
+build-backend = "hatchling.build"
+
+[project]
+name = "twscrape"
+version = "0.16.0"
+authors = [{ name = "vladkens", email = "v.pronsky@gmail.com" }]
+description = "Twitter GraphQL and Search API implementation with SNScrape data models"
+readme = "readme.md"
+requires-python = ">=3.10"
+keywords = ["twitter", "scrape", "scrapper", "api", "snscrape"]
+license = { text = "MIT" }
+classifiers = [
+  'Development Status :: 4 - Beta',
+  'License :: OSI Approved :: MIT License',
+  'Programming Language :: Python :: 3.10',
+  'Programming Language :: Python :: 3.11',
+  'Programming Language :: Python :: 3.12',
+  'Programming Language :: Python :: 3.13',
+]
+dependencies = [
+  "aiosqlite>=0.17.0",
+  "fake-useragent>=1.4.0",
+  "httpx>=0.26.0",
+  "loguru>=0.7.0",
+  "pyotp>=2.9.0",
+]
+
+[project.optional-dependencies]
+dev = [
+  "build>=1.2.2",
+  "pyright>=1.1.369",
+  "pytest-asyncio>=0.23.3",
+  "pytest-cov>=4.1.0",
+  "pytest-httpx>=0.28.0",
+  "pytest>=7.4.4",
+  "ruff>=0.1.11",
+]
+
+[project.urls]
+repository = "https://github.com/vladkens/twscrape"
+
+[project.scripts]
+twscrape = "twscrape.cli:run"
+
+[tool.setuptools]
+packages = ['twscrape']
+
+[tool.hatch.build.targets.wheel]
+packages = ["twscrape"]
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.pytest.ini_options]
+pythonpath = ["."]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "session"
+filterwarnings = ["ignore::DeprecationWarning"]
+
+[tool.ruff]
+line-length = 99
+
+[tool.ruff.lint]
+ignore = ["E501"]

apps/twscrape/readme.md

@@ -0,0 +1,344 @@
+# twscrape
+
+<div align="center">
+
+[<img src="https://badges.ws/pypi/v/twscrape" alt="version" />](https://pypi.org/project/twscrape)
+[<img src="https://badges.ws/pypi/python/twscrape" alt="py versions" />](https://pypi.org/project/twscrape)
+[<img src="https://badges.ws/pypi/dm/twscrape" alt="downloads" />](https://pypi.org/project/twscrape)
+[<img src="https://badges.ws/github/license/vladkens/twscrape" alt="license" />](https://github.com/vladkens/twscrape/blob/main/LICENSE)
+[<img src="https://badges.ws/badge/-/buy%20me%20a%20coffee/ff813f?icon=buymeacoffee&label" alt="donate" />](https://buymeacoffee.com/vladkens)
+
+</div>
+
+Twitter GraphQL API implementation with [SNScrape](https://github.com/JustAnotherArchivist/snscrape) data models.
+
+<div align="center">
+  <img src=".github/example.png" alt="example of cli usage" height="400px">
+</div>
+
+## Install
+
+```bash
+pip install twscrape
+```
+Or development version:
+```bash
+pip install git+https://github.com/vladkens/twscrape.git
+```
+
+## Features
+- Support both Search & GraphQL Twitter API
+- Async/Await functions (can run multiple scrapers in parallel at the same time)
+- Login flow (with receiving verification code from email)
+- Saving/restoring account sessions
+- Raw Twitter API responses & SNScrape models
+- Automatic account switching to smooth Twitter API rate limits
+
+## Usage
+
+This project requires authorized X/Twitter accounts to work with the API. You have two options:
+
+1. **Create Your Own Account**: While you can register a new account on X/Twitter yourself, it's can be difficult due to strict verification processes and high ban rates.
+
+2. **Use Ready Accounts**: For immediate access, you can get ready-to-use accounts with cookies from [our recommended provider](https://kutt.it/ueeM5f). Cookie-based accounts typically have fewer login issues.
+
+For optimal performance and to avoid IP-based restrictions, we also recommend using proxies from [our provider](https://kutt.it/eb3rXk).
+
+**Disclaimer**: While X/Twitter's Terms of Service discourage using multiple accounts, this is a common practice for data collection and research purposes. Use responsibly and at your own discretion.
+
+```python
+import asyncio
+from twscrape import API, gather
+from twscrape.logger import set_log_level
+
+async def main():
+    api = API()  # or API("path-to.db") – default is `accounts.db`
+
+    # ADD ACCOUNTS (for CLI usage see next readme section)
+
+    # Option 1. Adding account with cookies (more stable)
+    cookies = "abc=12; ct0=xyz"  # or '{"abc": "12", "ct0": "xyz"}'
+    await api.pool.add_account("user3", "pass3", "u3@mail.com", "mail_pass3", cookies=cookies)
+
+    # Option2. Adding account with login / password (less stable)
+    # email login / password required to receive the verification code via IMAP protocol
+    # (not all email providers are supported, e.g. ProtonMail)
+    await api.pool.add_account("user1", "pass1", "u1@example.com", "mail_pass1")
+    await api.pool.add_account("user2", "pass2", "u2@example.com", "mail_pass2")
+    await api.pool.login_all() # try to login to receive account cookies
+
+    # API USAGE
+
+    # search (latest tab)
+    await gather(api.search("elon musk", limit=20))  # list[Tweet]
+    # change search tab (product), can be: Top, Latest (default), Media
+    await gather(api.search("elon musk", limit=20, kv={"product": "Top"}))
+
+    # tweet info
+    tweet_id = 20
+    await api.tweet_details(tweet_id)  # Tweet
+    await gather(api.retweeters(tweet_id, limit=20))  # list[User]
+
+    # Note: this method have small pagination from X side, like 5 tweets per query
+    await gather(api.tweet_replies(tweet_id, limit=20))  # list[Tweet]
+
+    # get user by login
+    user_login = "xdevelopers"
+    await api.user_by_login(user_login)  # User
+
+    # user info
+    user_id = 2244994945
+    await api.user_by_id(user_id)  # User
+    await gather(api.following(user_id, limit=20))  # list[User]
+    await gather(api.followers(user_id, limit=20))  # list[User]
+    await gather(api.verified_followers(user_id, limit=20))  # list[User]
+    await gather(api.subscriptions(user_id, limit=20))  # list[User]
+    await gather(api.user_tweets(user_id, limit=20))  # list[Tweet]
+    await gather(api.user_tweets_and_replies(user_id, limit=20))  # list[Tweet]
+    await gather(api.user_media(user_id, limit=20))  # list[Tweet]
+
+    # list info
+    await gather(api.list_timeline(list_id=123456789))
+
+    # trends
+    await gather(api.trends("news"))  # list[Trend]
+    await gather(api.trends("sport"))  # list[Trend]
+    await gather(api.trends("VGltZWxpbmU6DAC2CwABAAAACHRyZW5kaW5nAAA"))  # list[Trend]
+
+    # NOTE 1: gather is a helper function to receive all data as list, FOR can be used as well:
+    async for tweet in api.search("elon musk"):
+        print(tweet.id, tweet.user.username, tweet.rawContent)  # tweet is `Tweet` object
+
+    # NOTE 2: all methods have `raw` version (returns `httpx.Response` object):
+    async for rep in api.search_raw("elon musk"):
+        print(rep.status_code, rep.json())  # rep is `httpx.Response` object
+
+    # change log level, default info
+    set_log_level("DEBUG")
+
+    # Tweet & User model can be converted to regular dict or json, e.g.:
+    doc = await api.user_by_id(user_id)  # User
+    doc.dict()  # -> python dict
+    doc.json()  # -> json string
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+### Stoping iteration with break
+
+In order to correctly release an account in case of `break` in loop, a special syntax must be used. Otherwise, Python's events loop will release lock on the account sometime in the future. See explanation [here](https://github.com/vladkens/twscrape/issues/27#issuecomment-1623395424).
+
+```python
+from contextlib import aclosing
+
+async with aclosing(api.search("elon musk")) as gen:
+    async for tweet in gen:
+        if tweet.id < 200:
+            break
+```
+
+## CLI
+
+### Get help on CLI commands
+
+```sh
+# show all commands
+twscrape
+
+# help on specific comand
+twscrape search --help
+```
+
+### Add accounts
+
+To add accounts use `add_accounts` command. Command syntax is:
+```sh
+twscrape add_accounts <file_path> <line_format>
+```
+
+Where:
+`<line_format>` is format of line if accounts file splited by delimeter. Possible tokens:
+- `username` – required
+- `password` – required
+- `email` – required
+- `email_password` – to receive email code (you can use `--manual` mode to get code)
+- `cookies` – can be any parsable format (string, json, base64 string, etc)
+- `_` – skip column from parse
+
+Tokens should be splited by delimeter, usually "`:`" used.
+
+Example:
+
+I have account files named `order-12345.txt` with format:
+```text
+username:password:email:email password:user_agent:cookies
+```
+
+Command to add accounts will be (user_agent column skiped with `_`):
+```sh
+twscrape add_accounts ./order-12345.txt username:password:email:email_password:_:cookies
+```
+
+### Login accounts
+
+_Note:_ If you added accounts with cookies, login not required.
+
+Run:
+
+```sh
+twscrape login_accounts
+```
+
+`twscrape` will start login flow for each new account. If X will ask to verify email and you provided `email_password` in `add_account`, then `twscrape` will try to receive verification code by IMAP protocol. After success login account cookies will be saved to db file for future use.
+
+#### Manual email verification
+
+In case your email provider not support IMAP protocol (ProtonMail, Tutanota, etc) or IMAP is disabled in settings, you can enter email verification code manually. To do this run login command with `--manual` flag.
+
+Example:
+
+```sh
+twscrape login_accounts --manual
+twscrape relogin user1 user2 --manual
+twscrape relogin_failed --manual
+```
+
+### Get list of accounts and their statuses
+
+```sh
+twscrape accounts
+
+# Output:
+# username  logged_in  active  last_used            total_req  error_msg
+# user1     True       True    2023-05-20 03:20:40  100        None
+# user2     True       True    2023-05-20 03:25:45  120        None
+# user3     False      False   None                 120        Login error
+```
+
+### Re-login accounts
+
+It is possible to re-login specific accounts:
+
+```sh
+twscrape relogin user1 user2
+```
+
+Or retry login for all failed logins:
+
+```sh
+twscrape relogin_failed
+```
+
+### Use different accounts file
+
+Useful if using a different set of accounts for different actions
+
+```
+twscrape --db test-accounts.db <command>
+```
+
+### Search commands
+
+```sh
+twscrape search "QUERY" --limit=20
+twscrape tweet_details TWEET_ID
+twscrape tweet_replies TWEET_ID --limit=20
+twscrape retweeters TWEET_ID --limit=20
+twscrape user_by_id USER_ID
+twscrape user_by_login USERNAME
+twscrape user_media USER_ID --limit=20
+twscrape following USER_ID --limit=20
+twscrape followers USER_ID --limit=20
+twscrape verified_followers USER_ID --limit=20
+twscrape subscriptions USER_ID --limit=20
+twscrape user_tweets USER_ID --limit=20
+twscrape user_tweets_and_replies USER_ID --limit=20
+twscrape trends sport
+```
+
+The default output is in the console (stdout), one document per line. So it can be redirected to the file.
+
+```sh
+twscrape search "elon mask lang:es" --limit=20 > data.txt
+```
+
+By default, parsed data is returned. The original tweet responses can be retrieved with `--raw` flag.
+
+```sh
+twscrape search "elon mask lang:es" --limit=20 --raw
+```
+
+### About `limit` param
+
+X API works through pagination, each API method can have different defaults for per page parameter (and this parameter can't be changed by caller). So `limit` param in `twscrape` is the desired number of objects (tweets or users, depending on the method). `twscrape` tries to return NO LESS objects than requested. If the X API returns less or more objects, `twscrape` will return whatever X gives.
+
+## Proxy
+
+There are few options to use proxies.
+
+1. You can add proxy per account
+
+```py
+proxy = "http://login:pass@example.com:8080"
+await api.pool.add_account("user4", "pass4", "u4@mail.com", "mail_pass4", proxy=proxy)
+```
+
+2. You can use global proxy for all accounts
+
+```py
+proxy = "http://login:pass@example.com:8080"
+api = API(proxy=proxy)
+doc = await api.user_by_login("elonmusk")
+```
+
+3. Use can set proxy with environemt variable `TWS_RPOXY`:
+
+```sh
+TWS_PROXY=socks5://user:pass@127.0.0.1:1080 twscrape user_by_login elonmusk
+```
+
+4. You can change proxy any time like:
+
+```py
+api.proxy = "socks5://user:pass@127.0.0.1:1080"
+doc = await api.user_by_login("elonmusk")  # new proxy will be used
+api.proxy = None
+doc = await api.user_by_login("elonmusk")  # no proxy used
+```
+
+5. Proxy priorities
+
+- `api.proxy` have top priority
+- `env.proxy` will be used if `api.proxy` is None
+- `acc.proxy` have lowest priotity
+
+So if you want to use proxy PER ACCOUNT, do NOT override proxy with env variable or by passing proxy param to API.
+
+_Note:_ If proxy not working, exception will be raised from API class.
+
+## Environment Variables
+
+- `TWS_PROXY` - global proxy for all accounts (e.g. `socks5://user:pass@127.0.0.1:1080`)
+- `TWS_WAIT_EMAIL_CODE` - timeout for email verification code during login (default: `30`, in seconds)
+- `TWS_RAISE_WHEN_NO_ACCOUNT` - raise `NoAccountError` exception when no available accounts, instead of waiting (default: `false`, values: `false`/`0`/`true`/`1`)
+
+## Limitations
+
+X/Twitter regularly [updates](https://x.com/elonmusk/status/1675187969420828672) their rate limits. Current basic behavior:
+- Request limits reset every 15 minutes for each endpoint individually
+- Each account has separate limits for different operations (search, profile views, etc.)
+
+API data limitations:
+- `user_tweets` & `user_tweets_and_replies` - can return ~3200 tweets maximum
+- Rate limits may vary based on account age and status
+
+## Articles
+- [How to still scrape millions of tweets in 2023](https://medium.com/@vladkens/how-to-still-scrape-millions-of-tweets-in-2023-using-twscrape-97f5d3881434)
+- [_(Add Article)_](https://github.com/vladkens/twscrape/edit/main/readme.md)
+
+## See also
+- [twitter-advanced-search](https://github.com/igorbrigadir/twitter-advanced-search) – guide on search filters
+- [twitter-api-client](https://github.com/trevorhobenshield/twitter-api-client) – Implementation of Twitter's v1, v2, and GraphQL APIs
+- [snscrape](https://github.com/JustAnotherArchivist/snscrape) – is a scraper for social networking services (SNS)
+- [twint](https://github.com/twintproject/twint) – Twitter Intelligence Tool

apps/twscrape/tests/conftest.py

@@ -0,0 +1,35 @@
+import pytest
+
+from twscrape.accounts_pool import AccountsPool
+from twscrape.api import API
+from twscrape.logger import set_log_level
+from twscrape.queue_client import QueueClient
+
+set_log_level("ERROR")
+
+
+@pytest.fixture
+def pool_mock(tmp_path):
+    db_path = tmp_path / "test.db"
+    yield AccountsPool(db_path)
+
+
+@pytest.fixture
+async def client_fixture(pool_mock: AccountsPool):
+    pool_mock._order_by = "username"
+
+    for x in range(1, 3):
+        await pool_mock.add_account(f"user{x}", f"pass{x}", f"email{x}", f"email_pass{x}")
+        await pool_mock.set_active(f"user{x}", True)
+
+    client = QueueClient(pool_mock, "SearchTimeline")
+    yield pool_mock, client
+
+
+@pytest.fixture
+async def api_mock(pool_mock: AccountsPool):
+    await pool_mock.add_account("user1", "pass1", "email1", "email_pass1")
+    await pool_mock.set_active("user1", True)
+
+    api = API(pool_mock)
+    yield api

apps/twscrape/tests/mocked-data/_issue_28_1.json

@@ -0,0 +1,847 @@
+{
+  "data": {
+    "threaded_conversation_with_injections_v2": {
+      "instructions": [
+        {
+          "type": "TimelineAddEntries",
+          "entries": [
+            {
+              "entryId": "tweet-1658409412799737856",
+              "sortIndex": "7564962624055037951",
+              "content": {
+                "entryType": "TimelineTimelineItem",
+                "__typename": "TimelineTimelineItem",
+                "itemContent": {
+                  "itemType": "TimelineTweet",
+                  "__typename": "TimelineTweet",
+                  "tweet_results": {
+                    "result": {
+                      "__typename": "Tweet",
+                      "rest_id": "1658409412799737856",
+                      "has_birdwatch_notes": false,
+                      "core": {
+                        "user_results": {
+                          "result": {
+                            "__typename": "User",
+                            "id": "VXNlcjoxMjk2ODAxOTcwNzc1MzY3Njgw",
+                            "rest_id": "1296801970775367680",
+                            "affiliates_highlighted_label": {},
+                            "has_graduated_access": true,
+                            "is_blue_verified": false,
+                            "legacy": {
+                              "can_dm": true,
+                              "can_media_tag": true,
+                              "created_at": "Fri Aug 21 13:31:26 +0000 2020",
+                              "default_profile": true,
+                              "default_profile_image": false,
+                              "description": "Por una unidad de izquierdas real .Bienvenidos a la #RedRoja. Sigue a quien te siga . Red de seguridad: @RedRoj4  @REDROJA@mastodon.world",
+                              "entities": {
+                                "description": {
+                                  "urls": []
+                                },
+                                "url": {
+                                  "urls": [
+                                    {
+                                      "display_url": "instagram.com/redroja4?igshi…",
+                                      "expanded_url": "https://instagram.com/redroja4?igshid=1d8cfo0z5wf2t",
+                                      "url": "https://t.co/yLqbCaFwck",
+                                      "indices": [
+                                        0,
+                                        23
+                                      ]
+                                    }
+                                  ]
+                                }
+                              },
+                              "fast_followers_count": 0,
+                              "favourites_count": 60839,
+                              "followers_count": 43398,
+                              "friends_count": 31537,
+                              "has_custom_timelines": true,
+                              "is_translator": false,
+                              "listed_count": 100,
+                              "location": "Izquierda ",
+                              "media_count": 1469,
+                              "name": "RED ROJA 🔻",
+                              "normal_followers_count": 43398,
+                              "pinned_tweet_ids_str": [
+                                "1592141363050860546"
+                              ],
+                              "possibly_sensitive": false,
+                              "profile_banner_url": "https://pbs.twimg.com/profile_banners/1296801970775367680/1618465956",
+                              "profile_image_url_https": "https://pbs.twimg.com/profile_images/1541282511816376322/4UQWhvwj_normal.jpg",
+                              "profile_interstitial_type": "",
+                              "screen_name": "REDROJA4",
+                              "statuses_count": 30422,
+                              "translator_type": "none",
+                              "url": "https://t.co/yLqbCaFwck",
+                              "verified": false,
+                              "want_retweets": false,
+                              "withheld_in_countries": []
+                            },
+                            "business_account": {}
+                          }
+                        }
+                      },
+                      "edit_control": {
+                        "edit_tweet_ids": [
+                          "1658409412799737856"
+                        ],
+                        "editable_until_msecs": "1684232390425",
+                        "is_edit_eligible": false,
+                        "edits_remaining": "5"
+                      },
+                      "edit_perspective": {
+                        "favorited": false,
+                        "retweeted": false
+                      },
+                      "is_translatable": true,
+                      "views": {
+                        "state": "Enabled"
+                      },
+                      "source": "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>",
+                      "legacy": {
+                        "bookmark_count": 0,
+                        "bookmarked": false,
+                        "created_at": "Tue May 16 09:49:50 +0000 2023",
+                        "conversation_id_str": "1658409412799737856",
+                        "display_text_range": [
+                          0,
+                          140
+                        ],
+                        "entities": {
+                          "user_mentions": [
+                            {
+                              "id_str": "2949065723",
+                              "name": "SisuMisu",
+                              "screen_name": "SisuMisu3",
+                              "indices": [
+                                3,
+                                13
+                              ]
+                            },
+                            {
+                              "id_str": "1860300192",
+                              "name": "Alejandra Jacinto",
+                              "screen_name": "AleJacintoUrang",
+                              "indices": [
+                                96,
+                                112
+                              ]
+                            }
+                          ],
+                          "urls": [],
+                          "hashtags": [
+                            {
+                              "indices": [
+                                131,
+                                136
+                              ],
+                              "text": "Sisu"
+                            }
+                          ],
+                          "symbols": []
+                        },
+                        "favorite_count": 0,
+                        "favorited": false,
+                        "full_text": "RT @SisuMisu3: Esta noche el circo está asegurado 😂😂😂\nAyuso se presenta al único debate.\nVamos \n@AleJacintoUrang \nEstamos contigo.\n#Sisu ht…",
+                        "is_quote_status": false,
+                        "lang": "es",
+                        "quote_count": 0,
+                        "reply_count": 0,
+                        "retweet_count": 42,
+                        "retweeted": false,
+                        "user_id_str": "1296801970775367680",
+                        "id_str": "1658409412799737856",
+                        "retweeted_status_result": {
+                          "result": {
+                            "__typename": "Tweet",
+                            "rest_id": "1658407192859492355",
+                            "has_birdwatch_notes": false,
+                            "core": {
+                              "user_results": {
+                                "result": {
+                                  "__typename": "User",
+                                  "id": "VXNlcjoyOTQ5MDY1NzIz",
+                                  "rest_id": "2949065723",
+                                  "affiliates_highlighted_label": {},
+                                  "has_graduated_access": true,
+                                  "is_blue_verified": false,
+                                  "legacy": {
+                                    "can_dm": true,
+                                    "can_media_tag": false,
+                                    "created_at": "Mon Dec 29 11:54:26 +0000 2014",
+                                    "default_profile": true,
+                                    "default_profile_image": false,
+                                    "description": "MI LUCHA ESTÁ EN TODAS PARTES. POR LOS DERECHOS DE LOS ANIMALES. POR UN MUNDO MEJOR.ACTIVISTA",
+                                    "entities": {
+                                      "description": {
+                                        "urls": []
+                                      }
+                                    },
+                                    "fast_followers_count": 0,
+                                    "favourites_count": 108021,
+                                    "followers_count": 15454,
+                                    "friends_count": 13032,
+                                    "has_custom_timelines": true,
+                                    "is_translator": false,
+                                    "listed_count": 34,
+                                    "location": "En donde me den amor ❤️😊🥰",
+                                    "media_count": 12985,
+                                    "name": "SisuMisu",
+                                    "normal_followers_count": 15454,
+                                    "pinned_tweet_ids_str": [
+                                      "1087440689368043520"
+                                    ],
+                                    "possibly_sensitive": false,
+                                    "profile_banner_url": "https://pbs.twimg.com/profile_banners/2949065723/1513372062",
+                                    "profile_image_url_https": "https://pbs.twimg.com/profile_images/1528038627317538817/wvoi8QLf_normal.jpg",
+                                    "profile_interstitial_type": "",
+                                    "screen_name": "SisuMisu3",
+                                    "statuses_count": 164333,
+                                    "translator_type": "none",
+                                    "verified": false,
+                                    "want_retweets": false,
+                                    "withheld_in_countries": []
+                                  },
+                                  "professional": {
+                                    "rest_id": "1490288411835523072",
+                                    "professional_type": "Creator",
+                                    "category": [
+                                      {
+                                        "id": 934,
+                                        "name": "Social Media Influencer",
+                                        "icon_name": "IconBriefcaseStroke"
+                                      }
+                                    ]
+                                  },
+                                  "business_account": {}
+                                }
+                              }
+                            },
+                            "edit_control": {
+                              "edit_tweet_ids": [
+                                "1658407192859492355"
+                              ],
+                              "editable_until_msecs": "1684231861000",
+                              "is_edit_eligible": true,
+                              "edits_remaining": "5"
+                            },
+                            "edit_perspective": {
+                              "favorited": false,
+                              "retweeted": false
+                            },
+                            "is_translatable": true,
+                            "views": {
+                              "count": "1110",
+                              "state": "EnabledWithCount"
+                            },
+                            "source": "<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>",
+                            "legacy": {
+                              "bookmark_count": 0,
+                              "bookmarked": false,
+                              "created_at": "Tue May 16 09:41:01 +0000 2023",
+                              "conversation_id_str": "1658407192859492355",
+                              "display_text_range": [
+                                0,
+                                121
+                              ],
+                              "entities": {
+                                "media": [
+                                  {
+                                    "display_url": "pic.twitter.com/apswi6Wwiw",
+                                    "expanded_url": "https://twitter.com/SisuMisu3/status/1658407192859492355/photo/1",
+                                    "id_str": "1658407190594453505",
+                                    "indices": [
+                                      122,
+                                      145
+                                    ],
+                                    "media_url_https": "https://pbs.twimg.com/media/FwPYjaBWAAETpRI.jpg",
+                                    "type": "photo",
+                                    "url": "https://t.co/apswi6Wwiw",
+                                    "features": {
+                                      "large": {
+                                        "faces": [
+                                          {
+                                            "x": 273,
+                                            "y": 253,
+                                            "h": 351,
+                                            "w": 351
+                                          }
+                                        ]
+                                      },
+                                      "medium": {
+                                        "faces": [
+                                          {
+                                            "x": 255,
+                                            "y": 237,
+                                            "h": 329,
+                                            "w": 329
+                                          }
+                                        ]
+                                      },
+                                      "small": {
+                                        "faces": [
+                                          {
+                                            "x": 145,
+                                            "y": 134,
+                                            "h": 186,
+                                            "w": 186
+                                          }
+                                        ]
+                                      },
+                                      "orig": {
+                                        "faces": [
+                                          {
+                                            "x": 273,
+                                            "y": 253,
+                                            "h": 351,
+                                            "w": 351
+                                          }
+                                        ]
+                                      }
+                                    },
+                                    "sizes": {
+                                      "large": {
+                                        "h": 1280,
+                                        "w": 1024,
+                                        "resize": "fit"
+                                      },
+                                      "medium": {
+                                        "h": 1200,
+                                        "w": 960,
+                                        "resize": "fit"
+                                      },
+                                      "small": {
+                                        "h": 680,
+                                        "w": 544,
+                                        "resize": "fit"
+                                      },
+                                      "thumb": {
+                                        "h": 150,
+                                        "w": 150,
+                                        "resize": "crop"
+                                      }
+                                    },
+                                    "original_info": {
+                                      "height": 1280,
+                                      "width": 1024,
+                                      "focus_rects": [
+                                        {
+                                          "x": 0,
+                                          "y": 642,
+                                          "w": 1024,
+                                          "h": 573
+                                        },
+                                        {
+                                          "x": 0,
+                                          "y": 256,
+                                          "w": 1024,
+                                          "h": 1024
+                                        },
+                                        {
+                                          "x": 0,
+                                          "y": 113,
+                                          "w": 1024,
+                                          "h": 1167
+                                        },
+                                        {
+                                          "x": 32,
+                                          "y": 0,
+                                          "w": 640,
+                                          "h": 1280
+                                        },
+                                        {
+                                          "x": 0,
+                                          "y": 0,
+                                          "w": 1024,
+                                          "h": 1280
+                                        }
+                                      ]
+                                    }
+                                  }
+                                ],
+                                "user_mentions": [
+                                  {
+                                    "id_str": "1860300192",
+                                    "name": "Alejandra Jacinto",
+                                    "screen_name": "AleJacintoUrang",
+                                    "indices": [
+                                      81,
+                                      97
+                                    ]
+                                  }
+                                ],
+                                "urls": [],
+                                "hashtags": [
+                                  {
+                                    "indices": [
+                                      116,
+                                      121
+                                    ],
+                                    "text": "Sisu"
+                                  }
+                                ],
+                                "symbols": []
+                              },
+                              "extended_entities": {
+                                "media": [
+                                  {
+                                    "display_url": "pic.twitter.com/apswi6Wwiw",
+                                    "expanded_url": "https://twitter.com/SisuMisu3/status/1658407192859492355/photo/1",
+                                    "id_str": "1658407190594453505",
+                                    "indices": [
+                                      122,
+                                      145
+                                    ],
+                                    "media_key": "3_1658407190594453505",
+                                    "media_url_https": "https://pbs.twimg.com/media/FwPYjaBWAAETpRI.jpg",
+                                    "type": "photo",
+                                    "url": "https://t.co/apswi6Wwiw",
+                                    "ext_media_availability": {
+                                      "status": "Available"
+                                    },
+                                    "features": {
+                                      "large": {
+                                        "faces": [
+                                          {
+                                            "x": 273,
+                                            "y": 253,
+                                            "h": 351,
+                                            "w": 351
+                                          }
+                                        ]
+                                      },
+                                      "medium": {
+                                        "faces": [
+                                          {
+                                            "x": 255,
+                                            "y": 237,
+                                            "h": 329,
+                                            "w": 329
+                                          }
+                                        ]
+                                      },
+                                      "small": {
+                                        "faces": [
+                                          {
+                                            "x": 145,
+                                            "y": 134,
+                                            "h": 186,
+                                            "w": 186
+                                          }
+                                        ]
+                                      },
+                                      "orig": {
+                                        "faces": [
+                                          {
+                                            "x": 273,
+                                            "y": 253,
+                                            "h": 351,
+                                            "w": 351
+                                          }
+                                        ]
+                                      }
+                                    },
+                                    "sizes": {
+                                      "large": {
+                                        "h": 1280,
+                                        "w": 1024,
+                                        "resize": "fit"
+                                      },
+                                      "medium": {
+                                        "h": 1200,
+                                        "w": 960,
+                                        "resize": "fit"
+                                      },
+                                      "small": {
+                                        "h": 680,
+                                        "w": 544,
+                                        "resize": "fit"
+                                      },
+                                      "thumb": {
+                                        "h": 150,
+                                        "w": 150,
+                                        "resize": "crop"
+                                      }
+                                    },
+                                    "original_info": {
+                                      "height": 1280,
+                                      "width": 1024,
+                                      "focus_rects": [
+                                        {
+                                          "x": 0,
+                                          "y": 642,
+                                          "w": 1024,
+                                          "h": 573
+                                        },
+                                        {
+                                          "x": 0,
+                                          "y": 256,
+                                          "w": 1024,
+                                          "h": 1024
+                                        },
+                                        {
+                                          "x": 0,
+                                          "y": 113,
+                                          "w": 1024,
+                                          "h": 1167
+                                        },
+                                        {
+                                          "x": 32,
+                                          "y": 0,
+                                          "w": 640,
+                                          "h": 1280
+                                        },
+                                        {
+                                          "x": 0,
+                                          "y": 0,
+                                          "w": 1024,
+                                          "h": 1280
+                                        }
+                                      ]
+                                    }
+                                  }
+                                ]
+                              },
+                              "favorite_count": 60,
+                              "favorited": false,
+                              "full_text": "Esta noche el circo está asegurado 😂😂😂\nAyuso se presenta al único debate.\nVamos \n@AleJacintoUrang \nEstamos contigo.\n#Sisu https://t.co/apswi6Wwiw",
+                              "is_quote_status": false,
+                              "lang": "es",
+                              "possibly_sensitive": false,
+                              "possibly_sensitive_editable": true,
+                              "quote_count": 2,
+                              "reply_count": 5,
+                              "retweet_count": 42,
+                              "retweeted": false,
+                              "user_id_str": "2949065723",
+                              "id_str": "1658407192859492355"
+                            }
+                          }
+                        }
+                      },
+                      "quick_promote_eligibility": {
+                        "eligibility": "IneligibleNotProfessional"
+                      }
+                    }
+                  },
+                  "tweetDisplayType": "Tweet",
+                  "hasModeratedReplies": false
+                }
+              }
+            },
+            {
+              "entryId": "tweetdetailrelatedtweets-1677037919880507392",
+              "sortIndex": "1",
+              "content": {
+                "entryType": "TimelineTimelineModule",
+                "__typename": "TimelineTimelineModule",
+                "items": [
+                  {
+                    "entryId": "tweetdetailrelatedtweets-1677037919880507392-tweet-1677037919880507392",
+                    "item": {
+                      "itemContent": {
+                        "itemType": "TimelineTweet",
+                        "__typename": "TimelineTweet",
+                        "tweet_results": {
+                          "result": {
+                            "__typename": "Tweet",
+                            "rest_id": "1677037919880507392",
+                            "has_birdwatch_notes": false,
+                            "core": {
+                              "user_results": {
+                                "result": {
+                                  "__typename": "User",
+                                  "id": "VXNlcjoyOTQ5MDY1NzIz",
+                                  "rest_id": "2949065723",
+                                  "affiliates_highlighted_label": {},
+                                  "has_graduated_access": true,
+                                  "is_blue_verified": false,
+                                  "legacy": {
+                                    "can_dm": true,
+                                    "can_media_tag": false,
+                                    "created_at": "Mon Dec 29 11:54:26 +0000 2014",
+                                    "default_profile": true,
+                                    "default_profile_image": false,
+                                    "description": "MI LUCHA ESTÁ EN TODAS PARTES. POR LOS DERECHOS DE LOS ANIMALES. POR UN MUNDO MEJOR.ACTIVISTA",
+                                    "entities": {
+                                      "description": {
+                                        "urls": []
+                                      }
+                                    },
+                                    "fast_followers_count": 0,
+                                    "favourites_count": 108021,
+                                    "followers_count": 15454,
+                                    "friends_count": 13032,
+                                    "has_custom_timelines": true,
+                                    "is_translator": false,
+                                    "listed_count": 34,
+                                    "location": "En donde me den amor ❤️😊🥰",
+                                    "media_count": 12985,
+                                    "name": "SisuMisu",
+                                    "normal_followers_count": 15454,
+                                    "pinned_tweet_ids_str": [
+                                      "1087440689368043520"
+                                    ],
+                                    "possibly_sensitive": false,
+                                    "profile_banner_url": "https://pbs.twimg.com/profile_banners/2949065723/1513372062",
+                                    "profile_image_url_https": "https://pbs.twimg.com/profile_images/1528038627317538817/wvoi8QLf_normal.jpg",
+                                    "profile_interstitial_type": "",
+                                    "screen_name": "SisuMisu3",
+                                    "statuses_count": 164333,
+                                    "translator_type": "none",
+                                    "verified": false,
+                                    "want_retweets": false,
+                                    "withheld_in_countries": []
+                                  },
+                                  "professional": {
+                                    "rest_id": "1490288411835523072",
+                                    "professional_type": "Creator",
+                                    "category": [
+                                      {
+                                        "id": 934,
+                                        "name": "Social Media Influencer",
+                                        "icon_name": "IconBriefcaseStroke"
+                                      }
+                                    ]
+                                  },
+                                  "business_account": {}
+                                }
+                              }
+                            },
+                            "edit_control": {
+                              "edit_tweet_ids": [
+                                "1677037919880507392"
+                              ],
+                              "editable_until_msecs": "1688675572000",
+                              "is_edit_eligible": true,
+                              "edits_remaining": "5"
+                            },
+                            "edit_perspective": {
+                              "favorited": false,
+                              "retweeted": false
+                            },
+                            "is_translatable": true,
+                            "views": {
+                              "count": "75",
+                              "state": "EnabledWithCount"
+                            },
+                            "source": "<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>",
+                            "quoted_status_result": {
+                              "result": {
+                                "__typename": "Tweet",
+                                "rest_id": "1677035434214412289",
+                                "has_birdwatch_notes": false,
+                                "core": {
+                                  "user_results": {
+                                    "result": {
+                                      "__typename": "User",
+                                      "id": "VXNlcjoxMzA1NDYwNDg5",
+                                      "rest_id": "1305460489",
+                                      "affiliates_highlighted_label": {},
+                                      "has_graduated_access": true,
+                                      "is_blue_verified": false,
+                                      "legacy": {
+                                        "can_dm": false,
+                                        "can_media_tag": false,
+                                        "created_at": "Tue Mar 26 19:56:20 +0000 2013",
+                                        "default_profile": false,
+                                        "default_profile_image": false,
+                                        "description": "Apartidario\n\"No puedo decir que no estoy en desacuerdo contigo\"\nMi humilde opinión . . .",
+                                        "entities": {
+                                          "description": {
+                                            "urls": []
+                                          }
+                                        },
+                                        "fast_followers_count": 0,
+                                        "favourites_count": 24976,
+                                        "followers_count": 3394,
+                                        "friends_count": 3386,
+                                        "has_custom_timelines": true,
+                                        "is_translator": false,
+                                        "listed_count": 35,
+                                        "location": "Madrid, Comunidad de Madrid",
+                                        "media_count": 5191,
+                                        "name": "P.S.V.",
+                                        "normal_followers_count": 3394,
+                                        "pinned_tweet_ids_str": [
+                                          "1075291506767003648"
+                                        ],
+                                        "possibly_sensitive": false,
+                                        "profile_banner_url": "https://pbs.twimg.com/profile_banners/1305460489/1437468007",
+                                        "profile_image_url_https": "https://pbs.twimg.com/profile_images/869271246885793794/6YZVDC_r_normal.jpg",
+                                        "profile_interstitial_type": "",
+                                        "screen_name": "PShermanV",
+                                        "statuses_count": 175081,
+                                        "translator_type": "none",
+                                        "verified": false,
+                                        "want_retweets": false,
+                                        "withheld_in_countries": []
+                                      },
+                                      "professional": {
+                                        "rest_id": "1580505200997138433",
+                                        "professional_type": "Business",
+                                        "category": [
+                                          {
+                                            "id": 15,
+                                            "name": "Entertainment & Recreation",
+                                            "icon_name": "IconBriefcaseStroke"
+                                          }
+                                        ]
+                                      },
+                                      "business_account": {}
+                                    }
+                                  }
+                                },
+                                "edit_control": {
+                                  "edit_tweet_ids": [
+                                    "1677035434214412289"
+                                  ],
+                                  "editable_until_msecs": "1688674979000",
+                                  "is_edit_eligible": false,
+                                  "edits_remaining": "5"
+                                },
+                                "edit_perspective": {
+                                  "favorited": false,
+                                  "retweeted": false
+                                },
+                                "is_translatable": false,
+                                "views": {
+                                  "count": "83",
+                                  "state": "EnabledWithCount"
+                                },
+                                "source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
+                                "legacy": {
+                                  "bookmark_count": 0,
+                                  "bookmarked": false,
+                                  "created_at": "Thu Jul 06 19:22:59 +0000 2023",
+                                  "conversation_id_str": "1676995076461821958",
+                                  "display_text_range": [
+                                    28,
+                                    99
+                                  ],
+                                  "entities": {
+                                    "user_mentions": [
+                                      {
+                                        "id_str": "2949065723",
+                                        "name": "SisuMisu",
+                                        "screen_name": "SisuMisu3",
+                                        "indices": [
+                                          0,
+                                          10
+                                        ]
+                                      },
+                                      {
+                                        "id_str": "888853157912227840",
+                                        "name": "Rosa Isabel",
+                                        "screen_name": "RosaIsa38309247",
+                                        "indices": [
+                                          11,
+                                          27
+                                        ]
+                                      }
+                                    ],
+                                    "urls": [],
+                                    "hashtags": [],
+                                    "symbols": []
+                                  },
+                                  "favorite_count": 1,
+                                  "favorited": false,
+                                  "full_text": "@SisuMisu3 @RosaIsa38309247 O ESPABILAMOS O A LIMPIAR LAS ESCOPETAS\nANTE AQUELLO DE LOS 26 MILLONES",
+                                  "in_reply_to_screen_name": "SisuMisu3",
+                                  "in_reply_to_status_id_str": "1676995076461821958",
+                                  "in_reply_to_user_id_str": "2949065723",
+                                  "is_quote_status": false,
+                                  "lang": "es",
+                                  "quote_count": 1,
+                                  "reply_count": 0,
+                                  "retweet_count": 0,
+                                  "retweeted": false,
+                                  "user_id_str": "1305460489",
+                                  "id_str": "1677035434214412289"
+                                }
+                              }
+                            },
+                            "legacy": {
+                              "bookmark_count": 0,
+                              "bookmarked": false,
+                              "created_at": "Thu Jul 06 19:32:52 +0000 2023",
+                              "conversation_id_str": "1677037919880507392",
+                              "display_text_range": [
+                                0,
+                                90
+                              ],
+                              "entities": {
+                                "user_mentions": [],
+                                "urls": [],
+                                "hashtags": [],
+                                "symbols": []
+                              },
+                              "favorite_count": 2,
+                              "favorited": false,
+                              "full_text": "Tengo fé en que todo saldrá bien.\nNi como esperamos pero nos librará de este sin sentido .",
+                              "is_quote_status": true,
+                              "lang": "es",
+                              "quote_count": 0,
+                              "quoted_status_id_str": "1677035434214412289",
+                              "quoted_status_permalink": {
+                                "url": "https://t.co/X6fr18ZfNN",
+                                "expanded": "https://twitter.com/PShermanV/status/1677035434214412289",
+                                "display": "twitter.com/PShermanV/stat…"
+                              },
+                              "reply_count": 0,
+                              "retweet_count": 0,
+                              "retweeted": false,
+                              "user_id_str": "2949065723",
+                              "id_str": "1677037919880507392"
+                            },
+                            "quick_promote_eligibility": {
+                              "eligibility": "IneligibleNotProfessional"
+                            }
+                          }
+                        },
+                        "tweetDisplayType": "Tweet"
+                      },
+                      "clientEventInfo": {
+                        "component": "related_tweet",
+                        "details": {
+                          "conversationDetails": {
+                            "conversationSection": "RelatedTweet"
+                          }
+                        }
+                      }
+                    }
+                  }
+                ],
+                "displayType": "Vertical",
+                "header": {
+                  "displayType": "Classic",
+                  "text": "Discover more",
+                  "socialContext": {
+                    "type": "TimelineGeneralContext",
+                    "contextType": "TextOnly",
+                    "text": "Sourced from across Twitter"
+                  },
+                  "sticky": true
+                },
+                "clientEventInfo": {
+                  "component": "related_tweet",
+                  "details": {
+                    "conversationDetails": {
+                      "conversationSection": "RelatedTweet"
+                    }
+                  }
+                }
+              }
+            }
+          ]
+        },
+        {
+          "type": "TimelineTerminateTimeline",
+          "direction": "Top"
+        }
+      ]
+    }
+  }
+}

apps/twscrape/tests/mocked-data/_issue_28_2.json

@@ -0,0 +1,914 @@
+{
+  "data": {
+    "threaded_conversation_with_injections_v2": {
+      "instructions": [
+        {
+          "type": "TimelineAddEntries",
+          "entries": [
+            {
+              "entryId": "tweet-1658421690001502208",
+              "sortIndex": "7564950346853273599",
+              "content": {
+                "entryType": "TimelineTimelineItem",
+                "__typename": "TimelineTimelineItem",
+                "itemContent": {
+                  "itemType": "TimelineTweet",
+                  "__typename": "TimelineTweet",
+                  "tweet_results": {
+                    "result": {
+                      "__typename": "Tweet",
+                      "rest_id": "1658421690001502208",
+                      "has_birdwatch_notes": false,
+                      "core": {
+                        "user_results": {
+                          "result": {
+                            "__typename": "User",
+                            "id": "VXNlcjo0MzYxNDk1OTg=",
+                            "rest_id": "436149598",
+                            "affiliates_highlighted_label": {},
+                            "has_graduated_access": true,
+                            "is_blue_verified": false,
+                            "legacy": {
+                              "can_dm": false,
+                              "can_media_tag": true,
+                              "created_at": "Tue Dec 13 21:10:47 +0000 2011",
+                              "default_profile": false,
+                              "default_profile_image": false,
+                              "description": "De Salamanca,",
+                              "entities": {
+                                "description": {
+                                  "urls": []
+                                }
+                              },
+                              "fast_followers_count": 0,
+                              "favourites_count": 18457,
+                              "followers_count": 3177,
+                              "friends_count": 3040,
+                              "has_custom_timelines": true,
+                              "is_translator": false,
+                              "listed_count": 5,
+                              "location": "españa",
+                              "media_count": 1613,
+                              "name": "Moreno",
+                              "normal_followers_count": 3177,
+                              "pinned_tweet_ids_str": [],
+                              "possibly_sensitive": false,
+                              "profile_banner_url": "https://pbs.twimg.com/profile_banners/436149598/1368225000",
+                              "profile_image_url_https": "https://pbs.twimg.com/profile_images/1664391740881510400/TMMIzBow_normal.jpg",
+                              "profile_interstitial_type": "",
+                              "screen_name": "morenoagosto",
+                              "statuses_count": 24367,
+                              "translator_type": "none",
+                              "verified": false,
+                              "want_retweets": false,
+                              "withheld_in_countries": []
+                            },
+                            "business_account": {}
+                          }
+                        }
+                      },
+                      "edit_control": {
+                        "edit_tweet_ids": [
+                          "1658421690001502208"
+                        ],
+                        "editable_until_msecs": "1684235317000",
+                        "is_edit_eligible": true,
+                        "edits_remaining": "5"
+                      },
+                      "edit_perspective": {
+                        "favorited": false,
+                        "retweeted": false
+                      },
+                      "is_translatable": false,
+                      "views": {
+                        "count": "67",
+                        "state": "EnabledWithCount"
+                      },
+                      "source": "<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>",
+                      "quoted_status_result": {
+                        "result": {
+                          "__typename": "Tweet",
+                          "rest_id": "1658407192859492355",
+                          "has_birdwatch_notes": false,
+                          "core": {
+                            "user_results": {
+                              "result": {
+                                "__typename": "User",
+                                "id": "VXNlcjoyOTQ5MDY1NzIz",
+                                "rest_id": "2949065723",
+                                "affiliates_highlighted_label": {},
+                                "has_graduated_access": true,
+                                "is_blue_verified": false,
+                                "legacy": {
+                                  "can_dm": true,
+                                  "can_media_tag": false,
+                                  "created_at": "Mon Dec 29 11:54:26 +0000 2014",
+                                  "default_profile": true,
+                                  "default_profile_image": false,
+                                  "description": "MI LUCHA ESTÁ EN TODAS PARTES. POR LOS DERECHOS DE LOS ANIMALES. POR UN MUNDO MEJOR.ACTIVISTA",
+                                  "entities": {
+                                    "description": {
+                                      "urls": []
+                                    }
+                                  },
+                                  "fast_followers_count": 0,
+                                  "favourites_count": 108032,
+                                  "followers_count": 15455,
+                                  "friends_count": 13034,
+                                  "has_custom_timelines": true,
+                                  "is_translator": false,
+                                  "listed_count": 34,
+                                  "location": "En donde me den amor ❤️😊🥰",
+                                  "media_count": 12987,
+                                  "name": "SisuMisu",
+                                  "normal_followers_count": 15455,
+                                  "pinned_tweet_ids_str": [
+                                    "1087440689368043520"
+                                  ],
+                                  "possibly_sensitive": false,
+                                  "profile_banner_url": "https://pbs.twimg.com/profile_banners/2949065723/1513372062",
+                                  "profile_image_url_https": "https://pbs.twimg.com/profile_images/1528038627317538817/wvoi8QLf_normal.jpg",
+                                  "profile_interstitial_type": "",
+                                  "screen_name": "SisuMisu3",
+                                  "statuses_count": 164349,
+                                  "translator_type": "none",
+                                  "verified": false,
+                                  "want_retweets": false,
+                                  "withheld_in_countries": []
+                                },
+                                "professional": {
+                                  "rest_id": "1490288411835523072",
+                                  "professional_type": "Creator",
+                                  "category": [
+                                    {
+                                      "id": 934,
+                                      "name": "Social Media Influencer",
+                                      "icon_name": "IconBriefcaseStroke"
+                                    }
+                                  ]
+                                },
+                                "business_account": {}
+                              }
+                            }
+                          },
+                          "edit_control": {
+                            "edit_tweet_ids": [
+                              "1658407192859492355"
+                            ],
+                            "editable_until_msecs": "1684231861000",
+                            "is_edit_eligible": true,
+                            "edits_remaining": "5"
+                          },
+                          "edit_perspective": {
+                            "favorited": false,
+                            "retweeted": false
+                          },
+                          "is_translatable": true,
+                          "views": {
+                            "count": "1110",
+                            "state": "EnabledWithCount"
+                          },
+                          "source": "<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>",
+                          "legacy": {
+                            "bookmark_count": 0,
+                            "bookmarked": false,
+                            "created_at": "Tue May 16 09:41:01 +0000 2023",
+                            "conversation_id_str": "1658407192859492355",
+                            "display_text_range": [
+                              0,
+                              121
+                            ],
+                            "entities": {
+                              "media": [
+                                {
+                                  "display_url": "pic.twitter.com/apswi6Wwiw",
+                                  "expanded_url": "https://twitter.com/SisuMisu3/status/1658407192859492355/photo/1",
+                                  "id_str": "1658407190594453505",
+                                  "indices": [
+                                    122,
+                                    145
+                                  ],
+                                  "media_url_https": "https://pbs.twimg.com/media/FwPYjaBWAAETpRI.jpg",
+                                  "type": "photo",
+                                  "url": "https://t.co/apswi6Wwiw",
+                                  "features": {
+                                    "large": {
+                                      "faces": [
+                                        {
+                                          "x": 273,
+                                          "y": 253,
+                                          "h": 351,
+                                          "w": 351
+                                        }
+                                      ]
+                                    },
+                                    "medium": {
+                                      "faces": [
+                                        {
+                                          "x": 255,
+                                          "y": 237,
+                                          "h": 329,
+                                          "w": 329
+                                        }
+                                      ]
+                                    },
+                                    "small": {
+                                      "faces": [
+                                        {
+                                          "x": 145,
+                                          "y": 134,
+                                          "h": 186,
+                                          "w": 186
+                                        }
+                                      ]
+                                    },
+                                    "orig": {
+                                      "faces": [
+                                        {
+                                          "x": 273,
+                                          "y": 253,
+                                          "h": 351,
+                                          "w": 351
+                                        }
+                                      ]
+                                    }
+                                  },
+                                  "sizes": {
+                                    "large": {
+                                      "h": 1280,
+                                      "w": 1024,
+                                      "resize": "fit"
+                                    },
+                                    "medium": {
+                                      "h": 1200,
+                                      "w": 960,
+                                      "resize": "fit"
+                                    },
+                                    "small": {
+                                      "h": 680,
+                                      "w": 544,
+                                      "resize": "fit"
+                                    },
+                                    "thumb": {
+                                      "h": 150,
+                                      "w": 150,
+                                      "resize": "crop"
+                                    }
+                                  },
+                                  "original_info": {
+                                    "height": 1280,
+                                    "width": 1024,
+                                    "focus_rects": [
+                                      {
+                                        "x": 0,
+                                        "y": 642,
+                                        "w": 1024,
+                                        "h": 573
+                                      },
+                                      {
+                                        "x": 0,
+                                        "y": 256,
+                                        "w": 1024,
+                                        "h": 1024
+                                      },
+                                      {
+                                        "x": 0,
+                                        "y": 113,
+                                        "w": 1024,
+                                        "h": 1167
+                                      },
+                                      {
+                                        "x": 32,
+                                        "y": 0,
+                                        "w": 640,
+                                        "h": 1280
+                                      },
+                                      {
+                                        "x": 0,
+                                        "y": 0,
+                                        "w": 1024,
+                                        "h": 1280
+                                      }
+                                    ]
+                                  }
+                                }
+                              ],
+                              "user_mentions": [
+                                {
+                                  "id_str": "1860300192",
+                                  "name": "Alejandra Jacinto",
+                                  "screen_name": "AleJacintoUrang",
+                                  "indices": [
+                                    81,
+                                    97
+                                  ]
+                                }
+                              ],
+                              "urls": [],
+                              "hashtags": [
+                                {
+                                  "indices": [
+                                    116,
+                                    121
+                                  ],
+                                  "text": "Sisu"
+                                }
+                              ],
+                              "symbols": []
+                            },
+                            "extended_entities": {
+                              "media": [
+                                {
+                                  "display_url": "pic.twitter.com/apswi6Wwiw",
+                                  "expanded_url": "https://twitter.com/SisuMisu3/status/1658407192859492355/photo/1",
+                                  "id_str": "1658407190594453505",
+                                  "indices": [
+                                    122,
+                                    145
+                                  ],
+                                  "media_key": "3_1658407190594453505",
+                                  "media_url_https": "https://pbs.twimg.com/media/FwPYjaBWAAETpRI.jpg",
+                                  "type": "photo",
+                                  "url": "https://t.co/apswi6Wwiw",
+                                  "ext_media_availability": {
+                                    "status": "Available"
+                                  },
+                                  "features": {
+                                    "large": {
+                                      "faces": [
+                                        {
+                                          "x": 273,
+                                          "y": 253,
+                                          "h": 351,
+                                          "w": 351
+                                        }
+                                      ]
+                                    },
+                                    "medium": {
+                                      "faces": [
+                                        {
+                                          "x": 255,
+                                          "y": 237,
+                                          "h": 329,
+                                          "w": 329
+                                        }
+                                      ]
+                                    },
+                                    "small": {
+                                      "faces": [
+                                        {
+                                          "x": 145,
+                                          "y": 134,
+                                          "h": 186,
+                                          "w": 186
+                                        }
+                                      ]
+                                    },
+                                    "orig": {
+                                      "faces": [
+                                        {
+                                          "x": 273,
+                                          "y": 253,
+                                          "h": 351,
+                                          "w": 351
+                                        }
+                                      ]
+                                    }
+                                  },
+                                  "sizes": {
+                                    "large": {
+                                      "h": 1280,
+                                      "w": 1024,
+                                      "resize": "fit"
+                                    },
+                                    "medium": {
+                                      "h": 1200,
+                                      "w": 960,
+                                      "resize": "fit"
+                                    },
+                                    "small": {
+                                      "h": 680,
+                                      "w": 544,
+                                      "resize": "fit"
+                                    },
+                                    "thumb": {
+                                      "h": 150,
+                                      "w": 150,
+                                      "resize": "crop"
+                                    }
+                                  },
+                                  "original_info": {
+                                    "height": 1280,
+                                    "width": 1024,
+                                    "focus_rects": [
+                                      {
+                                        "x": 0,
+                                        "y": 642,
+                                        "w": 1024,
+                                        "h": 573
+                                      },
+                                      {
+                                        "x": 0,
+                                        "y": 256,
+                                        "w": 1024,
+                                        "h": 1024
+                                      },
+                                      {
+                                        "x": 0,
+                                        "y": 113,
+                                        "w": 1024,
+                                        "h": 1167
+                                      },
+                                      {
+                                        "x": 32,
+                                        "y": 0,
+                                        "w": 640,
+                                        "h": 1280
+                                      },
+                                      {
+                                        "x": 0,
+                                        "y": 0,
+                                        "w": 1024,
+                                        "h": 1280
+                                      }
+                                    ]
+                                  }
+                                }
+                              ]
+                            },
+                            "favorite_count": 60,
+                            "favorited": false,
+                            "full_text": "Esta noche el circo está asegurado 😂😂😂\nAyuso se presenta al único debate.\nVamos \n@AleJacintoUrang \nEstamos contigo.\n#Sisu https://t.co/apswi6Wwiw",
+                            "is_quote_status": false,
+                            "lang": "es",
+                            "possibly_sensitive": false,
+                            "possibly_sensitive_editable": true,
+                            "quote_count": 2,
+                            "reply_count": 5,
+                            "retweet_count": 42,
+                            "retweeted": false,
+                            "user_id_str": "2949065723",
+                            "id_str": "1658407192859492355"
+                          }
+                        }
+                      },
+                      "legacy": {
+                        "bookmark_count": 0,
+                        "bookmarked": false,
+                        "created_at": "Tue May 16 10:38:37 +0000 2023",
+                        "conversation_id_str": "1658421690001502208",
+                        "display_text_range": [
+                          0,
+                          35
+                        ],
+                        "entities": {
+                          "user_mentions": [],
+                          "urls": [],
+                          "hashtags": [],
+                          "symbols": []
+                        },
+                        "favorite_count": 2,
+                        "favorited": false,
+                        "full_text": "Se sabe si le permiten pinganillo??",
+                        "is_quote_status": true,
+                        "lang": "es",
+                        "quote_count": 1,
+                        "quoted_status_id_str": "1658407192859492355",
+                        "quoted_status_permalink": {
+                          "url": "https://t.co/03JmJW6BYc",
+                          "expanded": "https://twitter.com/SisuMisu3/status/1658407192859492355",
+                          "display": "twitter.com/SisuMisu3/stat…"
+                        },
+                        "reply_count": 0,
+                        "retweet_count": 0,
+                        "retweeted": false,
+                        "user_id_str": "436149598",
+                        "id_str": "1658421690001502208"
+                      },
+                      "quick_promote_eligibility": {
+                        "eligibility": "IneligibleNotProfessional"
+                      }
+                    }
+                  },
+                  "tweetDisplayType": "Tweet",
+                  "hasModeratedReplies": false
+                }
+              }
+            },
+            {
+              "entryId": "tweetdetailrelatedtweets-1676889656225411072",
+              "sortIndex": "1",
+              "content": {
+                "entryType": "TimelineTimelineModule",
+                "__typename": "TimelineTimelineModule",
+                "items": [
+                  {
+                    "entryId": "tweetdetailrelatedtweets-1676889656225411072-tweet-1676889656225411072",
+                    "item": {
+                      "itemContent": {
+                        "itemType": "TimelineTweet",
+                        "__typename": "TimelineTweet",
+                        "tweet_results": {
+                          "result": {
+                            "__typename": "Tweet",
+                            "rest_id": "1676889656225411072",
+                            "has_birdwatch_notes": false,
+                            "core": {
+                              "user_results": {
+                                "result": {
+                                  "__typename": "User",
+                                  "id": "VXNlcjo0MzYxNDk1OTg=",
+                                  "rest_id": "436149598",
+                                  "affiliates_highlighted_label": {},
+                                  "has_graduated_access": true,
+                                  "is_blue_verified": false,
+                                  "legacy": {
+                                    "can_dm": false,
+                                    "can_media_tag": true,
+                                    "created_at": "Tue Dec 13 21:10:47 +0000 2011",
+                                    "default_profile": false,
+                                    "default_profile_image": false,
+                                    "description": "De Salamanca,",
+                                    "entities": {
+                                      "description": {
+                                        "urls": []
+                                      }
+                                    },
+                                    "fast_followers_count": 0,
+                                    "favourites_count": 18457,
+                                    "followers_count": 3177,
+                                    "friends_count": 3040,
+                                    "has_custom_timelines": true,
+                                    "is_translator": false,
+                                    "listed_count": 5,
+                                    "location": "españa",
+                                    "media_count": 1613,
+                                    "name": "Moreno",
+                                    "normal_followers_count": 3177,
+                                    "pinned_tweet_ids_str": [],
+                                    "possibly_sensitive": false,
+                                    "profile_banner_url": "https://pbs.twimg.com/profile_banners/436149598/1368225000",
+                                    "profile_image_url_https": "https://pbs.twimg.com/profile_images/1664391740881510400/TMMIzBow_normal.jpg",
+                                    "profile_interstitial_type": "",
+                                    "screen_name": "morenoagosto",
+                                    "statuses_count": 24367,
+                                    "translator_type": "none",
+                                    "verified": false,
+                                    "want_retweets": false,
+                                    "withheld_in_countries": []
+                                  },
+                                  "business_account": {}
+                                }
+                              }
+                            },
+                            "edit_control": {
+                              "edit_tweet_ids": [
+                                "1676889656225411072"
+                              ],
+                              "editable_until_msecs": "1688640223000",
+                              "is_edit_eligible": true,
+                              "edits_remaining": "5"
+                            },
+                            "edit_perspective": {
+                              "favorited": false,
+                              "retweeted": false
+                            },
+                            "is_translatable": true,
+                            "views": {
+                              "count": "40",
+                              "state": "EnabledWithCount"
+                            },
+                            "source": "<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>",
+                            "quoted_status_result": {
+                              "result": {
+                                "__typename": "Tweet",
+                                "rest_id": "1676697529461743622",
+                                "has_birdwatch_notes": false,
+                                "core": {
+                                  "user_results": {
+                                    "result": {
+                                      "__typename": "User",
+                                      "id": "VXNlcjozMjg0OTg3MTA=",
+                                      "rest_id": "328498710",
+                                      "affiliates_highlighted_label": {},
+                                      "has_graduated_access": true,
+                                      "is_blue_verified": false,
+                                      "legacy": {
+                                        "can_dm": false,
+                                        "can_media_tag": true,
+                                        "created_at": "Sun Jul 03 13:57:45 +0000 2011",
+                                        "default_profile": true,
+                                        "default_profile_image": false,
+                                        "description": "** Erik Urano ** // *** Flat Erik *** / Astronauta a tiempo parcial // Extraterrestre a tiempo completo// Neovalladolor // MNGMNT & BKNG luis@sonidomuchacho.com",
+                                        "entities": {
+                                          "description": {
+                                            "urls": []
+                                          },
+                                          "url": {
+                                            "urls": [
+                                              {
+                                                "display_url": "linktr.ee/erikurano",
+                                                "expanded_url": "https://linktr.ee/erikurano",
+                                                "url": "https://t.co/qrNiieuzZw",
+                                                "indices": [
+                                                  0,
+                                                  23
+                                                ]
+                                              }
+                                            ]
+                                          }
+                                        },
+                                        "fast_followers_count": 0,
+                                        "favourites_count": 35402,
+                                        "followers_count": 15175,
+                                        "friends_count": 1602,
+                                        "has_custom_timelines": true,
+                                        "is_translator": false,
+                                        "listed_count": 59,
+                                        "location": "",
+                                        "media_count": 1344,
+                                        "name": "DON DATA®️",
+                                        "normal_followers_count": 15175,
+                                        "pinned_tweet_ids_str": [
+                                          "1674336677597159424"
+                                        ],
+                                        "possibly_sensitive": false,
+                                        "profile_banner_url": "https://pbs.twimg.com/profile_banners/328498710/1687971178",
+                                        "profile_image_url_https": "https://pbs.twimg.com/profile_images/1674098569953902592/5tisrAy__normal.jpg",
+                                        "profile_interstitial_type": "",
+                                        "screen_name": "Erik_Urano",
+                                        "statuses_count": 12901,
+                                        "translator_type": "none",
+                                        "url": "https://t.co/qrNiieuzZw",
+                                        "verified": false,
+                                        "want_retweets": false,
+                                        "withheld_in_countries": []
+                                      },
+                                      "professional": {
+                                        "rest_id": "1554576748209229824",
+                                        "professional_type": "Creator",
+                                        "category": [
+                                          {
+                                            "id": 1006,
+                                            "name": "Musical Artist",
+                                            "icon_name": "IconBriefcaseStroke"
+                                          }
+                                        ]
+                                      },
+                                      "business_account": {}
+                                    }
+                                  }
+                                },
+                                "edit_control": {
+                                  "edit_tweet_ids": [
+                                    "1676697529461743622"
+                                  ],
+                                  "editable_until_msecs": "1688594417000",
+                                  "is_edit_eligible": true,
+                                  "edits_remaining": "5"
+                                },
+                                "edit_perspective": {
+                                  "favorited": false,
+                                  "retweeted": false
+                                },
+                                "is_translatable": false,
+                                "views": {
+                                  "count": "973266",
+                                  "state": "EnabledWithCount"
+                                },
+                                "source": "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>",
+                                "legacy": {
+                                  "bookmark_count": 0,
+                                  "bookmarked": false,
+                                  "created_at": "Wed Jul 05 21:00:17 +0000 2023",
+                                  "conversation_id_str": "1676697529461743622",
+                                  "display_text_range": [
+                                    0,
+                                    71
+                                  ],
+                                  "entities": {
+                                    "media": [
+                                      {
+                                        "display_url": "pic.twitter.com/chX4MREd5H",
+                                        "expanded_url": "https://twitter.com/Erik_Urano/status/1676697529461743622/video/1",
+                                        "id_str": "1676697491226468353",
+                                        "indices": [
+                                          72,
+                                          95
+                                        ],
+                                        "media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/1676697491226468353/pu/img/7VGLjWMYpFsXWvq1.jpg",
+                                        "type": "photo",
+                                        "url": "https://t.co/chX4MREd5H",
+                                        "features": {},
+                                        "sizes": {
+                                          "large": {
+                                            "h": 716,
+                                            "w": 1280,
+                                            "resize": "fit"
+                                          },
+                                          "medium": {
+                                            "h": 671,
+                                            "w": 1200,
+                                            "resize": "fit"
+                                          },
+                                          "small": {
+                                            "h": 380,
+                                            "w": 680,
+                                            "resize": "fit"
+                                          },
+                                          "thumb": {
+                                            "h": 150,
+                                            "w": 150,
+                                            "resize": "crop"
+                                          }
+                                        },
+                                        "original_info": {
+                                          "height": 716,
+                                          "width": 1280
+                                        }
+                                      }
+                                    ],
+                                    "user_mentions": [],
+                                    "urls": [],
+                                    "hashtags": [],
+                                    "symbols": []
+                                  },
+                                  "extended_entities": {
+                                    "media": [
+                                      {
+                                        "display_url": "pic.twitter.com/chX4MREd5H",
+                                        "expanded_url": "https://twitter.com/Erik_Urano/status/1676697529461743622/video/1",
+                                        "id_str": "1676697491226468353",
+                                        "indices": [
+                                          72,
+                                          95
+                                        ],
+                                        "media_key": "7_1676697491226468353",
+                                        "media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/1676697491226468353/pu/img/7VGLjWMYpFsXWvq1.jpg",
+                                        "type": "video",
+                                        "url": "https://t.co/chX4MREd5H",
+                                        "additional_media_info": {
+                                          "monetizable": false
+                                        },
+                                        "mediaStats": {
+                                          "viewCount": 622775
+                                        },
+                                        "ext_media_availability": {
+                                          "status": "Available"
+                                        },
+                                        "features": {},
+                                        "sizes": {
+                                          "large": {
+                                            "h": 716,
+                                            "w": 1280,
+                                            "resize": "fit"
+                                          },
+                                          "medium": {
+                                            "h": 671,
+                                            "w": 1200,
+                                            "resize": "fit"
+                                          },
+                                          "small": {
+                                            "h": 380,
+                                            "w": 680,
+                                            "resize": "fit"
+                                          },
+                                          "thumb": {
+                                            "h": 150,
+                                            "w": 150,
+                                            "resize": "crop"
+                                          }
+                                        },
+                                        "original_info": {
+                                          "height": 716,
+                                          "width": 1280
+                                        },
+                                        "video_info": {
+                                          "aspect_ratio": [
+                                            320,
+                                            179
+                                          ],
+                                          "duration_millis": 8536,
+                                          "variants": [
+                                            {
+                                              "content_type": "application/x-mpegURL",
+                                              "url": "https://video.twimg.com/ext_tw_video/1676697491226468353/pu/pl/hmKfq_gU8h5Pj_bi.m3u8?tag=12&container=fmp4"
+                                            },
+                                            {
+                                              "bitrate": 256000,
+                                              "content_type": "video/mp4",
+                                              "url": "https://video.twimg.com/ext_tw_video/1676697491226468353/pu/vid/482x270/CxOsBliV2KQSDySg.mp4?tag=12"
+                                            },
+                                            {
+                                              "bitrate": 2176000,
+                                              "content_type": "video/mp4",
+                                              "url": "https://video.twimg.com/ext_tw_video/1676697491226468353/pu/vid/1280x716/ir6MOu-zeqHCOMDZ.mp4?tag=12"
+                                            },
+                                            {
+                                              "bitrate": 832000,
+                                              "content_type": "video/mp4",
+                                              "url": "https://video.twimg.com/ext_tw_video/1676697491226468353/pu/vid/642x360/vyOPi9cH5pjAcgSS.mp4?tag=12"
+                                            }
+                                          ]
+                                        }
+                                      }
+                                    ]
+                                  },
+                                  "favorite_count": 9694,
+                                  "favorited": false,
+                                  "full_text": "Frase a empezar a utilizar cada vez que me pregunten “¿Que tal estás?”: https://t.co/chX4MREd5H",
+                                  "is_quote_status": false,
+                                  "lang": "es",
+                                  "possibly_sensitive": false,
+                                  "possibly_sensitive_editable": true,
+                                  "quote_count": 401,
+                                  "reply_count": 123,
+                                  "retweet_count": 2289,
+                                  "retweeted": false,
+                                  "user_id_str": "328498710",
+                                  "id_str": "1676697529461743622"
+                                }
+                              }
+                            },
+                            "legacy": {
+                              "bookmark_count": 0,
+                              "bookmarked": false,
+                              "created_at": "Thu Jul 06 09:43:43 +0000 2023",
+                              "conversation_id_str": "1676889656225411072",
+                              "display_text_range": [
+                                0,
+                                66
+                              ],
+                              "entities": {
+                                "user_mentions": [],
+                                "urls": [],
+                                "hashtags": [],
+                                "symbols": []
+                              },
+                              "favorite_count": 0,
+                              "favorited": false,
+                              "full_text": "No, solo quiere decir que aún no ha empezado a darle al \" morapio\"",
+                              "is_quote_status": true,
+                              "lang": "es",
+                              "quote_count": 0,
+                              "quoted_status_id_str": "1676697529461743622",
+                              "quoted_status_permalink": {
+                                "url": "https://t.co/SIZUcNQRrS",
+                                "expanded": "https://twitter.com/Erik_Urano/status/1676697529461743622",
+                                "display": "twitter.com/Erik_Urano/sta…"
+                              },
+                              "reply_count": 0,
+                              "retweet_count": 0,
+                              "retweeted": false,
+                              "user_id_str": "436149598",
+                              "id_str": "1676889656225411072"
+                            },
+                            "quick_promote_eligibility": {
+                              "eligibility": "IneligibleNotProfessional"
+                            }
+                          }
+                        },
+                        "tweetDisplayType": "Tweet"
+                      },
+                      "clientEventInfo": {
+                        "component": "related_tweet",
+                        "details": {
+                          "conversationDetails": {
+                            "conversationSection": "RelatedTweet"
+                          }
+                        }
+                      }
+                    }
+                  }
+                ],
+                "displayType": "Vertical",
+                "header": {
+                  "displayType": "Classic",
+                  "text": "Discover more",
+                  "socialContext": {
+                    "type": "TimelineGeneralContext",
+                    "contextType": "TextOnly",
+                    "text": "Sourced from across Twitter"
+                  },
+                  "sticky": true
+                },
+                "clientEventInfo": {
+                  "component": "related_tweet",
+                  "details": {
+                    "conversationDetails": {
+                      "conversationSection": "RelatedTweet"
+                    }
+                  }
+                }
+              }
+            }
+          ]
+        },
+        {
+          "type": "TimelineTerminateTimeline",
+          "direction": "Top"
+        }
+      ]
+    }
+  }
+}

apps/twscrape/tests/mocked-data/_issue_42.json

@@ -0,0 +1,376 @@
+{
+  "entryId": "tweet-1665951747842641921",
+  "sortIndex": "1682387836784869348",
+  "content": {
+      "entryType": "TimelineTimelineItem",
+      "__typename": "TimelineTimelineItem",
+      "itemContent": {
+          "itemType": "TimelineTweet",
+          "__typename": "TimelineTweet",
+          "tweet_results": {
+              "result": {
+                  "__typename": "Tweet",
+                  "rest_id": "1665951747842641921",
+                  "core": {
+                      "user_results": {
+                          "result": {
+                              "__typename": "User",
+                              "id": "VXNlcjoxNTM5Nzc0OTEzOTY3NzE0MzA0",
+                              "rest_id": "1539774913967714304",
+                              "affiliates_highlighted_label": {},
+                              "has_graduated_access": true,
+                              "is_blue_verified": false,
+                              "profile_image_shape": "Square",
+                              "legacy": {
+                                  "can_dm": false,
+                                  "can_media_tag": true,
+                                  "created_at": "Thu Jun 23 00:59:21 +0000 2022",
+                                  "default_profile": true,
+                                  "default_profile_image": false,
+                                  "description": "AVIC International Holding Corporation (AVIC INTL) is a global share-holding enterprise affiliated to Aviation Industry Corporation of China (AVIC).",
+                                  "entities": {
+                                      "description": {
+                                          "urls": []
+                                      }
+                                  },
+                                  "fast_followers_count": 0,
+                                  "favourites_count": 118,
+                                  "followers_count": 5911,
+                                  "friends_count": 110,
+                                  "has_custom_timelines": true,
+                                  "is_translator": false,
+                                  "listed_count": 3,
+                                  "location": "",
+                                  "media_count": 172,
+                                  "name": "AVIC INTL",
+                                  "normal_followers_count": 5911,
+                                  "pinned_tweet_ids_str": [],
+                                  "possibly_sensitive": false,
+                                  "profile_banner_url": "https://pbs.twimg.com/profile_banners/1539774913967714304/1656380265",
+                                  "profile_image_url_https": "https://pbs.twimg.com/profile_images/1542841676573466625/Twwv8q0a_normal.jpg",
+                                  "profile_interstitial_type": "",
+                                  "screen_name": "_AVICINTL",
+                                  "statuses_count": 244,
+                                  "translator_type": "none",
+                                  "verified": false,
+                                  "verified_type": "Business",
+                                  "want_retweets": false,
+                                  "withheld_in_countries": []
+                              },
+                              "professional": {
+                                  "rest_id": "1565386974147387392",
+                                  "professional_type": "Business",
+                                  "category": []
+                              }
+                          }
+                      }
+                  },
+                  "edit_control": {
+                      "edit_tweet_ids": [
+                          "1665951747842641921"
+                      ],
+                      "editable_until_msecs": "1686032423229",
+                      "is_edit_eligible": false,
+                      "edits_remaining": "5"
+                  },
+                  "edit_perspective": {
+                      "favorited": false,
+                      "retweeted": false
+                  },
+                  "is_translatable": false,
+                  "views": {
+                      "state": "Enabled"
+                  },
+                  "source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
+                  "legacy": {
+                      "bookmark_count": 0,
+                      "bookmarked": false,
+                      "created_at": "Tue Jun 06 05:20:23 +0000 2023",
+                      "conversation_id_str": "1665951747842641921",
+                      "display_text_range": [
+                          0,
+                          140
+                      ],
+                      "entities": {
+                          "user_mentions": [
+                              {
+                                  "id_str": "87775422",
+                                  "name": "China Daily",
+                                  "screen_name": "ChinaDaily",
+                                  "indices": [
+                                      3,
+                                      14
+                                  ]
+                              }
+                          ],
+                          "urls": [],
+                          "hashtags": [],
+                          "symbols": []
+                      },
+                      "favorite_count": 0,
+                      "favorited": false,
+                      "full_text": "RT @ChinaDaily: Today marks the arrival of a traditional Chinese solar term called mangzhong, or Grain in Ear, signifying a busy farming pe…",
+                      "is_quote_status": false,
+                      "lang": "en",
+                      "quote_count": 0,
+                      "reply_count": 0,
+                      "retweet_count": 10,
+                      "retweeted": false,
+                      "user_id_str": "1539774913967714304",
+                      "id_str": "1665951747842641921",
+                      "retweeted_status_result": {
+                          "result": {
+                              "__typename": "Tweet",
+                              "rest_id": "1665863576060575744",
+                              "core": {
+                                  "user_results": {
+                                      "result": {
+                                          "__typename": "User",
+                                          "id": "VXNlcjo4Nzc3NTQyMg==",
+                                          "rest_id": "87775422",
+                                          "affiliates_highlighted_label": {},
+                                          "has_graduated_access": true,
+                                          "is_blue_verified": true,
+                                          "profile_image_shape": "Circle",
+                                          "legacy": {
+                                              "can_dm": true,
+                                              "can_media_tag": true,
+                                              "created_at": "Thu Nov 05 20:30:10 +0000 2009",
+                                              "default_profile": false,
+                                              "default_profile_image": false,
+                                              "description": "Start a conversation as we share news and analysis from #China and beyond. \nFB: China Daily \nTelegram: https://t.co/7nB28KWLCr…\nIns: chinadailynews",
+                                              "entities": {
+                                                  "description": {
+                                                      "urls": [
+                                                          {
+                                                              "display_url": "t.me/chinadaily_off",
+                                                              "expanded_url": "http://t.me/chinadaily_off",
+                                                              "url": "https://t.co/7nB28KWLCr",
+                                                              "indices": [
+                                                                  103,
+                                                                  126
+                                                              ]
+                                                          }
+                                                      ]
+                                                  }
+                                              },
+                                              "fast_followers_count": 0,
+                                              "favourites_count": 1155,
+                                              "followers_count": 4161285,
+                                              "friends_count": 568,
+                                              "has_custom_timelines": false,
+                                              "is_translator": false,
+                                              "listed_count": 6975,
+                                              "location": "Beijing, China",
+                                              "media_count": 110028,
+                                              "name": "China Daily",
+                                              "normal_followers_count": 4161285,
+                                              "pinned_tweet_ids_str": [
+                                                  "1679729828105445377"
+                                              ],
+                                              "possibly_sensitive": false,
+                                              "profile_banner_url": "https://pbs.twimg.com/profile_banners/87775422/1683168821",
+                                              "profile_image_url_https": "https://pbs.twimg.com/profile_images/1598185470353022976/-KlKi0WI_normal.jpg",
+                                              "profile_interstitial_type": "",
+                                              "screen_name": "ChinaDaily",
+                                              "statuses_count": 204098,
+                                              "translator_type": "none",
+                                              "verified": false,
+                                              "want_retweets": false,
+                                              "withheld_in_countries": []
+                                          },
+                                          "professional": {
+                                              "rest_id": "1473187902209536003",
+                                              "professional_type": "Creator",
+                                              "category": [
+                                                  {
+                                                      "id": 580,
+                                                      "name": "Media & News Company",
+                                                      "icon_name": "IconBriefcaseStroke"
+                                                  }
+                                              ]
+                                          }
+                                      }
+                                  }
+                              },
+                              "edit_control": {
+                                  "edit_tweet_ids": [
+                                      "1665863576060575744"
+                                  ],
+                                  "editable_until_msecs": "1686011401000",
+                                  "is_edit_eligible": true,
+                                  "edits_remaining": "5"
+                              },
+                              "edit_perspective": {
+                                  "favorited": false,
+                                  "retweeted": false
+                              },
+                              "is_translatable": false,
+                              "views": {
+                                  "count": "5188",
+                                  "state": "EnabledWithCount"
+                              },
+                              "source": "<a href=\"http://erased21340332_45IhsckRBy.com\" rel=\"nofollow\">erased21340332_45IhsckRBy</a>",
+                              "legacy": {
+                                  "bookmark_count": 0,
+                                  "bookmarked": false,
+                                  "created_at": "Mon Jun 05 23:30:01 +0000 2023",
+                                  "conversation_id_str": "1665863576060575744",
+                                  "display_text_range": [
+                                      0,
+                                      128
+                                  ],
+                                  "entities": {
+                                      "media": [
+                                          {
+                                              "display_url": "pic.twitter.com/SQMrX99bWr",
+                                              "expanded_url": "https://twitter.com/ChinaDaily/status/1665863576060575744/video/1",
+                                              "id_str": "1665861064259686400",
+                                              "indices": [
+                                                  129,
+                                                  152
+                                              ],
+                                              "media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/1665861064259686400/pu/img/D1qkMrHVMTV_ifFk.jpg",
+                                              "type": "photo",
+                                              "url": "https://t.co/SQMrX99bWr",
+                                              "features": {},
+                                              "sizes": {
+                                                  "large": {
+                                                      "h": 640,
+                                                      "w": 296,
+                                                      "resize": "fit"
+                                                  },
+                                                  "medium": {
+                                                      "h": 640,
+                                                      "w": 296,
+                                                      "resize": "fit"
+                                                  },
+                                                  "small": {
+                                                      "h": 640,
+                                                      "w": 296,
+                                                      "resize": "fit"
+                                                  },
+                                                  "thumb": {
+                                                      "h": 150,
+                                                      "w": 150,
+                                                      "resize": "crop"
+                                                  }
+                                              },
+                                              "original_info": {
+                                                  "height": 640,
+                                                  "width": 296
+                                              }
+                                          }
+                                      ],
+                                      "user_mentions": [],
+                                      "urls": [],
+                                      "hashtags": [],
+                                      "symbols": []
+                                  },
+                                  "extended_entities": {
+                                      "media": [
+                                          {
+                                              "display_url": "pic.twitter.com/SQMrX99bWr",
+                                              "expanded_url": "https://twitter.com/ChinaDaily/status/1665863576060575744/video/1",
+                                              "id_str": "1665861064259686400",
+                                              "indices": [
+                                                  129,
+                                                  152
+                                              ],
+                                              "media_key": "7_1665861064259686400",
+                                              "media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/1665861064259686400/pu/img/D1qkMrHVMTV_ifFk.jpg",
+                                              "type": "video",
+                                              "url": "https://t.co/SQMrX99bWr",
+                                              "additional_media_info": {
+                                                  "monetizable": false
+                                              },
+                                              "mediaStats": {
+                                                  "viewCount": 961
+                                              },
+                                              "ext_media_availability": {
+                                                  "status": "Available"
+                                              },
+                                              "features": {},
+                                              "sizes": {
+                                                  "large": {
+                                                      "h": 640,
+                                                      "w": 296,
+                                                      "resize": "fit"
+                                                  },
+                                                  "medium": {
+                                                      "h": 640,
+                                                      "w": 296,
+                                                      "resize": "fit"
+                                                  },
+                                                  "small": {
+                                                      "h": 640,
+                                                      "w": 296,
+                                                      "resize": "fit"
+                                                  },
+                                                  "thumb": {
+                                                      "h": 150,
+                                                      "w": 150,
+                                                      "resize": "crop"
+                                                  }
+                                              },
+                                              "original_info": {
+                                                  "height": 640,
+                                                  "width": 296
+                                              },
+                                              "video_info": {
+                                                  "aspect_ratio": [
+                                                      37,
+                                                      80
+                                                  ],
+                                                  "duration_millis": 16000,
+                                                  "variants": [
+                                                      {
+                                                          "content_type": "application/x-mpegURL",
+                                                          "url": "https://video.twimg.com/ext_tw_video/1665861064259686400/pu/pl/pPnoCo_jLID-udSS.m3u8?tag=12&container=fmp4"
+                                                      },
+                                                      {
+                                                          "bitrate": 632000,
+                                                          "content_type": "video/mp4",
+                                                          "url": "https://video.twimg.com/ext_tw_video/1665861064259686400/pu/vid/296x640/qQVrB3CenvchQsuP.mp4?tag=12"
+                                                      }
+                                                  ]
+                                              }
+                                          }
+                                      ]
+                                  },
+                                  "favorite_count": 27,
+                                  "favorited": false,
+                                  "full_text": "Today marks the arrival of a traditional Chinese solar term called mangzhong, or Grain in Ear, signifying a busy farming period. https://t.co/SQMrX99bWr",
+                                  "is_quote_status": false,
+                                  "lang": "en",
+                                  "possibly_sensitive": false,
+                                  "possibly_sensitive_editable": true,
+                                  "quote_count": 5,
+                                  "reply_count": 4,
+                                  "retweet_count": 10,
+                                  "retweeted": false,
+                                  "user_id_str": "87775422",
+                                  "id_str": "1665863576060575744"
+                              }
+                          }
+                      }
+                  },
+                  "quick_promote_eligibility": {
+                      "eligibility": "IneligibleNotProfessional"
+                  }
+              }
+          },
+          "tweetDisplayType": "Tweet"
+      },
+      "clientEventInfo": {
+          "component": "tweet",
+          "element": "tweet",
+          "details": {
+              "timelinesDetails": {
+                  "injectionType": "RankedOrganicTweet",
+                  "controllerData": "DAACDAABDAABCgABAAAAAAAAAAAKAAkPUKvlsZRgAAAAAAA="
+              }
+          }
+      }
+  }
+}

apps/twscrape/tests/mocked-data/_issue_42_regr.json

@@ -0,0 +1,699 @@
+{
+  "data": {
+    "threaded_conversation_with_injections_v2": {
+      "instructions": [
+        {
+          "type": "TimelineAddEntries",
+          "entries": [
+            {
+              "entryId": "tweet-1665951747842641921",
+              "sortIndex": "7557420289012133886",
+              "content": {
+                "entryType": "TimelineTimelineItem",
+                "__typename": "TimelineTimelineItem",
+                "itemContent": {
+                  "itemType": "TimelineTweet",
+                  "__typename": "TimelineTweet",
+                  "tweet_results": {
+                    "result": {
+                      "__typename": "Tweet",
+                      "rest_id": "1665951747842641921",
+                      "has_birdwatch_notes": false,
+                      "core": {
+                        "user_results": {
+                          "result": {
+                            "__typename": "User",
+                            "id": "VXNlcjoxNTM5Nzc0OTEzOTY3NzE0MzA0",
+                            "rest_id": "1539774913967714304",
+                            "affiliates_highlighted_label": {},
+                            "has_graduated_access": true,
+                            "is_blue_verified": false,
+                            "profile_image_shape": "Square",
+                            "legacy": {
+                              "can_dm": false,
+                              "can_media_tag": true,
+                              "created_at": "Thu Jun 23 00:59:21 +0000 2022",
+                              "default_profile": true,
+                              "default_profile_image": false,
+                              "description": "AVIC International Holding Corporation (AVIC INTL) is a global share-holding enterprise affiliated to Aviation Industry Corporation of China (AVIC).",
+                              "entities": {
+                                "description": {
+                                  "urls": []
+                                }
+                              },
+                              "fast_followers_count": 0,
+                              "favourites_count": 145,
+                              "followers_count": 11672,
+                              "friends_count": 118,
+                              "has_custom_timelines": true,
+                              "is_translator": false,
+                              "listed_count": 6,
+                              "location": "",
+                              "media_count": 195,
+                              "name": "AVIC INTL",
+                              "normal_followers_count": 11672,
+                              "pinned_tweet_ids_str": [],
+                              "possibly_sensitive": false,
+                              "profile_banner_url": "https://pbs.twimg.com/profile_banners/1539774913967714304/1656380265",
+                              "profile_image_url_https": "https://pbs.twimg.com/profile_images/1542841676573466625/Twwv8q0a_normal.jpg",
+                              "profile_interstitial_type": "",
+                              "screen_name": "_AVICINTL",
+                              "statuses_count": 268,
+                              "translator_type": "none",
+                              "verified": false,
+                              "verified_type": "Business",
+                              "want_retweets": false,
+                              "withheld_in_countries": []
+                            },
+                            "professional": {
+                              "rest_id": "1565386974147387392",
+                              "professional_type": "Business",
+                              "category": []
+                            }
+                          }
+                        }
+                      },
+                      "edit_control": {
+                        "edit_tweet_ids": [
+                          "1665951747842641921"
+                        ],
+                        "editable_until_msecs": "1686032423229",
+                        "is_edit_eligible": false,
+                        "edits_remaining": "5"
+                      },
+                      "is_translatable": false,
+                      "views": {
+                        "state": "Enabled"
+                      },
+                      "source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
+                      "legacy": {
+                        "bookmark_count": 0,
+                        "bookmarked": false,
+                        "created_at": "Tue Jun 06 05:20:23 +0000 2023",
+                        "conversation_id_str": "1665951747842641921",
+                        "display_text_range": [
+                          0,
+                          140
+                        ],
+                        "entities": {
+                          "user_mentions": [
+                            {
+                              "id_str": "87775422",
+                              "name": "China Daily",
+                              "screen_name": "ChinaDaily",
+                              "indices": [
+                                3,
+                                14
+                              ]
+                            }
+                          ],
+                          "urls": [],
+                          "hashtags": [],
+                          "symbols": []
+                        },
+                        "favorite_count": 0,
+                        "favorited": false,
+                        "full_text": "RT @ChinaDaily: Today marks the arrival of a traditional Chinese solar term called mangzhong, or Grain in Ear, signifying a busy farming pe…",
+                        "is_quote_status": false,
+                        "lang": "en",
+                        "quote_count": 0,
+                        "reply_count": 0,
+                        "retweet_count": 10,
+                        "retweeted": false,
+                        "user_id_str": "1539774913967714304",
+                        "id_str": "1665951747842641921",
+                        "retweeted_status_result": {
+                          "result": {
+                            "__typename": "Tweet",
+                            "rest_id": "1665863576060575744",
+                            "has_birdwatch_notes": false,
+                            "core": {
+                              "user_results": {
+                                "result": {
+                                  "__typename": "User",
+                                  "id": "VXNlcjo4Nzc3NTQyMg==",
+                                  "rest_id": "87775422",
+                                  "affiliates_highlighted_label": {},
+                                  "has_graduated_access": true,
+                                  "is_blue_verified": true,
+                                  "profile_image_shape": "Circle",
+                                  "legacy": {
+                                    "can_dm": true,
+                                    "can_media_tag": true,
+                                    "created_at": "Thu Nov 05 20:30:10 +0000 2009",
+                                    "default_profile": false,
+                                    "default_profile_image": false,
+                                    "description": "Start a conversation as we share news and analysis from #China and beyond. \nFB: China Daily \nTelegram: https://t.co/GxDi5xGRTF\nIns: chinadailynews",
+                                    "entities": {
+                                      "description": {
+                                        "urls": [
+                                          {
+                                            "display_url": "t.me/+043Tnnn4bd4yN…",
+                                            "expanded_url": "https://t.me/+043Tnnn4bd4yNTll",
+                                            "url": "https://t.co/GxDi5xGRTF",
+                                            "indices": [
+                                              103,
+                                              126
+                                            ]
+                                          }
+                                        ]
+                                      }
+                                    },
+                                    "fast_followers_count": 0,
+                                    "favourites_count": 1157,
+                                    "followers_count": 4160536,
+                                    "friends_count": 156,
+                                    "has_custom_timelines": false,
+                                    "is_translator": false,
+                                    "listed_count": 6974,
+                                    "location": "Beijing, China",
+                                    "media_count": 111994,
+                                    "name": "China Daily",
+                                    "normal_followers_count": 4160536,
+                                    "pinned_tweet_ids_str": [
+                                      "1700166894593224767"
+                                    ],
+                                    "possibly_sensitive": false,
+                                    "profile_banner_url": "https://pbs.twimg.com/profile_banners/87775422/1683168821",
+                                    "profile_image_url_https": "https://pbs.twimg.com/profile_images/1598185470353022976/-KlKi0WI_normal.jpg",
+                                    "profile_interstitial_type": "",
+                                    "screen_name": "ChinaDaily",
+                                    "statuses_count": 207177,
+                                    "translator_type": "none",
+                                    "verified": false,
+                                    "want_retweets": false,
+                                    "withheld_in_countries": []
+                                  },
+                                  "professional": {
+                                    "rest_id": "1473187902209536003",
+                                    "professional_type": "Creator",
+                                    "category": [
+                                      {
+                                        "id": 580,
+                                        "name": "Media & News Company",
+                                        "icon_name": "IconBriefcaseStroke"
+                                      }
+                                    ]
+                                  }
+                                }
+                              }
+                            },
+                            "edit_control": {
+                              "edit_tweet_ids": [
+                                "1665863576060575744"
+                              ],
+                              "editable_until_msecs": "1686011401000",
+                              "is_edit_eligible": true,
+                              "edits_remaining": "5"
+                            },
+                            "is_translatable": false,
+                            "views": {
+                              "count": "5229",
+                              "state": "EnabledWithCount"
+                            },
+                            "source": "<a href=\"http://erased21340332_45IhsckRBy.com\" rel=\"nofollow\">erased21340332_45IhsckRBy</a>",
+                            "legacy": {
+                              "bookmark_count": 0,
+                              "bookmarked": false,
+                              "created_at": "Mon Jun 05 23:30:01 +0000 2023",
+                              "conversation_id_str": "1665863576060575744",
+                              "display_text_range": [
+                                0,
+                                128
+                              ],
+                              "entities": {
+                                "media": [
+                                  {
+                                    "display_url": "pic.twitter.com/SQMrX99bWr",
+                                    "expanded_url": "https://twitter.com/ChinaDaily/status/1665863576060575744/video/1",
+                                    "id_str": "1665861064259686400",
+                                    "indices": [
+                                      129,
+                                      152
+                                    ],
+                                    "media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/1665861064259686400/pu/img/D1qkMrHVMTV_ifFk.jpg",
+                                    "type": "photo",
+                                    "url": "https://t.co/SQMrX99bWr",
+                                    "features": {},
+                                    "sizes": {
+                                      "large": {
+                                        "h": 640,
+                                        "w": 296,
+                                        "resize": "fit"
+                                      },
+                                      "medium": {
+                                        "h": 640,
+                                        "w": 296,
+                                        "resize": "fit"
+                                      },
+                                      "small": {
+                                        "h": 640,
+                                        "w": 296,
+                                        "resize": "fit"
+                                      },
+                                      "thumb": {
+                                        "h": 150,
+                                        "w": 150,
+                                        "resize": "crop"
+                                      }
+                                    },
+                                    "original_info": {
+                                      "height": 640,
+                                      "width": 296
+                                    }
+                                  }
+                                ],
+                                "user_mentions": [],
+                                "urls": [],
+                                "hashtags": [],
+                                "symbols": []
+                              },
+                              "extended_entities": {
+                                "media": [
+                                  {
+                                    "display_url": "pic.twitter.com/SQMrX99bWr",
+                                    "expanded_url": "https://twitter.com/ChinaDaily/status/1665863576060575744/video/1",
+                                    "id_str": "1665861064259686400",
+                                    "indices": [
+                                      129,
+                                      152
+                                    ],
+                                    "media_key": "7_1665861064259686400",
+                                    "media_url_https": "https://pbs.twimg.com/ext_tw_video_thumb/1665861064259686400/pu/img/D1qkMrHVMTV_ifFk.jpg",
+                                    "type": "video",
+                                    "url": "https://t.co/SQMrX99bWr",
+                                    "additional_media_info": {
+                                      "monetizable": false
+                                    },
+                                    "mediaStats": {
+                                      "viewCount": 964
+                                    },
+                                    "ext_media_availability": {
+                                      "status": "Available"
+                                    },
+                                    "features": {},
+                                    "sizes": {
+                                      "large": {
+                                        "h": 640,
+                                        "w": 296,
+                                        "resize": "fit"
+                                      },
+                                      "medium": {
+                                        "h": 640,
+                                        "w": 296,
+                                        "resize": "fit"
+                                      },
+                                      "small": {
+                                        "h": 640,
+                                        "w": 296,
+                                        "resize": "fit"
+                                      },
+                                      "thumb": {
+                                        "h": 150,
+                                        "w": 150,
+                                        "resize": "crop"
+                                      }
+                                    },
+                                    "original_info": {
+                                      "height": 640,
+                                      "width": 296
+                                    },
+                                    "video_info": {
+                                      "aspect_ratio": [
+                                        37,
+                                        80
+                                      ],
+                                      "duration_millis": 16000,
+                                      "variants": [
+                                        {
+                                          "content_type": "application/x-mpegURL",
+                                          "url": "https://video.twimg.com/ext_tw_video/1665861064259686400/pu/pl/pPnoCo_jLID-udSS.m3u8?tag=12&container=fmp4"
+                                        },
+                                        {
+                                          "bitrate": 632000,
+                                          "content_type": "video/mp4",
+                                          "url": "https://video.twimg.com/ext_tw_video/1665861064259686400/pu/vid/296x640/qQVrB3CenvchQsuP.mp4?tag=12"
+                                        }
+                                      ]
+                                    }
+                                  }
+                                ]
+                              },
+                              "favorite_count": 26,
+                              "favorited": false,
+                              "full_text": "Today marks the arrival of a traditional Chinese solar term called mangzhong, or Grain in Ear, signifying a busy farming period. https://t.co/SQMrX99bWr",
+                              "is_quote_status": false,
+                              "lang": "en",
+                              "possibly_sensitive": false,
+                              "possibly_sensitive_editable": true,
+                              "quote_count": 5,
+                              "reply_count": 4,
+                              "retweet_count": 10,
+                              "retweeted": false,
+                              "user_id_str": "87775422",
+                              "id_str": "1665863576060575744"
+                            }
+                          }
+                        }
+                      },
+                      "quick_promote_eligibility": {
+                        "eligibility": "IneligibleNotProfessional"
+                      }
+                    }
+                  },
+                  "tweetDisplayType": "Tweet",
+                  "hasModeratedReplies": false
+                }
+              }
+            },
+            {
+              "entryId": "tweetdetailrelatedtweets-1700166894593224767",
+              "sortIndex": "1",
+              "content": {
+                "entryType": "TimelineTimelineModule",
+                "__typename": "TimelineTimelineModule",
+                "items": [
+                  {
+                    "entryId": "tweetdetailrelatedtweets-1700166894593224767-tweet-1700166894593224767",
+                    "item": {
+                      "itemContent": {
+                        "itemType": "TimelineTweet",
+                        "__typename": "TimelineTweet",
+                        "tweet_results": {
+                          "result": {
+                            "__typename": "Tweet",
+                            "rest_id": "1700166894593224767",
+                            "has_birdwatch_notes": false,
+                            "core": {
+                              "user_results": {
+                                "result": {
+                                  "__typename": "User",
+                                  "id": "VXNlcjo4Nzc3NTQyMg==",
+                                  "rest_id": "87775422",
+                                  "affiliates_highlighted_label": {},
+                                  "has_graduated_access": true,
+                                  "is_blue_verified": true,
+                                  "profile_image_shape": "Circle",
+                                  "legacy": {
+                                    "can_dm": true,
+                                    "can_media_tag": true,
+                                    "created_at": "Thu Nov 05 20:30:10 +0000 2009",
+                                    "default_profile": false,
+                                    "default_profile_image": false,
+                                    "description": "Start a conversation as we share news and analysis from #China and beyond. \nFB: China Daily \nTelegram: https://t.co/GxDi5xGRTF\nIns: chinadailynews",
+                                    "entities": {
+                                      "description": {
+                                        "urls": [
+                                          {
+                                            "display_url": "t.me/+043Tnnn4bd4yN…",
+                                            "expanded_url": "https://t.me/+043Tnnn4bd4yNTll",
+                                            "url": "https://t.co/GxDi5xGRTF",
+                                            "indices": [
+                                              103,
+                                              126
+                                            ]
+                                          }
+                                        ]
+                                      }
+                                    },
+                                    "fast_followers_count": 0,
+                                    "favourites_count": 1157,
+                                    "followers_count": 4160536,
+                                    "friends_count": 156,
+                                    "has_custom_timelines": false,
+                                    "is_translator": false,
+                                    "listed_count": 6974,
+                                    "location": "Beijing, China",
+                                    "media_count": 111994,
+                                    "name": "China Daily",
+                                    "normal_followers_count": 4160536,
+                                    "pinned_tweet_ids_str": [
+                                      "1700166894593224767"
+                                    ],
+                                    "possibly_sensitive": false,
+                                    "profile_banner_url": "https://pbs.twimg.com/profile_banners/87775422/1683168821",
+                                    "profile_image_url_https": "https://pbs.twimg.com/profile_images/1598185470353022976/-KlKi0WI_normal.jpg",
+                                    "profile_interstitial_type": "",
+                                    "screen_name": "ChinaDaily",
+                                    "statuses_count": 207177,
+                                    "translator_type": "none",
+                                    "verified": false,
+                                    "want_retweets": false,
+                                    "withheld_in_countries": []
+                                  },
+                                  "professional": {
+                                    "rest_id": "1473187902209536003",
+                                    "professional_type": "Creator",
+                                    "category": [
+                                      {
+                                        "id": 580,
+                                        "name": "Media & News Company",
+                                        "icon_name": "IconBriefcaseStroke"
+                                      }
+                                    ]
+                                  }
+                                }
+                              }
+                            },
+                            "edit_control": {
+                              "edit_tweet_ids": [
+                                "1700166894593224767"
+                              ],
+                              "editable_until_msecs": "1694189949000",
+                              "is_edit_eligible": true,
+                              "edits_remaining": "5"
+                            },
+                            "is_translatable": false,
+                            "views": {
+                              "count": "5501",
+                              "state": "EnabledWithCount"
+                            },
+                            "source": "<a href=\"https://studio.twitter.com\" rel=\"nofollow\">Twitter Media Studio</a>",
+                            "legacy": {
+                              "bookmark_count": 2,
+                              "bookmarked": false,
+                              "created_at": "Fri Sep 08 15:19:09 +0000 2023",
+                              "conversation_id_str": "1700166894593224767",
+                              "display_text_range": [
+                                0,
+                                273
+                              ],
+                              "entities": {
+                                "media": [
+                                  {
+                                    "display_url": "pic.twitter.com/f5PPXDfhSJ",
+                                    "expanded_url": "https://twitter.com/ChinaDaily/status/1700166894593224767/video/1",
+                                    "id_str": "1700160457841975296",
+                                    "indices": [
+                                      274,
+                                      297
+                                    ],
+                                    "media_url_https": "https://pbs.twimg.com/media/F5g0SArbcAAwuTG.jpg",
+                                    "type": "photo",
+                                    "url": "https://t.co/f5PPXDfhSJ",
+                                    "features": {},
+                                    "sizes": {
+                                      "large": {
+                                        "h": 1080,
+                                        "w": 1920,
+                                        "resize": "fit"
+                                      },
+                                      "medium": {
+                                        "h": 675,
+                                        "w": 1200,
+                                        "resize": "fit"
+                                      },
+                                      "small": {
+                                        "h": 383,
+                                        "w": 680,
+                                        "resize": "fit"
+                                      },
+                                      "thumb": {
+                                        "h": 150,
+                                        "w": 150,
+                                        "resize": "crop"
+                                      }
+                                    },
+                                    "original_info": {
+                                      "height": 1080,
+                                      "width": 1920
+                                    }
+                                  }
+                                ],
+                                "user_mentions": [
+                                  {
+                                    "id_str": "1139407343790116869",
+                                    "name": "Barrett",
+                                    "screen_name": "BarrettYouTube",
+                                    "indices": [
+                                      132,
+                                      147
+                                    ]
+                                  }
+                                ],
+                                "urls": [],
+                                "hashtags": [
+                                  {
+                                    "indices": [
+                                      46,
+                                      53
+                                    ],
+                                    "text": "Huawei"
+                                  },
+                                  {
+                                    "indices": [
+                                      113,
+                                      127
+                                    ],
+                                    "text": "UnboxingChina"
+                                  }
+                                ],
+                                "symbols": []
+                              },
+                              "extended_entities": {
+                                "media": [
+                                  {
+                                    "display_url": "pic.twitter.com/f5PPXDfhSJ",
+                                    "expanded_url": "https://twitter.com/ChinaDaily/status/1700166894593224767/video/1",
+                                    "id_str": "1700160457841975296",
+                                    "indices": [
+                                      274,
+                                      297
+                                    ],
+                                    "media_key": "13_1700160457841975296",
+                                    "media_url_https": "https://pbs.twimg.com/media/F5g0SArbcAAwuTG.jpg",
+                                    "type": "video",
+                                    "url": "https://t.co/f5PPXDfhSJ",
+                                    "additional_media_info": {
+                                      "title": "",
+                                      "description": "",
+                                      "embeddable": true,
+                                      "monetizable": false
+                                    },
+                                    "mediaStats": {
+                                      "viewCount": 1152
+                                    },
+                                    "ext_media_availability": {
+                                      "status": "Available"
+                                    },
+                                    "features": {},
+                                    "sizes": {
+                                      "large": {
+                                        "h": 1080,
+                                        "w": 1920,
+                                        "resize": "fit"
+                                      },
+                                      "medium": {
+                                        "h": 675,
+                                        "w": 1200,
+                                        "resize": "fit"
+                                      },
+                                      "small": {
+                                        "h": 383,
+                                        "w": 680,
+                                        "resize": "fit"
+                                      },
+                                      "thumb": {
+                                        "h": 150,
+                                        "w": 150,
+                                        "resize": "crop"
+                                      }
+                                    },
+                                    "original_info": {
+                                      "height": 1080,
+                                      "width": 1920
+                                    },
+                                    "video_info": {
+                                      "aspect_ratio": [
+                                        16,
+                                        9
+                                      ],
+                                      "duration_millis": 328283,
+                                      "variants": [
+                                        {
+                                          "bitrate": 2176000,
+                                          "content_type": "video/mp4",
+                                          "url": "https://video.twimg.com/amplify_video/1700160457841975296/vid/avc1/1280x720/LYAV1zSTKyjLuvz8.mp4?tag=14"
+                                        },
+                                        {
+                                          "bitrate": 832000,
+                                          "content_type": "video/mp4",
+                                          "url": "https://video.twimg.com/amplify_video/1700160457841975296/vid/avc1/640x360/C5UVyuvD1gwcUOHR.mp4?tag=14"
+                                        },
+                                        {
+                                          "bitrate": 288000,
+                                          "content_type": "video/mp4",
+                                          "url": "https://video.twimg.com/amplify_video/1700160457841975296/vid/avc1/480x270/5S486vJbzNu5bAoX.mp4?tag=14"
+                                        },
+                                        {
+                                          "content_type": "application/x-mpegURL",
+                                          "url": "https://video.twimg.com/amplify_video/1700160457841975296/pl/YOmM7kKSqpzjtysl.m3u8?tag=14&container=fmp4"
+                                        }
+                                      ]
+                                    }
+                                  }
+                                ]
+                              },
+                              "favorite_count": 41,
+                              "favorited": false,
+                              "full_text": "From satellite connection to gesture control, #Huawei's newly released Mate 60 series offers some cool features. #UnboxingChina and @BarrettYouTube had some fun unboxing the phone together, dived into all its functions, and talked about the future of China's tech industry. https://t.co/f5PPXDfhSJ",
+                              "is_quote_status": false,
+                              "lang": "en",
+                              "possibly_sensitive": false,
+                              "possibly_sensitive_editable": true,
+                              "quote_count": 3,
+                              "reply_count": 9,
+                              "retweet_count": 14,
+                              "retweeted": false,
+                              "user_id_str": "87775422",
+                              "id_str": "1700166894593224767"
+                            },
+                            "quick_promote_eligibility": {
+                              "eligibility": "IneligibleNotProfessional"
+                            }
+                          }
+                        },
+                        "tweetDisplayType": "Tweet"
+                      },
+                      "clientEventInfo": {
+                        "component": "related_tweet",
+                        "details": {
+                          "conversationDetails": {
+                            "conversationSection": "RelatedTweet"
+                          }
+                        }
+                      }
+                    }
+                  }
+                ],
+                "displayType": "Vertical",
+                "header": {
+                  "displayType": "Classic",
+                  "text": "Discover more",
+                  "socialContext": {
+                    "type": "TimelineGeneralContext",
+                    "contextType": "TextOnly",
+                    "text": "Sourced from across Twitter"
+                  },
+                  "sticky": true
+                },
+                "clientEventInfo": {
+                  "component": "related_tweet",
+                  "details": {
+                    "conversationDetails": {
+                      "conversationSection": "RelatedTweet"
+                    }
+                  }
+                }
+              }
+            }
+          ]
+        },
+        {
+          "type": "TimelineTerminateTimeline",
+          "direction": "Top"
+        }
+      ]
+    }
+  }
+}

apps/twscrape/tests/mocked-data/card_audiospace.json

@@ -0,0 +1,433 @@
+{
+  "data": {
+    "threaded_conversation_with_injections_v2": {
+      "instructions": [
+        {
+          "type": "TimelineAddEntries",
+          "entries": [
+            {
+              "entryId": "tweet-1789054061729173804",
+              "sortIndex": "7434317975125602003",
+              "content": {
+                "entryType": "TimelineTimelineItem",
+                "__typename": "TimelineTimelineItem",
+                "itemContent": {
+                  "itemType": "TimelineTweet",
+                  "__typename": "TimelineTweet",
+                  "tweet_results": {
+                    "result": {
+                      "__typename": "Tweet",
+                      "rest_id": "1789054061729173804",
+                      "has_birdwatch_notes": false,
+                      "core": {
+                        "user_results": {
+                          "result": {
+                            "__typename": "User",
+                            "id": "VXNlcjoyMjQ0OTk0OTQ1",
+                            "rest_id": "2244994945",
+                            "affiliates_highlighted_label": {
+                              "label": {
+                                "url": {
+                                  "url": "https://twitter.com/X",
+                                  "urlType": "DeepLink"
+                                },
+                                "badge": {
+                                  "url": "https://pbs.twimg.com/profile_images/1683899100922511378/5lY42eHs_bigger.jpg"
+                                },
+                                "description": "X",
+                                "userLabelType": "BusinessLabel",
+                                "userLabelDisplayType": "Badge"
+                              }
+                            },
+                            "has_graduated_access": true,
+                            "is_blue_verified": true,
+                            "profile_image_shape": "Square",
+                            "legacy": {
+                              "can_dm": true,
+                              "can_media_tag": true,
+                              "created_at": "Sat Dec 14 04:35:55 +0000 2013",
+                              "default_profile": false,
+                              "default_profile_image": false,
+                              "description": "The voice of the X Dev team and your official source for updates, news, and events, related to the X API.",
+                              "entities": {
+                                "description": {
+                                  "urls": []
+                                },
+                                "url": {
+                                  "urls": [
+                                    {
+                                      "display_url": "developer.x.com",
+                                      "expanded_url": "https://developer.x.com/",
+                                      "url": "https://t.co/O13IfbuPqq",
+                                      "indices": [
+                                        0,
+                                        23
+                                      ]
+                                    }
+                                  ]
+                                }
+                              },
+                              "fast_followers_count": 0,
+                              "favourites_count": 2148,
+                              "followers_count": 646225,
+                              "friends_count": 1777,
+                              "has_custom_timelines": true,
+                              "is_translator": false,
+                              "listed_count": 2684,
+                              "location": "127.0.0.1",
+                              "media_count": 820,
+                              "name": "Developers",
+                              "normal_followers_count": 646225,
+                              "pinned_tweet_ids_str": [
+                                "1770153912013615285"
+                              ],
+                              "possibly_sensitive": false,
+                              "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1690213128",
+                              "profile_image_url_https": "https://pbs.twimg.com/profile_images/1683501992314798080/xl1POYLw_normal.jpg",
+                              "profile_interstitial_type": "",
+                              "screen_name": "XDevelopers",
+                              "statuses_count": 4089,
+                              "translator_type": "regular",
+                              "url": "https://t.co/O13IfbuPqq",
+                              "verified": false,
+                              "verified_type": "Business",
+                              "want_retweets": false,
+                              "withheld_in_countries": []
+                            },
+                            "professional": {
+                              "rest_id": "1516891231749517312",
+                              "professional_type": "Business",
+                              "category": [
+                                {
+                                  "id": 1009,
+                                  "name": "Community",
+                                  "icon_name": "IconBriefcaseStroke"
+                                }
+                              ]
+                            },
+                            "tipjar_settings": {}
+                          }
+                        }
+                      },
+                      "card": {
+                        "rest_id": "https://t.co/7ajX0RPmaj",
+                        "legacy": {
+                          "binding_values": [
+                            {
+                              "key": "narrow_cast_space_type",
+                              "value": {
+                                "string_value": "0",
+                                "type": "STRING"
+                              }
+                            },
+                            {
+                              "key": "id",
+                              "value": {
+                                "string_value": "1vOxwjaWEbdJB",
+                                "type": "STRING"
+                              }
+                            },
+                            {
+                              "key": "card_url",
+                              "value": {
+                                "scribe_key": "card_url",
+                                "string_value": "https://t.co/7ajX0RPmaj",
+                                "type": "STRING"
+                              }
+                            }
+                          ],
+                          "card_platform": {
+                            "platform": {
+                              "audience": {
+                                "name": "production"
+                              },
+                              "device": {
+                                "name": "Swift",
+                                "version": "12"
+                              }
+                            }
+                          },
+                          "name": "3691233323:audiospace",
+                          "url": "https://t.co/7ajX0RPmaj",
+                          "user_refs_results": []
+                        }
+                      },
+                      "unmention_data": {},
+                      "edit_control": {
+                        "edit_tweet_ids": [
+                          "1789054061729173804"
+                        ],
+                        "editable_until_msecs": "1715382301990",
+                        "is_edit_eligible": false,
+                        "edits_remaining": "5"
+                      },
+                      "is_translatable": false,
+                      "views": {
+                        "state": "Enabled"
+                      },
+                      "source": "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>",
+                      "legacy": {
+                        "bookmark_count": 0,
+                        "bookmarked": false,
+                        "created_at": "Fri May 10 22:05:01 +0000 2024",
+                        "conversation_id_str": "1789054061729173804",
+                        "display_text_range": [
+                          0,
+                          140
+                        ],
+                        "entities": {
+                          "hashtags": [],
+                          "symbols": [],
+                          "timestamps": [],
+                          "urls": [],
+                          "user_mentions": [
+                            {
+                              "id_str": "857699969263964161",
+                              "name": "Suhem Parack",
+                              "screen_name": "suhemparack",
+                              "indices": [
+                                3,
+                                15
+                              ]
+                            },
+                            {
+                              "id_str": "2244994945",
+                              "name": "Developers",
+                              "screen_name": "XDevelopers",
+                              "indices": [
+                                109,
+                                121
+                              ]
+                            }
+                          ]
+                        },
+                        "favorite_count": 0,
+                        "favorited": false,
+                        "full_text": "RT @suhemparack: If you have an app that posts emergency services updates to X using the API, please join us @XDevelopers next week to lear…",
+                        "is_quote_status": false,
+                        "lang": "en",
+                        "quote_count": 0,
+                        "reply_count": 0,
+                        "retweet_count": 26,
+                        "retweeted": false,
+                        "user_id_str": "2244994945",
+                        "id_str": "1789054061729173804",
+                        "retweeted_status_result": {
+                          "result": {
+                            "__typename": "Tweet",
+                            "rest_id": "1789053970587271232",
+                            "has_birdwatch_notes": false,
+                            "core": {
+                              "user_results": {
+                                "result": {
+                                  "__typename": "User",
+                                  "id": "VXNlcjo4NTc2OTk5NjkyNjM5NjQxNjE=",
+                                  "rest_id": "857699969263964161",
+                                  "affiliates_highlighted_label": {
+                                    "label": {
+                                      "url": {
+                                        "url": "https://twitter.com/X",
+                                        "urlType": "DeepLink"
+                                      },
+                                      "badge": {
+                                        "url": "https://pbs.twimg.com/profile_images/1683899100922511378/5lY42eHs_bigger.jpg"
+                                      },
+                                      "description": "X",
+                                      "userLabelType": "BusinessLabel",
+                                      "userLabelDisplayType": "Badge"
+                                    }
+                                  },
+                                  "has_graduated_access": true,
+                                  "is_blue_verified": true,
+                                  "profile_image_shape": "Circle",
+                                  "legacy": {
+                                    "can_dm": true,
+                                    "can_media_tag": false,
+                                    "created_at": "Thu Apr 27 20:56:22 +0000 2017",
+                                    "default_profile": true,
+                                    "default_profile_image": false,
+                                    "description": "Partner Engineering @ 𝕏 Opinions my own. RTs != endorsements etc",
+                                    "entities": {
+                                      "description": {
+                                        "urls": []
+                                      },
+                                      "url": {
+                                        "urls": [
+                                          {
+                                            "display_url": "developer.x.com",
+                                            "expanded_url": "https://developer.x.com",
+                                            "url": "https://t.co/Rh0kWC6xS8",
+                                            "indices": [
+                                              0,
+                                              23
+                                            ]
+                                          }
+                                        ]
+                                      }
+                                    },
+                                    "fast_followers_count": 0,
+                                    "favourites_count": 141,
+                                    "followers_count": 4788,
+                                    "friends_count": 1577,
+                                    "has_custom_timelines": true,
+                                    "is_translator": false,
+                                    "listed_count": 67,
+                                    "location": "San Francisco, CA",
+                                    "media_count": 134,
+                                    "name": "Suhem Parack",
+                                    "normal_followers_count": 4788,
+                                    "pinned_tweet_ids_str": [
+                                      "1789053970587271232"
+                                    ],
+                                    "possibly_sensitive": false,
+                                    "profile_banner_url": "https://pbs.twimg.com/profile_banners/857699969263964161/1712432865",
+                                    "profile_image_url_https": "https://pbs.twimg.com/profile_images/1795573096063397892/HB6ShS6B_normal.jpg",
+                                    "profile_interstitial_type": "",
+                                    "screen_name": "suhemparack",
+                                    "statuses_count": 2214,
+                                    "translator_type": "none",
+                                    "url": "https://t.co/Rh0kWC6xS8",
+                                    "verified": false,
+                                    "want_retweets": false,
+                                    "withheld_in_countries": []
+                                  },
+                                  "professional": {
+                                    "rest_id": "1471588288142057479",
+                                    "professional_type": "Creator",
+                                    "category": []
+                                  },
+                                  "tipjar_settings": {}
+                                }
+                              }
+                            },
+                            "card": {
+                              "rest_id": "https://t.co/7ajX0RPmaj",
+                              "legacy": {
+                                "binding_values": [
+                                  {
+                                    "key": "narrow_cast_space_type",
+                                    "value": {
+                                      "string_value": "0",
+                                      "type": "STRING"
+                                    }
+                                  },
+                                  {
+                                    "key": "id",
+                                    "value": {
+                                      "string_value": "1vOxwjaWEbdJB",
+                                      "type": "STRING"
+                                    }
+                                  },
+                                  {
+                                    "key": "card_url",
+                                    "value": {
+                                      "scribe_key": "card_url",
+                                      "string_value": "https://t.co/7ajX0RPmaj",
+                                      "type": "STRING"
+                                    }
+                                  }
+                                ],
+                                "card_platform": {
+                                  "platform": {
+                                    "audience": {
+                                      "name": "production"
+                                    },
+                                    "device": {
+                                      "name": "Swift",
+                                      "version": "12"
+                                    }
+                                  }
+                                },
+                                "name": "3691233323:audiospace",
+                                "url": "https://t.co/7ajX0RPmaj",
+                                "user_refs_results": []
+                              }
+                            },
+                            "unmention_data": {},
+                            "edit_control": {
+                              "edit_tweet_ids": [
+                                "1789053970587271232"
+                              ],
+                              "editable_until_msecs": "1715382280000",
+                              "is_edit_eligible": false,
+                              "edits_remaining": "5"
+                            },
+                            "is_translatable": false,
+                            "views": {
+                              "count": "121718",
+                              "state": "EnabledWithCount"
+                            },
+                            "source": "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>",
+                            "legacy": {
+                              "bookmark_count": 9,
+                              "bookmarked": false,
+                              "created_at": "Fri May 10 22:04:40 +0000 2024",
+                              "conversation_id_str": "1789053970587271232",
+                              "display_text_range": [
+                                0,
+                                200
+                              ],
+                              "entities": {
+                                "hashtags": [],
+                                "symbols": [],
+                                "timestamps": [],
+                                "urls": [
+                                  {
+                                    "display_url": "x.com/i/spaces/1voxw…",
+                                    "expanded_url": "https://twitter.com/i/spaces/1vOxwjaWEbdJB",
+                                    "url": "https://t.co/7ajX0RPmaj",
+                                    "indices": [
+                                      177,
+                                      200
+                                    ]
+                                  }
+                                ],
+                                "user_mentions": [
+                                  {
+                                    "id_str": "2244994945",
+                                    "name": "Developers",
+                                    "screen_name": "XDevelopers",
+                                    "indices": [
+                                      92,
+                                      104
+                                    ]
+                                  }
+                                ]
+                              },
+                              "favorite_count": 108,
+                              "favorited": false,
+                              "full_text": "If you have an app that posts emergency services updates to X using the API, please join us @XDevelopers next week to learn how you can quickly migrate your app to the X API v2 https://t.co/7ajX0RPmaj",
+                              "is_quote_status": false,
+                              "lang": "en",
+                              "possibly_sensitive": false,
+                              "possibly_sensitive_editable": true,
+                              "quote_count": 5,
+                              "reply_count": 16,
+                              "retweet_count": 26,
+                              "retweeted": false,
+                              "user_id_str": "857699969263964161",
+                              "id_str": "1789053970587271232"
+                            }
+                          }
+                        }
+                      },
+                      "quick_promote_eligibility": {
+                        "eligibility": "IneligibleNotProfessional"
+                      }
+                    }
+                  },
+                  "tweetDisplayType": "Tweet",
+                  "hasModeratedReplies": false
+                }
+              }
+            }
+          ]
+        },
+        {
+          "type": "TimelineTerminateTimeline",
+          "direction": "Top"
+        }
+      ]
+    }
+  }
+}

apps/twscrape/tests/mocked-data/raw_trends.json

@@ -0,0 +1,564 @@
+{
+  "data": {
+    "timeline": {
+      "timeline": {
+        "instructions": [
+          {
+            "type": "TimelineClearCache"
+          },
+          {
+            "type": "TimelineTerminateTimeline",
+            "direction": "Bottom"
+          },
+          {
+            "type": "TimelineAddEntries",
+            "entries": [
+              {
+                "entryId": "trend-Hield",
+                "sortIndex": "1897838079637454848",
+                "content": {
+                  "entryType": "TimelineTimelineItem",
+                  "__typename": "TimelineTimelineItem",
+                  "itemContent": {
+                    "itemType": "TimelineTrend",
+                    "__typename": "TimelineTrend",
+                    "name": "Hield",
+                    "trend_url": {
+                      "url": "twitter://search/?query=Hield&src=trend_click&pc=true&vertical=trends",
+                      "urlType": "DeepLink",
+                      "urtEndpointOptions": {
+                        "requestParams": [
+                          {
+                            "key": "cd",
+                            "value": "HBgFSGllbGQAAA=="
+                          }
+                        ]
+                      }
+                    },
+                    "trend_metadata": {
+                      "domain_context": "Trending in Sports",
+                      "meta_description": "1,699 posts",
+                      "url": {
+                        "url": "twitter://search/?query=Hield&src=trend_click&pc=true&vertical=trends",
+                        "urlType": "DeepLink",
+                        "urtEndpointOptions": {
+                          "requestParams": [
+                            {
+                              "key": "cd",
+                              "value": "HBgFSGllbGQAAA=="
+                            }
+                          ]
+                        }
+                      }
+                    }
+                  },
+                  "feedbackInfo": {
+                    "clientEventInfo": {
+                      "action": "click",
+                      "component": "trends",
+                      "element": "feedback"
+                    },
+                    "feedbackKeys": [
+                      "-1336659320",
+                      "1628041751",
+                      "15718569",
+                      "-619311538",
+                      "-1457103155",
+                      "2103484841"
+                    ],
+                    "feedbackMetadata": "DwABCwAAAAEAAAAFSGllbGQGAAQAAQgABgAAAAMA"
+                  },
+                  "clientEventInfo": {
+                    "component": "unified_events",
+                    "element": "trend",
+                    "details": {
+                      "timelinesDetails": {
+                        "controllerData": "DAACDAAQDAABCgABo8eUwO2g7DYAAAAA"
+                      }
+                    }
+                  }
+                }
+              },
+              {
+                "entryId": "trend-Marchand",
+                "sortIndex": "1897838079637454847",
+                "content": {
+                  "entryType": "TimelineTimelineItem",
+                  "__typename": "TimelineTimelineItem",
+                  "itemContent": {
+                    "itemType": "TimelineTrend",
+                    "__typename": "TimelineTrend",
+                    "name": "Marchand",
+                    "trend_url": {
+                      "url": "twitter://search/?query=Marchand&src=trend_click&pc=true&vertical=trends",
+                      "urlType": "DeepLink",
+                      "urtEndpointOptions": {
+                        "requestParams": [
+                          {
+                            "key": "cd",
+                            "value": "HBgITWFyY2hhbmQAAA=="
+                          }
+                        ]
+                      }
+                    },
+                    "trend_metadata": {
+                      "domain_context": "Trending in Sports",
+                      "meta_description": "3,157 posts",
+                      "url": {
+                        "url": "twitter://search/?query=Marchand&src=trend_click&pc=true&vertical=trends",
+                        "urlType": "DeepLink",
+                        "urtEndpointOptions": {
+                          "requestParams": [
+                            {
+                              "key": "cd",
+                              "value": "HBgITWFyY2hhbmQAAA=="
+                            }
+                          ]
+                        }
+                      }
+                    }
+                  },
+                  "feedbackInfo": {
+                    "clientEventInfo": {
+                      "action": "click",
+                      "component": "trends",
+                      "element": "feedback"
+                    },
+                    "feedbackKeys": [
+                      "-1336659320",
+                      "1628041751",
+                      "15718569",
+                      "-619311538",
+                      "-1457103155",
+                      "2103484841"
+                    ],
+                    "feedbackMetadata": "DwABCwAAAAEAAAAITWFyY2hhbmQGAAQAAggABgAAAAMA"
+                  },
+                  "clientEventInfo": {
+                    "component": "unified_events",
+                    "element": "trend",
+                    "details": {
+                      "timelinesDetails": {
+                        "controllerData": "DAACDAAQDAABCgABo8eUwO2g7DYAAAAA"
+                      }
+                    }
+                  }
+                }
+              },
+              {
+                "entryId": "trend-Draymond",
+                "sortIndex": "1897838079637454846",
+                "content": {
+                  "entryType": "TimelineTimelineItem",
+                  "__typename": "TimelineTimelineItem",
+                  "itemContent": {
+                    "itemType": "TimelineTrend",
+                    "__typename": "TimelineTrend",
+                    "name": "Draymond",
+                    "trend_url": {
+                      "url": "twitter://search/?query=Draymond&src=trend_click&pc=true&vertical=trends",
+                      "urlType": "DeepLink",
+                      "urtEndpointOptions": {
+                        "requestParams": [
+                          {
+                            "key": "cd",
+                            "value": "HBgIRHJheW1vbmQAAA=="
+                          }
+                        ]
+                      }
+                    },
+                    "trend_metadata": {
+                      "domain_context": "Trending in Sports",
+                      "meta_description": "9,409 posts",
+                      "url": {
+                        "url": "twitter://search/?query=Draymond&src=trend_click&pc=true&vertical=trends",
+                        "urlType": "DeepLink",
+                        "urtEndpointOptions": {
+                          "requestParams": [
+                            {
+                              "key": "cd",
+                              "value": "HBgIRHJheW1vbmQAAA=="
+                            }
+                          ]
+                        }
+                      }
+                    }
+                  },
+                  "feedbackInfo": {
+                    "clientEventInfo": {
+                      "action": "click",
+                      "component": "trends",
+                      "element": "feedback"
+                    },
+                    "feedbackKeys": [
+                      "-1336659320",
+                      "1628041751",
+                      "15718569",
+                      "-619311538",
+                      "-1457103155",
+                      "2103484841"
+                    ],
+                    "feedbackMetadata": "DwABCwAAAAEAAAAIRHJheW1vbmQGAAQAAwgABgAAAAMA"
+                  },
+                  "clientEventInfo": {
+                    "component": "unified_events",
+                    "element": "trend",
+                    "details": {
+                      "timelinesDetails": {
+                        "controllerData": "DAACDAAQDAABCgABo8eUwO2g7DYAAAAA"
+                      }
+                    }
+                  }
+                }
+              },
+              {
+                "entryId": "trend-Shaq",
+                "sortIndex": "1897838079637454845",
+                "content": {
+                  "entryType": "TimelineTimelineItem",
+                  "__typename": "TimelineTimelineItem",
+                  "itemContent": {
+                    "itemType": "TimelineTrend",
+                    "__typename": "TimelineTrend",
+                    "name": "Shaq",
+                    "trend_url": {
+                      "url": "twitter://search/?query=Shaq&src=trend_click&pc=true&vertical=trends",
+                      "urlType": "DeepLink",
+                      "urtEndpointOptions": {
+                        "requestParams": [
+                          {
+                            "key": "cd",
+                            "value": "HBgEU2hhcQAA"
+                          }
+                        ]
+                      }
+                    },
+                    "trend_metadata": {
+                      "domain_context": "Trending in Sports",
+                      "meta_description": "12K posts",
+                      "url": {
+                        "url": "twitter://search/?query=Shaq&src=trend_click&pc=true&vertical=trends",
+                        "urlType": "DeepLink",
+                        "urtEndpointOptions": {
+                          "requestParams": [
+                            {
+                              "key": "cd",
+                              "value": "HBgEU2hhcQAA"
+                            }
+                          ]
+                        }
+                      }
+                    }
+                  },
+                  "feedbackInfo": {
+                    "clientEventInfo": {
+                      "action": "click",
+                      "component": "trends",
+                      "element": "feedback"
+                    },
+                    "feedbackKeys": [
+                      "-1336659320",
+                      "1628041751",
+                      "15718569",
+                      "-619311538",
+                      "-1457103155",
+                      "2103484841"
+                    ],
+                    "feedbackMetadata": "DwABCwAAAAEAAAAEU2hhcQYABAAECAAGAAAAAwA="
+                  },
+                  "clientEventInfo": {
+                    "component": "unified_events",
+                    "element": "trend",
+                    "details": {
+                      "timelinesDetails": {
+                        "controllerData": "DAACDAAQDAABCgABo8eUwO2g7DYAAAAA"
+                      }
+                    }
+                  }
+                }
+              },
+              {
+                "entryId": "trend-Chuck",
+                "sortIndex": "1897838079637454844",
+                "content": {
+                  "entryType": "TimelineTimelineItem",
+                  "__typename": "TimelineTimelineItem",
+                  "itemContent": {
+                    "itemType": "TimelineTrend",
+                    "__typename": "TimelineTrend",
+                    "name": "Chuck",
+                    "trend_url": {
+                      "url": "twitter://search/?query=Chuck&src=trend_click&pc=true&vertical=trends",
+                      "urlType": "DeepLink",
+                      "urtEndpointOptions": {
+                        "requestParams": [
+                          {
+                            "key": "cd",
+                            "value": "HBgFQ2h1Y2sAAA=="
+                          }
+                        ]
+                      }
+                    },
+                    "trend_metadata": {
+                      "domain_context": "Trending in Sports",
+                      "meta_description": "37.6K posts",
+                      "url": {
+                        "url": "twitter://search/?query=Chuck&src=trend_click&pc=true&vertical=trends",
+                        "urlType": "DeepLink",
+                        "urtEndpointOptions": {
+                          "requestParams": [
+                            {
+                              "key": "cd",
+                              "value": "HBgFQ2h1Y2sAAA=="
+                            }
+                          ]
+                        }
+                      }
+                    }
+                  },
+                  "feedbackInfo": {
+                    "clientEventInfo": {
+                      "action": "click",
+                      "component": "trends",
+                      "element": "feedback"
+                    },
+                    "feedbackKeys": [
+                      "-1336659320",
+                      "1628041751",
+                      "15718569",
+                      "-619311538",
+                      "-1457103155",
+                      "2103484841"
+                    ],
+                    "feedbackMetadata": "DwABCwAAAAEAAAAFQ2h1Y2sGAAQABQgABgAAAAMA"
+                  },
+                  "clientEventInfo": {
+                    "component": "unified_events",
+                    "element": "trend",
+                    "details": {
+                      "timelinesDetails": {
+                        "controllerData": "DAACDAAQDAABCgABo8eUwO2g7DYAAAAA"
+                      }
+                    }
+                  }
+                }
+              },
+              {
+                "entryId": "trend-Nets",
+                "sortIndex": "1897838079637454843",
+                "content": {
+                  "entryType": "TimelineTimelineItem",
+                  "__typename": "TimelineTimelineItem",
+                  "itemContent": {
+                    "itemType": "TimelineTrend",
+                    "__typename": "TimelineTrend",
+                    "name": "Nets",
+                    "trend_url": {
+                      "url": "twitter://search/?query=Nets&src=trend_click&pc=true&vertical=trends",
+                      "urlType": "DeepLink",
+                      "urtEndpointOptions": {
+                        "requestParams": [
+                          {
+                            "key": "cd",
+                            "value": "HBgETmV0cwAA"
+                          }
+                        ]
+                      }
+                    },
+                    "trend_metadata": {
+                      "domain_context": "Trending in Sports",
+                      "meta_description": "12K posts",
+                      "url": {
+                        "url": "twitter://search/?query=Nets&src=trend_click&pc=true&vertical=trends",
+                        "urlType": "DeepLink",
+                        "urtEndpointOptions": {
+                          "requestParams": [
+                            {
+                              "key": "cd",
+                              "value": "HBgETmV0cwAA"
+                            }
+                          ]
+                        }
+                      }
+                    }
+                  },
+                  "feedbackInfo": {
+                    "clientEventInfo": {
+                      "action": "click",
+                      "component": "trends",
+                      "element": "feedback"
+                    },
+                    "feedbackKeys": [
+                      "-1336659320",
+                      "1628041751",
+                      "15718569",
+                      "-619311538",
+                      "-1457103155",
+                      "2103484841"
+                    ],
+                    "feedbackMetadata": "DwABCwAAAAEAAAAETmV0cwYABAAGCAAGAAAAAwA="
+                  },
+                  "clientEventInfo": {
+                    "component": "unified_events",
+                    "element": "trend",
+                    "details": {
+                      "timelinesDetails": {
+                        "controllerData": "DAACDAAQDAABCgABo8eUwO2g7DYAAAAA"
+                      }
+                    }
+                  }
+                }
+              },
+              {
+                "entryId": "trend-Bengals",
+                "sortIndex": "1897838079637454842",
+                "content": {
+                  "entryType": "TimelineTimelineItem",
+                  "__typename": "TimelineTimelineItem",
+                  "itemContent": {
+                    "itemType": "TimelineTrend",
+                    "__typename": "TimelineTrend",
+                    "name": "Bengals",
+                    "trend_url": {
+                      "url": "twitter://search/?query=Bengals&src=trend_click&pc=true&vertical=trends",
+                      "urlType": "DeepLink",
+                      "urtEndpointOptions": {
+                        "requestParams": [
+                          {
+                            "key": "cd",
+                            "value": "HBgHQmVuZ2FscwAA"
+                          }
+                        ]
+                      }
+                    },
+                    "trend_metadata": {
+                      "domain_context": "Trending in Sports",
+                      "meta_description": "21.9K posts",
+                      "url": {
+                        "url": "twitter://search/?query=Bengals&src=trend_click&pc=true&vertical=trends",
+                        "urlType": "DeepLink",
+                        "urtEndpointOptions": {
+                          "requestParams": [
+                            {
+                              "key": "cd",
+                              "value": "HBgHQmVuZ2FscwAA"
+                            }
+                          ]
+                        }
+                      }
+                    }
+                  },
+                  "feedbackInfo": {
+                    "clientEventInfo": {
+                      "action": "click",
+                      "component": "trends",
+                      "element": "feedback"
+                    },
+                    "feedbackKeys": [
+                      "-1336659320",
+                      "1628041751",
+                      "15718569",
+                      "-619311538",
+                      "-1457103155",
+                      "2103484841"
+                    ],
+                    "feedbackMetadata": "DwABCwAAAAEAAAAHQmVuZ2FscwYABAAHCAAGAAAAAwA="
+                  },
+                  "clientEventInfo": {
+                    "component": "unified_events",
+                    "element": "trend",
+                    "details": {
+                      "timelinesDetails": {
+                        "controllerData": "DAACDAAQDAABCgABo8eUwO2g7DYAAAAA"
+                      }
+                    }
+                  }
+                }
+              },
+              {
+                "entryId": "cursor-top-1897838079637454849",
+                "sortIndex": "1897838079637454849",
+                "content": {
+                  "entryType": "TimelineTimelineCursor",
+                  "__typename": "TimelineTimelineCursor",
+                  "value": "DAAJAAA",
+                  "cursorType": "Top"
+                }
+              }
+            ]
+          }
+        ],
+        "responseObjects": {
+          "feedbackActions": [
+            {
+              "key": "-619311538",
+              "value": {
+                "feedbackType": "SeeFewer",
+                "prompt": "Not interested in this",
+                "confirmation": "Thanks. Refresh this page to update these trends.",
+                "feedbackUrl": "/2/guide/insert_feedback/trend_not_interested_in_this_feedback_key.json",
+                "hasUndoAction": false,
+                "confirmationDisplayType": "Inline"
+              }
+            },
+            {
+              "key": "15718569",
+              "value": {
+                "feedbackType": "SeeFewer",
+                "prompt": "This trend is abusive or harmful",
+                "confirmation": "Thanks. Refresh this page to update these trends.",
+                "feedbackUrl": "/2/guide/insert_feedback/trend_abusive_feedback_key.json",
+                "hasUndoAction": false,
+                "confirmationDisplayType": "Inline"
+              }
+            },
+            {
+              "key": "-1457103155",
+              "value": {
+                "feedbackType": "SeeFewer",
+                "prompt": "This trend is a duplicate",
+                "confirmation": "Thanks. Refresh this page to update these trends.",
+                "feedbackUrl": "/2/guide/insert_feedback/trend_duplicate_feedback_key.json",
+                "hasUndoAction": false,
+                "confirmationDisplayType": "Inline"
+              }
+            },
+            {
+              "key": "-1336659320",
+              "value": {
+                "feedbackType": "SeeFewer",
+                "prompt": "The associated content is not relevant",
+                "confirmation": "Thanks. Refresh this page to update these trends.",
+                "feedbackUrl": "/2/guide/insert_feedback/trend_irrelevant_content_feedback_key.json",
+                "hasUndoAction": false,
+                "confirmationDisplayType": "BottomSheet"
+              }
+            },
+            {
+              "key": "1628041751",
+              "value": {
+                "feedbackType": "SeeFewer",
+                "prompt": "This trend is spam",
+                "confirmation": "Thanks. Refresh this page to update these trends.",
+                "feedbackUrl": "/2/guide/insert_feedback/trend_spam_feedback_key.json",
+                "hasUndoAction": false,
+                "confirmationDisplayType": "Inline"
+              }
+            },
+            {
+              "key": "2103484841",
+              "value": {
+                "feedbackType": "SeeFewer",
+                "prompt": "This trend is harmful or spammy",
+                "confirmation": "Thanks. Refresh this page to update these trends.",
+                "feedbackUrl": "/2/guide/insert_feedback/trend_abusive_or_harmful_feedback_key.json",
+                "hasUndoAction": false,
+                "confirmationDisplayType": "BottomSheet"
+              }
+            }
+          ]
+        }
+      }
+    }
+  }
+}

apps/twscrape/tests/mocked-data/raw_user_by_id.json

@@ -0,0 +1,100 @@
+{
+  "data": {
+    "user": {
+      "result": {
+        "__typename": "User",
+        "id": "VXNlcjoyMjQ0OTk0OTQ1",
+        "rest_id": "2244994945",
+        "affiliates_highlighted_label": {
+          "label": {
+            "url": {
+              "url": "https://twitter.com/X",
+              "urlType": "DeepLink"
+            },
+            "badge": {
+              "url": "https://pbs.twimg.com/profile_images/1683899100922511378/5lY42eHs_bigger.jpg"
+            },
+            "description": "X",
+            "userLabelType": "BusinessLabel",
+            "userLabelDisplayType": "Badge"
+          }
+        },
+        "has_graduated_access": true,
+        "is_blue_verified": true,
+        "profile_image_shape": "Square",
+        "legacy": {
+          "following": false,
+          "can_dm": true,
+          "can_media_tag": true,
+          "created_at": "Sat Dec 14 04:35:55 +0000 2013",
+          "default_profile": false,
+          "default_profile_image": false,
+          "description": "The voice of the X Dev team and your official source for updates, news, and events, related to the X API.",
+          "entities": {
+            "description": {
+              "urls": []
+            },
+            "url": {
+              "urls": [
+                {
+                  "display_url": "docs.x.com",
+                  "expanded_url": "https://docs.x.com/",
+                  "url": "https://t.co/NIwHVIVyrN",
+                  "indices": [
+                    0,
+                    23
+                  ]
+                }
+              ]
+            }
+          },
+          "fast_followers_count": 0,
+          "favourites_count": 2167,
+          "followers_count": 700242,
+          "friends_count": 746,
+          "has_custom_timelines": true,
+          "is_translator": false,
+          "listed_count": 2682,
+          "location": "127.0.0.1",
+          "media_count": 821,
+          "name": "Developers",
+          "normal_followers_count": 700242,
+          "pinned_tweet_ids_str": [
+            "1851463451929186504"
+          ],
+          "possibly_sensitive": false,
+          "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1690213128",
+          "profile_image_url_https": "https://pbs.twimg.com/profile_images/1683501992314798080/xl1POYLw_normal.jpg",
+          "profile_interstitial_type": "",
+          "screen_name": "XDevelopers",
+          "statuses_count": 4079,
+          "translator_type": "regular",
+          "url": "https://t.co/NIwHVIVyrN",
+          "verified": false,
+          "verified_type": "Business",
+          "want_retweets": false,
+          "withheld_in_countries": []
+        },
+        "professional": {
+          "rest_id": "1516891231749517312",
+          "professional_type": "Business",
+          "category": [
+            {
+              "id": 1009,
+              "name": "Community",
+              "icon_name": "IconBriefcaseStroke"
+            }
+          ]
+        },
+        "tipjar_settings": {},
+        "business_account": {},
+        "highlights_info": {
+          "can_highlight_tweets": true,
+          "highlighted_tweets": "2"
+        },
+        "creator_subscriptions_count": 0,
+        "has_hidden_subscriptions_on_profile": false
+      }
+    }
+  }
+}

apps/twscrape/tests/mocked-data/raw_user_by_login.json

@@ -0,0 +1,128 @@
+{
+  "data": {
+    "user": {
+      "result": {
+        "__typename": "User",
+        "id": "VXNlcjoyMjQ0OTk0OTQ1",
+        "rest_id": "2244994945",
+        "affiliates_highlighted_label": {
+          "label": {
+            "url": {
+              "url": "https://twitter.com/X",
+              "urlType": "DeepLink"
+            },
+            "badge": {
+              "url": "https://pbs.twimg.com/profile_images/1683899100922511378/5lY42eHs_bigger.jpg"
+            },
+            "description": "X",
+            "userLabelType": "BusinessLabel",
+            "userLabelDisplayType": "Badge"
+          }
+        },
+        "has_graduated_access": true,
+        "is_blue_verified": true,
+        "profile_image_shape": "Square",
+        "legacy": {
+          "following": false,
+          "can_dm": true,
+          "can_media_tag": true,
+          "created_at": "Sat Dec 14 04:35:55 +0000 2013",
+          "default_profile": false,
+          "default_profile_image": false,
+          "description": "The voice of the X Dev team and your official source for updates, news, and events, related to the X API.",
+          "entities": {
+            "description": {
+              "urls": []
+            },
+            "url": {
+              "urls": [
+                {
+                  "display_url": "docs.x.com",
+                  "expanded_url": "https://docs.x.com/",
+                  "url": "https://t.co/NIwHVIVyrN",
+                  "indices": [
+                    0,
+                    23
+                  ]
+                }
+              ]
+            }
+          },
+          "fast_followers_count": 0,
+          "favourites_count": 2167,
+          "followers_count": 700242,
+          "friends_count": 746,
+          "has_custom_timelines": true,
+          "is_translator": false,
+          "listed_count": 2682,
+          "location": "127.0.0.1",
+          "media_count": 821,
+          "name": "Developers",
+          "normal_followers_count": 700242,
+          "pinned_tweet_ids_str": [
+            "1851463451929186504"
+          ],
+          "possibly_sensitive": false,
+          "profile_banner_url": "https://pbs.twimg.com/profile_banners/2244994945/1690213128",
+          "profile_image_url_https": "https://pbs.twimg.com/profile_images/1683501992314798080/xl1POYLw_normal.jpg",
+          "profile_interstitial_type": "",
+          "screen_name": "XDevelopers",
+          "statuses_count": 4079,
+          "translator_type": "regular",
+          "url": "https://t.co/NIwHVIVyrN",
+          "verified": false,
+          "verified_type": "Business",
+          "want_retweets": false,
+          "withheld_in_countries": []
+        },
+        "professional": {
+          "rest_id": "1516891231749517312",
+          "professional_type": "Business",
+          "category": [
+            {
+              "id": 1009,
+              "name": "Community",
+              "icon_name": "IconBriefcaseStroke"
+            }
+          ]
+        },
+        "tipjar_settings": {},
+        "legacy_extended_profile": {},
+        "is_profile_translatable": false,
+        "has_hidden_subscriptions_on_profile": false,
+        "verification_info": {
+          "reason": {
+            "description": {
+              "text": "This account is verified because it's an affiliate of @X on X. Learn more",
+              "entities": [
+                {
+                  "from_index": 54,
+                  "to_index": 56,
+                  "ref": {
+                    "url": "https://twitter.com/X",
+                    "url_type": "ExternalUrl"
+                  }
+                },
+                {
+                  "from_index": 63,
+                  "to_index": 73,
+                  "ref": {
+                    "url": "https://help.twitter.com/en/rules-and-policies/profile-labels",
+                    "url_type": "ExternalUrl"
+                  }
+                }
+              ]
+            },
+            "verified_since_msec": "1395425700472"
+          }
+        },
+        "highlights_info": {
+          "can_highlight_tweets": true,
+          "highlighted_tweets": "2"
+        },
+        "business_account": {},
+        "creator_subscriptions_count": 0
+      }
+    }
+  }
+}

apps/twscrape/tests/test_api.py

@@ -0,0 +1,61 @@
+import os
+
+import pytest
+
+from twscrape.accounts_pool import NoAccountError
+from twscrape.api import API
+from twscrape.utils import gather, get_env_bool
+
+
+class MockedError(Exception):
+    pass
+
+
+GQL_GEN = [
+    "search",
+    "tweet_replies",
+    "retweeters",
+    "followers",
+    "following",
+    "user_tweets",
+    "user_tweets_and_replies",
+    "list_timeline",
+    "trends",
+]
+
+
+async def test_gql_params(api_mock: API, monkeypatch):
+    for func in GQL_GEN:
+        args = []
+
+        def mock_gql_items(*a, **kw):
+            args.append((a, kw))
+            raise MockedError()
+
+        try:
+            monkeypatch.setattr(api_mock, "_gql_items", mock_gql_items)
+            await gather(getattr(api_mock, func)("user1", limit=100, kv={"count": 100}))
+        except MockedError:
+            pass
+
+        assert len(args) == 1, f"{func} not called once"
+        assert args[0][1]["limit"] == 100, f"limit not changed in {func}"
+        assert args[0][0][1]["count"] == 100, f"count not changed in {func}"
+
+
+async def test_raise_when_no_account(api_mock: API):
+    await api_mock.pool.delete_accounts(["user1"])
+    assert len(await api_mock.pool.get_all()) == 0
+
+    assert get_env_bool("TWS_RAISE_WHEN_NO_ACCOUNT") is False
+    os.environ["TWS_RAISE_WHEN_NO_ACCOUNT"] = "1"
+    assert get_env_bool("TWS_RAISE_WHEN_NO_ACCOUNT") is True
+
+    with pytest.raises(NoAccountError):
+        await gather(api_mock.search("foo", limit=10))
+
+    with pytest.raises(NoAccountError):
+        await api_mock.user_by_id(123)
+
+    del os.environ["TWS_RAISE_WHEN_NO_ACCOUNT"]
+    assert get_env_bool("TWS_RAISE_WHEN_NO_ACCOUNT") is False

apps/twscrape/tests/test_parser.py

@@ -0,0 +1,490 @@
+import json
+import os
+from typing import Callable
+
+from twscrape import API, gather
+from twscrape.models import (
+    AudiospaceCard,
+    BroadcastCard,
+    PollCard,
+    SummaryCard,
+    Trend,
+    Tweet,
+    User,
+    UserRef,
+    parse_tweet,
+)
+
+BASE_DIR = os.path.dirname(__file__)
+DATA_DIR = os.path.join(BASE_DIR, "mocked-data")
+os.makedirs(DATA_DIR, exist_ok=True)
+
+
+def get_api():
+    api = API()
+    # To be sure all tests are mocked
+    api.pool = None  # type: ignore
+    return api
+
+
+class FakeRep:
+    text: str
+
+    def __init__(self, text: str):
+        self.text = text
+
+    def json(self):
+        return json.loads(self.text)
+
+
+def fake_rep(filename: str):
+    filename = filename if filename.endswith(".json") else f"{filename}.json"
+    filename = filename if filename.startswith("/") else os.path.join(DATA_DIR, filename)
+
+    with open(filename) as fp:
+        return FakeRep(fp.read())
+
+
+def mock_rep(fn: Callable, filename: str, as_generator=False):
+    rep = fake_rep(filename)
+
+    async def cb_rep(*args, **kwargs):
+        return rep
+
+    async def cb_gen(*args, **kwargs):
+        yield rep
+
+    assert "__self__" in dir(fn)
+    cb = cb_gen if as_generator else cb_rep
+    cb.__name__ = fn.__name__
+    cb.__self__ = fn.__self__  # pyright: ignore
+    setattr(fn.__self__, fn.__name__, cb)  # pyright: ignore
+
+
+def check_tweet(doc: Tweet | None):
+    assert doc is not None
+    assert isinstance(doc.id, int)
+    assert isinstance(doc.id_str, str)
+    assert str(doc.id) == doc.id_str
+
+    assert doc.url is not None
+    assert doc.id_str in doc.url
+    assert doc.user is not None
+
+    assert isinstance(doc.conversationId, int)
+    assert isinstance(doc.conversationIdStr, str)
+    assert str(doc.conversationId) == doc.conversationIdStr
+
+    if doc.inReplyToTweetId is not None:
+        assert isinstance(doc.inReplyToTweetId, int)
+        assert isinstance(doc.inReplyToTweetIdStr, str)
+        assert str(doc.inReplyToTweetId) == doc.inReplyToTweetIdStr
+
+    if doc.inReplyToUser:
+        check_user_ref(doc.inReplyToUser)
+
+    if doc.mentionedUsers:
+        for x in doc.mentionedUsers:
+            check_user_ref(x)
+
+    obj = doc.dict()
+    assert doc.id == obj["id"]
+    assert doc.id_str == obj["id_str"]
+    assert doc.user.id == obj["user"]["id"]
+
+    assert "url" in obj
+    assert "_type" in obj
+    assert obj["_type"] == "snscrape.modules.twitter.Tweet"
+
+    assert "url" in obj["user"]
+    assert "_type" in obj["user"]
+    assert obj["user"]["_type"] == "snscrape.modules.twitter.User"
+
+    txt = doc.json()
+    assert isinstance(txt, str)
+    assert str(doc.id) in txt
+
+    if doc.media is not None:
+        if len(doc.media.photos) > 0:
+            assert doc.media.photos[0].url is not None
+
+        if len(doc.media.videos) > 0:
+            for x in doc.media.videos:
+                assert x.thumbnailUrl is not None
+                assert x.duration is not None
+                for v in x.variants:
+                    assert v.url is not None
+                    assert v.bitrate is not None
+                    assert v.contentType is not None
+
+    if doc.retweetedTweet is not None:
+        try:
+            assert doc.rawContent.endswith(doc.retweetedTweet.rawContent), "content should be full"
+        except AssertionError as e:
+            print("\n" + "-" * 60)
+            print(doc.url)
+            print("1:", doc.rawContent)
+            print("2:", doc.retweetedTweet.rawContent)
+            print("-" * 60)
+            raise e
+
+    check_user(doc.user)
+    assert doc.bookmarkedCount is not None
+
+
+def check_user(doc: User):
+    assert doc.id is not None
+    assert isinstance(doc.id, int)
+    assert isinstance(doc.id_str, str)
+    assert str(doc.id) == doc.id_str
+
+    assert doc.username is not None
+    assert doc.descriptionLinks is not None
+    assert doc.pinnedIds is not None
+    if doc.pinnedIds:
+        for x in doc.pinnedIds:
+            assert isinstance(x, int)
+
+    if len(doc.descriptionLinks) > 0:
+        for x in doc.descriptionLinks:
+            assert x.url is not None
+            assert x.text is not None
+            assert x.tcourl is not None
+
+    obj = doc.dict()
+    assert doc.id == obj["id"]
+    assert doc.username == obj["username"]
+
+    txt = doc.json()
+    assert isinstance(txt, str)
+    assert str(doc.id) in txt
+
+
+def check_user_ref(doc: UserRef):
+    assert isinstance(doc.id, int)
+    assert isinstance(doc.id_str, str)
+    assert str(doc.id) == doc.id_str
+
+    assert doc.username is not None
+    assert doc.displayname is not None
+
+    obj = doc.dict()
+    assert doc.id == obj["id"]
+    assert doc.id_str == obj["id_str"]
+
+
+def check_trend(doc: Trend):
+    assert doc.name is not None
+    assert doc.trend_url is not None
+    assert doc.trend_metadata is not None
+    assert isinstance(doc.grouped_trends, list) or doc.grouped_trends is None
+
+    assert doc.trend_url.url is not None
+    assert doc.trend_url.urlType is not None
+    assert doc.trend_url.urlEndpointOptions
+
+
+async def test_search():
+    api = get_api()
+    mock_rep(api.search_raw, "raw_search", as_generator=True)
+
+    items = await gather(api.search("elon musk lang:en", limit=20))
+    assert len(items) > 0
+
+    bookmarks_count = 0
+    for doc in items:
+        check_tweet(doc)
+        bookmarks_count += doc.bookmarkedCount
+
+    assert bookmarks_count > 0, "`bookmark_fields` key is changed or unluck search data"
+
+
+async def test_user_by_id():
+    api = get_api()
+    mock_rep(api.user_by_id_raw, "raw_user_by_id")
+
+    doc = await api.user_by_id(2244994945)
+    assert doc is not None
+    assert doc.id == 2244994945
+    assert doc.username == "XDevelopers"
+
+    obj = doc.dict()
+    assert doc.id == obj["id"]
+    assert doc.username == obj["username"]
+
+    txt = doc.json()
+    assert isinstance(txt, str)
+    assert str(doc.id) in txt
+
+
+async def test_user_by_login():
+    api = get_api()
+    mock_rep(api.user_by_login_raw, "raw_user_by_login")
+
+    doc = await api.user_by_login("xdevelopers")
+    assert doc is not None
+    assert doc.id == 2244994945
+    assert doc.username == "XDevelopers"
+
+    obj = doc.dict()
+    assert doc.id == obj["id"]
+    assert doc.username == obj["username"]
+
+    txt = doc.json()
+    assert isinstance(txt, str)
+    assert str(doc.id) in txt
+
+
+async def test_tweet_details():
+    api = get_api()
+    mock_rep(api.tweet_details_raw, "raw_tweet_details")
+
+    doc = await api.tweet_details(1649191520250245121)
+    assert doc is not None, "tweet should not be None"
+    check_tweet(doc)
+
+    assert doc.id == 1649191520250245121
+    assert doc.user is not None, "tweet.user should not be None"
+
+
+async def test_tweet_replies():
+    api = get_api()
+    mock_rep(api.tweet_replies_raw, "raw_tweet_replies", as_generator=True)
+
+    twid = 1649191520250245121
+    tweets = await gather(api.tweet_replies(twid, limit=20))
+    assert len(tweets) > 0
+
+    for doc in tweets:
+        check_tweet(doc)
+        assert doc.inReplyToTweetId == twid
+
+
+async def test_followers():
+    api = get_api()
+    mock_rep(api.followers_raw, "raw_followers", as_generator=True)
+
+    users = await gather(api.followers(2244994945))
+    assert len(users) > 0
+
+    for doc in users:
+        check_user(doc)
+
+
+async def test_verified_followers():
+    api = get_api()
+    mock_rep(api.verified_followers_raw, "raw_verified_followers", as_generator=True)
+
+    users = await gather(api.verified_followers(2244994945))
+    assert len(users) > 0
+
+    for doc in users:
+        check_user(doc)
+        assert doc.blue is True, "snould be only Blue users"
+
+
+async def test_subscriptions():
+    api = get_api()
+    mock_rep(api.subscriptions_raw, "raw_subscriptions", as_generator=True)
+
+    users = await gather(api.subscriptions(44196397))
+    assert len(users) > 0
+
+    for doc in users:
+        check_user(doc)
+
+
+async def test_following():
+    api = get_api()
+    mock_rep(api.following_raw, "raw_following", as_generator=True)
+
+    users = await gather(api.following(2244994945))
+    assert len(users) > 0
+
+    for doc in users:
+        check_user(doc)
+
+
+async def test_retweters():
+    api = get_api()
+    mock_rep(api.retweeters_raw, "raw_retweeters", as_generator=True)
+
+    users = await gather(api.retweeters(1649191520250245121))
+    assert len(users) > 0
+
+    for doc in users:
+        check_user(doc)
+
+
+async def test_user_tweets():
+    api = get_api()
+    mock_rep(api.user_tweets_raw, "raw_user_tweets", as_generator=True)
+
+    tweets = await gather(api.user_tweets(2244994945))
+    assert len(tweets) > 0
+
+    is_any_pinned = False
+    for doc in tweets:
+        check_tweet(doc)
+        is_any_pinned = is_any_pinned or doc.id in doc.user.pinnedIds
+
+    assert is_any_pinned, "at least one tweet should be pinned (or bad luck with data)"
+
+
+async def test_user_tweets_and_replies():
+    api = get_api()
+    mock_rep(api.user_tweets_and_replies_raw, "raw_user_tweets_and_replies", as_generator=True)
+
+    tweets = await gather(api.user_tweets_and_replies(2244994945))
+    assert len(tweets) > 0
+
+    for doc in tweets:
+        check_tweet(doc)
+
+
+async def test_raw_user_media():
+    api = get_api()
+    mock_rep(api.user_media_raw, "raw_user_media", as_generator=True)
+
+    tweets = await gather(api.user_media(2244994945))
+    assert len(tweets) > 0
+
+    for doc in tweets:
+        check_tweet(doc)
+        assert doc.media is not None
+        media_count = len(doc.media.photos) + len(doc.media.videos) + len(doc.media.animated)
+        assert media_count > 0, f"{doc.url} should have media"
+
+
+async def test_list_timeline():
+    api = get_api()
+    mock_rep(api.list_timeline_raw, "raw_list_timeline", as_generator=True)
+
+    tweets = await gather(api.list_timeline(1494877848087187461))
+    assert len(tweets) > 0
+
+    for doc in tweets:
+        check_tweet(doc)
+
+
+async def test_trends():
+    api = get_api()
+    mock_rep(api.trends_raw, "raw_trends", as_generator=True)
+
+    items = await gather(api.trends("sport"))
+    assert len(items) > 0
+
+    for doc in items:
+        check_trend(doc)
+
+
+async def test_tweet_with_video():
+    api = get_api()
+
+    files = [
+        ("manual_tweet_with_video_1.json", 1671508600538161153),
+        ("manual_tweet_with_video_2.json", 1671753569412820992),
+    ]
+
+    for file, twid in files:
+        mock_rep(api.tweet_details_raw, file)
+        doc = await api.tweet_details(twid)
+        assert doc is not None
+        check_tweet(doc)
+
+
+async def test_issue_28():
+    api = get_api()
+
+    mock_rep(api.tweet_details_raw, "_issue_28_1")
+    doc = await api.tweet_details(1658409412799737856)
+    assert doc is not None
+    check_tweet(doc)
+
+    assert doc.id == 1658409412799737856
+    assert doc.user is not None
+
+    assert doc.retweetedTweet is not None
+    assert doc.retweetedTweet.viewCount is not None
+    assert doc.viewCount is not None  # views should come from retweetedTweet
+    assert doc.viewCount == doc.retweetedTweet.viewCount
+    check_tweet(doc.retweetedTweet)
+
+    mock_rep(api.tweet_details_raw, "_issue_28_2")
+    doc = await api.tweet_details(1658421690001502208)
+    assert doc is not None
+    check_tweet(doc)
+    assert doc.id == 1658421690001502208
+    assert doc.viewCount is not None
+
+    assert doc.quotedTweet is not None
+    assert doc.quotedTweet.id != doc.id
+    check_tweet(doc.quotedTweet)
+    assert doc.quotedTweet.viewCount is not None
+
+
+async def test_issue_42():
+    raw = fake_rep("_issue_42").json()
+    doc = parse_tweet(raw, 1665951747842641921)
+    assert doc is not None
+    assert doc.retweetedTweet is not None
+    assert doc.rawContent is not None
+    assert doc.retweetedTweet.rawContent is not None
+    assert doc.rawContent.endswith(doc.retweetedTweet.rawContent)
+
+
+async def test_issue_56():
+    raw = fake_rep("_issue_56").json()
+    doc = parse_tweet(raw, 1682072224013099008)
+    assert doc is not None
+    assert len(set([x.tcourl for x in doc.links])) == len(doc.links)
+    assert len(doc.links) == 5
+
+
+async def test_cards():
+    # Issues:
+    # - https://github.com/vladkens/twscrape/issues/72
+    # - https://github.com/vladkens/twscrape/issues/191
+
+    # Check SummaryCard
+    raw = fake_rep("card_summary").json()
+    doc = parse_tweet(raw, 1696922210588410217)
+    assert doc is not None
+    assert doc.card is not None
+    assert isinstance(doc.card, SummaryCard)
+    assert doc.card._type == "summary"
+    assert doc.card.title is not None
+    assert doc.card.description is not None
+    assert doc.card.url is not None
+
+    # Check PollCard
+    raw = fake_rep("card_poll").json()
+    doc = parse_tweet(raw, 1780666831310877100)
+    assert doc is not None
+    assert doc.card is not None
+    assert isinstance(doc.card, PollCard)
+    assert doc.card._type == "poll"
+    assert doc.card.finished is not None
+    assert doc.card.options is not None
+    assert len(doc.card.options) > 0
+    for x in doc.card.options:
+        assert x.label is not None
+        assert x.votesCount is not None
+
+    # Check BrodcastCard
+    raw = fake_rep("card_broadcast").json()
+    doc = parse_tweet(raw, 1790441814857826439)
+    assert doc is not None and doc.card is not None
+    assert doc.card._type == "broadcast"
+    assert isinstance(doc.card, BroadcastCard)
+    assert doc.card.title is not None
+    assert doc.card.url is not None
+    assert doc.card.photo is not None
+
+    # Check AudiospaceCard
+    raw = fake_rep("card_audiospace").json()
+    doc = parse_tweet(raw, 1789054061729173804)
+    assert doc is not None and doc.card is not None
+    assert doc.card._type == "audiospace"
+    assert isinstance(doc.card, AudiospaceCard)
+    assert doc.card.url is not None

apps/twscrape/tests/test_pool.py

@@ -0,0 +1,138 @@
+from twscrape.accounts_pool import AccountsPool
+from twscrape.utils import utc
+
+
+async def test_add_accounts(pool_mock: AccountsPool):
+    # should add account
+    await pool_mock.add_account("user1", "pass1", "email1", "email_pass1")
+    acc = await pool_mock.get("user1")
+    assert acc.username == "user1"
+    assert acc.password == "pass1"
+    assert acc.email == "email1"
+    assert acc.email_password == "email_pass1"
+
+    # should not add account with same username
+    await pool_mock.add_account("user1", "pass2", "email2", "email_pass2")
+    acc = await pool_mock.get("user1")
+    assert acc.username == "user1"
+    assert acc.password == "pass1"
+    assert acc.email == "email1"
+    assert acc.email_password == "email_pass1"
+
+    # should not add account with different username case
+    await pool_mock.add_account("USER1", "pass2", "email2", "email_pass2")
+    acc = await pool_mock.get("user1")
+    assert acc.username == "user1"
+    assert acc.password == "pass1"
+    assert acc.email == "email1"
+    assert acc.email_password == "email_pass1"
+
+    # should add account with different username
+    await pool_mock.add_account("user2", "pass2", "email2", "email_pass2")
+    acc = await pool_mock.get("user2")
+    assert acc.username == "user2"
+    assert acc.password == "pass2"
+    assert acc.email == "email2"
+    assert acc.email_password == "email_pass2"
+
+
+async def test_get_all(pool_mock: AccountsPool):
+    # should return empty list
+    accs = await pool_mock.get_all()
+    assert len(accs) == 0
+
+    # should return all accounts
+    await pool_mock.add_account("user1", "pass1", "email1", "email_pass1")
+    await pool_mock.add_account("user2", "pass2", "email2", "email_pass2")
+    accs = await pool_mock.get_all()
+    assert len(accs) == 2
+    assert accs[0].username == "user1"
+    assert accs[1].username == "user2"
+
+
+async def test_save(pool_mock: AccountsPool):
+    # should save account
+    await pool_mock.add_account("user1", "pass1", "email1", "email_pass1")
+    acc = await pool_mock.get("user1")
+    acc.password = "pass2"
+    await pool_mock.save(acc)
+    acc = await pool_mock.get("user1")
+    assert acc.password == "pass2"
+
+    # should not save account
+    acc = await pool_mock.get("user1")
+    acc.username = "user2"
+    await pool_mock.save(acc)
+    acc = await pool_mock.get("user1")
+    assert acc.username == "user1"
+
+
+async def test_get_for_queue(pool_mock: AccountsPool):
+    Q = "test_queue"
+
+    # should return account
+    await pool_mock.add_account("user1", "pass1", "email1", "email_pass1")
+    await pool_mock.set_active("user1", True)
+    acc = await pool_mock.get_for_queue(Q)
+    assert acc is not None
+    assert acc.username == "user1"
+    assert acc.active is True
+    assert acc.locks is not None
+    assert Q in acc.locks
+    assert acc.locks[Q] is not None
+
+    # should return None
+    acc = await pool_mock.get_for_queue(Q)
+    assert acc is None
+
+
+async def test_account_unlock(pool_mock: AccountsPool):
+    Q = "test_queue"
+
+    await pool_mock.add_account("user1", "pass1", "email1", "email_pass1")
+    await pool_mock.set_active("user1", True)
+    acc = await pool_mock.get_for_queue(Q)
+    assert acc is not None
+    assert acc.locks[Q] is not None
+
+    # should unlock account and make available for queue
+    await pool_mock.unlock(acc.username, Q)
+    acc = await pool_mock.get_for_queue(Q)
+    assert acc is not None
+    assert acc.locks[Q] is not None
+
+    # should update lock time
+    end_time = utc.ts() + 60  # + 1 minute
+    await pool_mock.lock_until(acc.username, Q, end_time)
+
+    acc = await pool_mock.get(acc.username)
+    assert int(acc.locks[Q].timestamp()) == end_time
+
+
+async def test_get_stats(pool_mock: AccountsPool):
+    Q = "SearchTimeline"
+
+    # should return empty stats
+    stats = await pool_mock.stats()
+    for k, v in stats.items():
+        assert v == 0, f"{k} should be 0"
+
+    # should increate total
+    await pool_mock.add_account("user1", "pass1", "email1", "email_pass1")
+    stats = await pool_mock.stats()
+    assert stats["total"] == 1
+    assert stats["active"] == 0
+
+    # should increate active
+    await pool_mock.set_active("user1", True)
+    stats = await pool_mock.stats()
+    assert stats["total"] == 1
+    assert stats["active"] == 1
+
+    # should update queue stats
+    acc = await pool_mock.get_for_queue(Q)
+    assert acc is not None
+    stats = await pool_mock.stats()
+    assert stats["total"] == 1
+    assert stats["active"] == 1
+    assert stats[f"locked_{Q}"] == 1

apps/twscrape/tests/test_queue_client.py

@@ -0,0 +1,158 @@
+from contextlib import aclosing
+
+import httpx
+from pytest_httpx import HTTPXMock
+
+from twscrape.accounts_pool import AccountsPool
+from twscrape.queue_client import QueueClient
+
+DB_FILE = "/tmp/twscrape_test_queue_client.db"
+URL = "https://example.com/api"
+CF = tuple[AccountsPool, QueueClient]
+
+
+async def get_locked(pool: AccountsPool) -> set[str]:
+    rep = await pool.get_all()
+    return set([x.username for x in rep if x.locks.get("SearchTimeline", None) is not None])
+
+
+async def test_lock_account_when_used(httpx_mock: HTTPXMock, client_fixture):
+    pool, client = client_fixture
+
+    locked = await get_locked(pool)
+    assert len(locked) == 0
+
+    # should lock account on getting it
+    await client.__aenter__()
+    locked = await get_locked(pool)
+    assert len(locked) == 1
+    assert "user1" in locked
+
+    # keep locked on request
+    httpx_mock.add_response(url=URL, json={"foo": "bar"}, status_code=200)
+    assert (await client.get(URL)).json() == {"foo": "bar"}
+
+    locked = await get_locked(pool)
+    assert len(locked) == 1
+    assert "user1" in locked
+
+    # unlock on exit
+    await client.__aexit__(None, None, None)
+    locked = await get_locked(pool)
+    assert len(locked) == 0
+
+
+async def test_do_not_switch_account_on_200(httpx_mock: HTTPXMock, client_fixture: CF):
+    pool, client = client_fixture
+
+    # get account and lock it
+    await client.__aenter__()
+    locked1 = await get_locked(pool)
+    assert len(locked1) == 1
+
+    # make several requests with status=200
+    for x in range(1):
+        httpx_mock.add_response(url=URL, json={"foo": x}, status_code=200)
+        rep = await client.get(URL)
+        assert rep is not None
+        assert rep.json() == {"foo": x}
+
+    # account should not be switched
+    locked2 = await get_locked(pool)
+    assert locked1 == locked2
+
+    # unlock on exit
+    await client.__aexit__(None, None, None)
+    locked3 = await get_locked(pool)
+    assert len(locked3) == 0
+
+
+async def test_switch_acc_on_http_error(httpx_mock: HTTPXMock, client_fixture: CF):
+    pool, client = client_fixture
+
+    # locked account on enter
+    await client.__aenter__()
+    locked1 = await get_locked(pool)
+    assert len(locked1) == 1
+
+    # fail one request, account should be switched
+    httpx_mock.add_response(url=URL, json={"foo": "1"}, status_code=403)
+    httpx_mock.add_response(url=URL, json={"foo": "2"}, status_code=200)
+
+    rep = await client.get(URL)
+    assert rep is not None
+    assert rep.json() == {"foo": "2"}
+
+    locked2 = await get_locked(pool)
+    assert len(locked2) == 2
+
+    # unlock on exit (failed account still should locked)
+    await client.__aexit__(None, None, None)
+    locked3 = await get_locked(pool)
+    assert len(locked3) == 1
+    assert locked1 == locked3  # failed account locked
+
+
+async def test_retry_with_same_acc_on_network_error(httpx_mock: HTTPXMock, client_fixture: CF):
+    pool, client = client_fixture
+
+    # locked account on enter
+    await client.__aenter__()
+    locked1 = await get_locked(pool)
+    assert len(locked1) == 1
+
+    # timeout on first request, account should not be switched
+    httpx_mock.add_exception(httpx.ReadTimeout("Unable to read within timeout"))
+    httpx_mock.add_response(url=URL, json={"foo": "2"}, status_code=200)
+
+    rep = await client.get(URL)
+    assert rep is not None
+    assert rep.json() == {"foo": "2"}
+
+    locked2 = await get_locked(pool)
+    assert locked2 == locked1
+
+    # check username added to request obj (for logging)
+    username = getattr(rep, "__username", None)
+    assert username is not None
+
+
+async def test_ctx_closed_on_break(httpx_mock: HTTPXMock, client_fixture: CF):
+    pool, client = client_fixture
+
+    async def get_data_stream():
+        async with client as c:
+            counter = 0
+            while True:
+                counter += 1
+                check_retry = counter == 2
+                before_ctx = c.ctx
+
+                if check_retry:
+                    httpx_mock.add_response(url=URL, json={"counter": counter}, status_code=403)
+                    httpx_mock.add_response(url=URL, json={"counter": counter}, status_code=200)
+                else:
+                    httpx_mock.add_response(url=URL, json={"counter": counter}, status_code=200)
+
+                rep = await c.get(URL)
+
+                if check_retry:
+                    assert before_ctx != c.ctx
+                elif before_ctx is not None:
+                    assert before_ctx == c.ctx
+
+                assert rep is not None
+                assert rep.json() == {"counter": counter}
+                yield rep.json()["counter"]
+
+                if counter == 9:
+                    return
+
+    # need to use async with to break to work
+    async with aclosing(get_data_stream()) as gen:
+        async for x in gen:
+            if x == 3:
+                break
+
+    # ctx should be None after break
+    assert client.ctx is None

apps/twscrape/tests/test_utils.py

@@ -0,0 +1,27 @@
+import pytest
+
+from twscrape.utils import parse_cookies
+
+
+def test_cookies_parse():
+    val = "abc=123; def=456; ghi=789"
+    assert parse_cookies(val) == {"abc": "123", "def": "456", "ghi": "789"}
+
+    val = '{"abc": "123", "def": "456", "ghi": "789"}'
+    assert parse_cookies(val) == {"abc": "123", "def": "456", "ghi": "789"}
+
+    val = '[{"name": "abc", "value": "123"}, {"name": "def", "value": "456"}, {"name": "ghi", "value": "789"}]'
+    assert parse_cookies(val) == {"abc": "123", "def": "456", "ghi": "789"}
+
+    val = "eyJhYmMiOiAiMTIzIiwgImRlZiI6ICI0NTYiLCAiZ2hpIjogIjc4OSJ9"
+    assert parse_cookies(val) == {"abc": "123", "def": "456", "ghi": "789"}
+
+    val = "W3sibmFtZSI6ICJhYmMiLCAidmFsdWUiOiAiMTIzIn0sIHsibmFtZSI6ICJkZWYiLCAidmFsdWUiOiAiNDU2In0sIHsibmFtZSI6ICJnaGkiLCAidmFsdWUiOiAiNzg5In1d"
+    assert parse_cookies(val) == {"abc": "123", "def": "456", "ghi": "789"}
+
+    val = '{"cookies": {"abc": "123", "def": "456", "ghi": "789"}}'
+    assert parse_cookies(val) == {"abc": "123", "def": "456", "ghi": "789"}
+
+    with pytest.raises(ValueError, match=r"Invalid cookie value: .+"):
+        val = "{invalid}"
+        assert parse_cookies(val) == {}

apps/twscrape/twscrape/__init__.py

@@ -0,0 +1,7 @@
+# ruff: noqa: F401
+from .account import Account
+from .accounts_pool import AccountsPool, NoAccountError
+from .api import API
+from .logger import set_log_level
+from .models import *  # noqa: F403
+from .utils import gather

apps/twscrape/twscrape/account.py

@@ -0,0 +1,75 @@
+import json
+import os
+import sqlite3
+from dataclasses import asdict, dataclass, field
+from datetime import datetime
+
+from httpx import AsyncClient, AsyncHTTPTransport
+
+from .models import JSONTrait
+from .utils import utc
+
+TOKEN = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
+
+
+@dataclass
+class Account(JSONTrait):
+    username: str
+    password: str
+    email: str
+    email_password: str
+    user_agent: str
+    active: bool
+    locks: dict[str, datetime] = field(default_factory=dict)  # queue: datetime
+    stats: dict[str, int] = field(default_factory=dict)  # queue: requests
+    headers: dict[str, str] = field(default_factory=dict)
+    cookies: dict[str, str] = field(default_factory=dict)
+    mfa_code: str | None = None
+    proxy: str | None = None
+    error_msg: str | None = None
+    last_used: datetime | None = None
+    _tx: str | None = None
+
+    @staticmethod
+    def from_rs(rs: sqlite3.Row):
+        doc = dict(rs)
+        doc["locks"] = {k: utc.from_iso(v) for k, v in json.loads(doc["locks"]).items()}
+        doc["stats"] = {k: v for k, v in json.loads(doc["stats"]).items() if isinstance(v, int)}
+        doc["headers"] = json.loads(doc["headers"])
+        doc["cookies"] = json.loads(doc["cookies"])
+        doc["active"] = bool(doc["active"])
+        doc["last_used"] = utc.from_iso(doc["last_used"]) if doc["last_used"] else None
+        return Account(**doc)
+
+    def to_rs(self):
+        rs = asdict(self)
+        rs["locks"] = json.dumps(rs["locks"], default=lambda x: x.isoformat())
+        rs["stats"] = json.dumps(rs["stats"])
+        rs["headers"] = json.dumps(rs["headers"])
+        rs["cookies"] = json.dumps(rs["cookies"])
+        rs["last_used"] = rs["last_used"].isoformat() if rs["last_used"] else None
+        return rs
+
+    def make_client(self, proxy: str | None = None) -> AsyncClient:
+        proxies = [proxy, os.getenv("TWS_PROXY"), self.proxy]
+        proxies = [x for x in proxies if x is not None]
+        proxy = proxies[0] if proxies else None
+
+        transport = AsyncHTTPTransport(retries=2)
+        client = AsyncClient(proxy=proxy, follow_redirects=True, transport=transport)
+
+        # saved from previous usage
+        client.cookies.update(self.cookies)
+        client.headers.update(self.headers)
+
+        # default settings
+        client.headers["user-agent"] = self.user_agent
+        client.headers["content-type"] = "application/json"
+        client.headers["authorization"] = TOKEN
+        client.headers["x-twitter-active-user"] = "yes"
+        client.headers["x-twitter-client-language"] = "en"
+
+        if "ct0" in client.cookies:
+            client.headers["x-csrf-token"] = client.cookies["ct0"]
+
+        return client

apps/twscrape/twscrape/accounts_pool.py

@@ -0,0 +1,385 @@
+import asyncio
+import sqlite3
+import uuid
+from datetime import datetime, timezone
+from typing import TypedDict
+
+from fake_useragent import UserAgent
+from httpx import HTTPStatusError
+
+from .account import Account
+from .db import execute, fetchall, fetchone
+from .logger import logger
+from .login import LoginConfig, login
+from .utils import get_env_bool, parse_cookies, utc
+
+
+class NoAccountError(Exception):
+    pass
+
+
+class AccountInfo(TypedDict):
+    username: str
+    logged_in: bool
+    active: bool
+    last_used: datetime | None
+    total_req: int
+    error_msg: str | None
+
+
+def guess_delim(line: str):
+    lp, rp = tuple([x.strip() for x in line.split("username")])
+    return rp[0] if not lp else lp[-1]
+
+
+class AccountsPool:
+    # _order_by: str = "RANDOM()"
+    _order_by: str = "username"
+
+    def __init__(
+        self,
+        db_file="accounts.db",
+        login_config: LoginConfig | None = None,
+        raise_when_no_account=False,
+    ):
+        self._db_file = db_file
+        self._login_config = login_config or LoginConfig()
+        self._raise_when_no_account = raise_when_no_account
+
+    async def load_from_file(self, filepath: str, line_format: str):
+        line_delim = guess_delim(line_format)
+        tokens = line_format.split(line_delim)
+
+        required = set(["username", "password", "email", "email_password"])
+        if not required.issubset(tokens):
+            raise ValueError(f"Invalid line format: {line_format}")
+
+        accounts = []
+        with open(filepath, "r") as f:
+            lines = f.read().split("\n")
+            lines = [x.strip() for x in lines if x.strip()]
+
+            for line in lines:
+                data = [x.strip() for x in line.split(line_delim)]
+                if len(data) < len(tokens):
+                    raise ValueError(f"Invalid line: {line}")
+
+                data = data[: len(tokens)]
+                vals = {k: v for k, v in zip(tokens, data) if k != "_"}
+                accounts.append(vals)
+
+        for x in accounts:
+            await self.add_account(**x)
+
+    async def add_account(
+        self,
+        username: str,
+        password: str,
+        email: str,
+        email_password: str,
+        user_agent: str | None = None,
+        proxy: str | None = None,
+        cookies: str | None = None,
+        mfa_code: str | None = None,
+    ):
+        qs = "SELECT * FROM accounts WHERE username = :username"
+        rs = await fetchone(self._db_file, qs, {"username": username})
+        if rs:
+            logger.warning(f"Account {username} already exists")
+            return
+
+        account = Account(
+            username=username,
+            password=password,
+            email=email,
+            email_password=email_password,
+            user_agent=user_agent or UserAgent().safari,
+            active=False,
+            locks={},
+            stats={},
+            headers={},
+            cookies=parse_cookies(cookies) if cookies else {},
+            proxy=proxy,
+            mfa_code=mfa_code,
+        )
+
+        if "ct0" in account.cookies:
+            account.active = True
+
+        await self.save(account)
+        logger.info(f"Account {username} added successfully (active={account.active})")
+
+    async def delete_accounts(self, usernames: str | list[str]):
+        usernames = usernames if isinstance(usernames, list) else [usernames]
+        usernames = list(set(usernames))
+        if not usernames:
+            logger.warning("No usernames provided")
+            return
+
+        qs = f"""DELETE FROM accounts WHERE username IN ({",".join([f'"{x}"' for x in usernames])})"""
+        await execute(self._db_file, qs)
+
+    async def delete_inactive(self):
+        qs = "DELETE FROM accounts WHERE active = false"
+        await execute(self._db_file, qs)
+
+    async def get(self, username: str):
+        qs = "SELECT * FROM accounts WHERE username = :username"
+        rs = await fetchone(self._db_file, qs, {"username": username})
+        if not rs:
+            raise ValueError(f"Account {username} not found")
+        return Account.from_rs(rs)
+
+    async def get_all(self):
+        qs = "SELECT * FROM accounts"
+        rs = await fetchall(self._db_file, qs)
+        return [Account.from_rs(x) for x in rs]
+
+    async def get_account(self, username: str):
+        qs = "SELECT * FROM accounts WHERE username = :username"
+        rs = await fetchone(self._db_file, qs, {"username": username})
+        if not rs:
+            return None
+        return Account.from_rs(rs)
+
+    async def save(self, account: Account):
+        data = account.to_rs()
+        cols = list(data.keys())
+
+        qs = f"""
+        INSERT INTO accounts ({",".join(cols)}) VALUES ({",".join([f":{x}" for x in cols])})
+        ON CONFLICT(username) DO UPDATE SET {",".join([f"{x}=excluded.{x}" for x in cols])}
+        """
+        await execute(self._db_file, qs, data)
+
+    async def login(self, account: Account):
+        try:
+            await login(account, cfg=self._login_config)
+            logger.info(f"Logged in to {account.username} successfully")
+            return True
+        except HTTPStatusError as e:
+            rep = e.response
+            logger.error(f"Failed to login '{account.username}': {rep.status_code} - {rep.text}")
+            return False
+        except Exception as e:
+            logger.error(f"Failed to login '{account.username}': {e}")
+            return False
+        finally:
+            await self.save(account)
+
+    async def login_all(self, usernames: list[str] | None = None):
+        if usernames is None:
+            qs = "SELECT * FROM accounts WHERE active = false AND error_msg IS NULL"
+        else:
+            us = ",".join([f'"{x}"' for x in usernames])
+            qs = f"SELECT * FROM accounts WHERE username IN ({us})"
+
+        rs = await fetchall(self._db_file, qs)
+        accounts = [Account.from_rs(rs) for rs in rs]
+        # await asyncio.gather(*[login(x) for x in self.accounts])
+
+        counter = {"total": len(accounts), "success": 0, "failed": 0}
+        for i, x in enumerate(accounts, start=1):
+            logger.info(f"[{i}/{len(accounts)}] Logging in {x.username} - {x.email}")
+            status = await self.login(x)
+            counter["success" if status else "failed"] += 1
+        return counter
+
+    async def relogin(self, usernames: str | list[str]):
+        usernames = usernames if isinstance(usernames, list) else [usernames]
+        usernames = list(set(usernames))
+        if not usernames:
+            logger.warning("No usernames provided")
+            return
+
+        qs = f"""
+        UPDATE accounts SET
+            active = false,
+            locks = json_object(),
+            last_used = NULL,
+            error_msg = NULL,
+            headers = json_object(),
+            cookies = json_object(),
+            user_agent = "{UserAgent().safari}"
+        WHERE username IN ({",".join([f'"{x}"' for x in usernames])})
+        """
+
+        await execute(self._db_file, qs)
+        await self.login_all(usernames)
+
+    async def relogin_failed(self):
+        qs = "SELECT username FROM accounts WHERE active = false AND error_msg IS NOT NULL"
+        rs = await fetchall(self._db_file, qs)
+        await self.relogin([x["username"] for x in rs])
+
+    async def reset_locks(self):
+        qs = "UPDATE accounts SET locks = json_object()"
+        await execute(self._db_file, qs)
+
+    async def set_active(self, username: str, active: bool):
+        qs = "UPDATE accounts SET active = :active WHERE username = :username"
+        await execute(self._db_file, qs, {"username": username, "active": active})
+
+    async def lock_until(self, username: str, queue: str, unlock_at: int, req_count=0):
+        qs = f"""
+        UPDATE accounts SET
+            locks = json_set(locks, '$.{queue}', datetime({unlock_at}, 'unixepoch')),
+            stats = json_set(stats, '$.{queue}', COALESCE(json_extract(stats, '$.{queue}'), 0) + {req_count}),
+            last_used = datetime({utc.ts()}, 'unixepoch')
+        WHERE username = :username
+        """
+        await execute(self._db_file, qs, {"username": username})
+
+    async def unlock(self, username: str, queue: str, req_count=0):
+        qs = f"""
+        UPDATE accounts SET
+            locks = json_remove(locks, '$.{queue}'),
+            stats = json_set(stats, '$.{queue}', COALESCE(json_extract(stats, '$.{queue}'), 0) + {req_count}),
+            last_used = datetime({utc.ts()}, 'unixepoch')
+        WHERE username = :username
+        """
+        await execute(self._db_file, qs, {"username": username})
+
+    async def _get_and_lock(self, queue: str, condition: str):
+        # if space in condition, it's a subquery, otherwise it's username
+        condition = f"({condition})" if " " in condition else f"'{condition}'"
+
+        if int(sqlite3.sqlite_version_info[1]) >= 35:
+            qs = f"""
+            UPDATE accounts SET
+                locks = json_set(locks, '$.{queue}', datetime('now', '+15 minutes')),
+                last_used = datetime({utc.ts()}, 'unixepoch')
+            WHERE username = {condition}
+            RETURNING *
+            """
+            rs = await fetchone(self._db_file, qs)
+        else:
+            tx = uuid.uuid4().hex
+            qs = f"""
+            UPDATE accounts SET
+                locks = json_set(locks, '$.{queue}', datetime('now', '+15 minutes')),
+                last_used = datetime({utc.ts()}, 'unixepoch'),
+                _tx = '{tx}'
+            WHERE username = {condition}
+            """
+            await execute(self._db_file, qs)
+
+            qs = f"SELECT * FROM accounts WHERE _tx = '{tx}'"
+            rs = await fetchone(self._db_file, qs)
+
+        return Account.from_rs(rs) if rs else None
+
+    async def get_for_queue(self, queue: str):
+        q = f"""
+        SELECT username FROM accounts
+        WHERE active = true AND (
+            locks IS NULL
+            OR json_extract(locks, '$.{queue}') IS NULL
+            OR json_extract(locks, '$.{queue}') < datetime('now')
+        )
+        ORDER BY {self._order_by}
+        LIMIT 1
+        """
+
+        return await self._get_and_lock(queue, q)
+
+    async def get_for_queue_or_wait(self, queue: str) -> Account | None:
+        msg_shown = False
+        while True:
+            account = await self.get_for_queue(queue)
+            if not account:
+                if self._raise_when_no_account or get_env_bool("TWS_RAISE_WHEN_NO_ACCOUNT"):
+                    raise NoAccountError(f"No account available for queue {queue}")
+
+                if not msg_shown:
+                    nat = await self.next_available_at(queue)
+                    if not nat:
+                        logger.warning("No active accounts. Stopping...")
+                        return None
+
+                    msg = f'No account available for queue "{queue}". Next available at {nat}'
+                    logger.info(msg)
+                    msg_shown = True
+
+                await asyncio.sleep(5)
+                continue
+            else:
+                if msg_shown:
+                    logger.info(f"Continuing with account {account.username} on queue {queue}")
+
+            return account
+
+    async def next_available_at(self, queue: str):
+        qs = f"""
+        SELECT json_extract(locks, '$."{queue}"') as lock_until
+        FROM accounts
+        WHERE active = true AND json_extract(locks, '$."{queue}"') IS NOT NULL
+        ORDER BY lock_until ASC
+        LIMIT 1
+        """
+        rs = await fetchone(self._db_file, qs)
+        if rs:
+            now, trg = utc.now(), utc.from_iso(rs[0])
+            if trg < now:
+                return "now"
+
+            at_local = datetime.now() + (trg - now)
+            return at_local.strftime("%H:%M:%S")
+
+        return None
+
+    async def mark_inactive(self, username: str, error_msg: str | None):
+        qs = """
+        UPDATE accounts SET active = false, error_msg = :error_msg
+        WHERE username = :username
+        """
+        await execute(self._db_file, qs, {"username": username, "error_msg": error_msg})
+
+    async def stats(self):
+        def locks_count(queue: str):
+            return f"""
+            SELECT COUNT(*) FROM accounts
+            WHERE json_extract(locks, '$.{queue}') IS NOT NULL
+                AND json_extract(locks, '$.{queue}') > datetime('now')
+            """
+
+        qs = "SELECT DISTINCT(f.key) as k from accounts, json_each(locks) f"
+        rs = await fetchall(self._db_file, qs)
+        gql_ops = [x["k"] for x in rs]
+
+        config = [
+            ("total", "SELECT COUNT(*) FROM accounts"),
+            ("active", "SELECT COUNT(*) FROM accounts WHERE active = true"),
+            ("inactive", "SELECT COUNT(*) FROM accounts WHERE active = false"),
+            *[(f"locked_{x}", locks_count(x)) for x in gql_ops],
+        ]
+
+        qs = f"SELECT {','.join([f'({q}) as {k}' for k, q in config])}"
+        rs = await fetchone(self._db_file, qs)
+        return dict(rs) if rs else {}
+
+    async def accounts_info(self):
+        accounts = await self.get_all()
+
+        items: list[AccountInfo] = []
+        for x in accounts:
+            item: AccountInfo = {
+                "username": x.username,
+                "logged_in": (x.headers or {}).get("authorization", "") != "",
+                "active": x.active,
+                "last_used": x.last_used,
+                "total_req": sum(x.stats.values()),
+                "error_msg": str(x.error_msg)[0:60],
+            }
+            items.append(item)
+
+        old_time = datetime(1970, 1, 1).replace(tzinfo=timezone.utc)
+        items = sorted(items, key=lambda x: x["username"].lower())
+        items = sorted(
+            items,
+            key=lambda x: x["last_used"] or old_time if x["total_req"] > 0 else old_time,
+            reverse=True,
+        )
+        items = sorted(items, key=lambda x: x["active"], reverse=True)
+        # items = sorted(items, key=lambda x: x["total_req"], reverse=True)
+        return items

apps/twscrape/twscrape/api.py

@@ -0,0 +1,514 @@
+from contextlib import aclosing
+from typing import Literal
+
+from httpx import Response
+
+from .accounts_pool import AccountsPool
+from .logger import set_log_level
+from .models import Tweet, User, parse_trends, parse_tweet, parse_tweets, parse_user, parse_users
+from .queue_client import QueueClient
+from .utils import encode_params, find_obj, get_by_path
+
+# OP_{NAME} – {NAME} should be same as second part of GQL ID (required to auto-update script)
+OP_SearchTimeline = "U3QTLwGF8sZCHDuWIMSAmg/SearchTimeline"
+OP_UserByRestId = "5vdJ5sWkbSRDiiNZvwc2Yg/UserByRestId"
+OP_UserByScreenName = "32pL5BWe9WKeSK1MoPvFQQ/UserByScreenName"
+OP_TweetDetail = "Ez6kRPyXbqNlhBwcNMpU-Q/TweetDetail"
+OP_Followers = "OGScL-RC4DFMsRGOCjPR6g/Followers"
+OP_Following = "o5eNLkJb03ayTQa97Cpp7w/Following"
+OP_Retweeters = "niCJ2QyTuAgZWv01E7mqJQ/Retweeters"
+OP_UserTweets = "Y9WM4Id6UcGFE8Z-hbnixw/UserTweets"
+OP_UserTweetsAndReplies = "pZXwh96YGRqmBbbxu7Vk2Q/UserTweetsAndReplies"
+OP_ListLatestTweetsTimeline = "H_dAKg97dSn3FOMfrNS8nw/ListLatestTweetsTimeline"
+OP_BlueVerifiedFollowers = "WijS8Cwfqhtk5hDN9q7sgw/BlueVerifiedFollowers"
+OP_UserCreatorSubscriptions = "H4p-DZU4gYqcZulycazCZw/UserCreatorSubscriptions"
+OP_UserMedia = "ophTtKkfXcUKnXlxh9fU5w/UserMedia"
+OP_Bookmarks = "1vFR5f4iSCQZLzjdSsNYwA/Bookmarks"
+OP_GenericTimelineById = "5u36Lskx1dfACjC_WHmH3Q/GenericTimelineById"
+
+GQL_URL = "https://x.com/i/api/graphql"
+GQL_FEATURES = {  # search values here (view source) https://x.com/
+    "articles_preview_enabled": False,
+    "c9s_tweet_anatomy_moderator_badge_enabled": True,
+    "communities_web_enable_tweet_community_results_fetch": True,
+    "creator_subscriptions_quote_tweet_preview_enabled": False,
+    "creator_subscriptions_tweet_preview_api_enabled": True,
+    "freedom_of_speech_not_reach_fetch_enabled": True,
+    "graphql_is_translatable_rweb_tweet_is_translatable_enabled": True,
+    "longform_notetweets_consumption_enabled": True,
+    "longform_notetweets_inline_media_enabled": True,
+    "longform_notetweets_rich_text_read_enabled": True,
+    "responsive_web_edit_tweet_api_enabled": True,
+    "responsive_web_enhance_cards_enabled": False,
+    "responsive_web_graphql_exclude_directive_enabled": True,
+    "responsive_web_graphql_skip_user_profile_image_extensions_enabled": False,
+    "responsive_web_graphql_timeline_navigation_enabled": True,
+    "responsive_web_media_download_video_enabled": False,
+    "responsive_web_twitter_article_tweet_consumption_enabled": True,
+    "rweb_tipjar_consumption_enabled": True,
+    "rweb_video_timestamps_enabled": True,
+    "standardized_nudges_misinfo": True,
+    "tweet_awards_web_tipping_enabled": False,
+    "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": True,
+    "tweet_with_visibility_results_prefer_gql_media_interstitial_enabled": False,
+    "tweetypie_unmention_optimization_enabled": True,
+    "verified_phone_label_enabled": False,
+    "view_counts_everywhere_api_enabled": True,
+    "responsive_web_grok_analyze_button_fetch_trends_enabled": False,
+    "premium_content_api_read_enabled": False,
+    "profile_label_improvements_pcf_label_in_post_enabled": False,
+    "responsive_web_grok_share_attachment_enabled": False,
+    "responsive_web_grok_analyze_post_followups_enabled": False,
+    "responsive_web_grok_image_annotation_enabled": False,
+    "responsive_web_grok_analysis_button_from_backend": False,
+    "responsive_web_jetfuel_frame": False,
+}
+
+KV = dict | None
+TrendId = Literal["trending", "news", "sport", "entertainment"] | str
+
+
+class API:
+    # Note: kv is variables, ft is features from original GQL request
+    pool: AccountsPool
+
+    def __init__(
+        self,
+        pool: AccountsPool | str | None = None,
+        debug=False,
+        proxy: str | None = None,
+        raise_when_no_account=False,
+    ):
+        if isinstance(pool, AccountsPool):
+            self.pool = pool
+        elif isinstance(pool, str):
+            self.pool = AccountsPool(db_file=pool, raise_when_no_account=raise_when_no_account)
+        else:
+            self.pool = AccountsPool(raise_when_no_account=raise_when_no_account)
+
+        self.proxy = proxy
+        self.debug = debug
+        if self.debug:
+            set_log_level("DEBUG")
+
+    # general helpers
+
+    def _is_end(self, rep: Response, q: str, res: list, cur: str | None, cnt: int, lim: int):
+        new_count = len(res)
+        new_total = cnt + new_count
+
+        is_res = new_count > 0
+        is_cur = cur is not None
+        is_lim = lim > 0 and new_total >= lim
+
+        return rep if is_res else None, new_total, is_cur and not is_lim
+
+    def _get_cursor(self, obj: dict, cursor_type="Bottom") -> str | None:
+        if cur := find_obj(obj, lambda x: x.get("cursorType") == cursor_type):
+            return cur.get("value")
+        return None
+
+    # gql helpers
+
+    async def _gql_items(
+        self, op: str, kv: dict, ft: dict | None = None, limit=-1, cursor_type="Bottom"
+    ):
+        queue, cur, cnt, active = op.split("/")[-1], None, 0, True
+        kv, ft = {**kv}, {**GQL_FEATURES, **(ft or {})}
+
+        async with QueueClient(self.pool, queue, self.debug, proxy=self.proxy) as client:
+            while active:
+                params = {"variables": kv, "features": ft}
+                if cur is not None:
+                    params["variables"]["cursor"] = cur
+                if queue in ("SearchTimeline", "ListLatestTweetsTimeline"):
+                    params["fieldToggles"] = {"withArticleRichContentState": False}
+                if queue in ("UserMedia",):
+                    params["fieldToggles"] = {"withArticlePlainText": False}
+
+                rep = await client.get(f"{GQL_URL}/{op}", params=encode_params(params))
+                if rep is None:
+                    return
+
+                obj = rep.json()
+                els = get_by_path(obj, "entries") or []
+                els = [
+                    x
+                    for x in els
+                    if not (
+                        x["entryId"].startswith("cursor-")
+                        or x["entryId"].startswith("messageprompt-")
+                    )
+                ]
+                cur = self._get_cursor(obj, cursor_type)
+
+                rep, cnt, active = self._is_end(rep, queue, els, cur, cnt, limit)
+                if rep is None:
+                    return
+
+                yield rep
+
+    async def _gql_item(self, op: str, kv: dict, ft: dict | None = None):
+        ft = ft or {}
+        queue = op.split("/")[-1]
+        async with QueueClient(self.pool, queue, self.debug, proxy=self.proxy) as client:
+            params = {"variables": {**kv}, "features": {**GQL_FEATURES, **ft}}
+            return await client.get(f"{GQL_URL}/{op}", params=encode_params(params))
+
+    # search
+
+    async def search_raw(self, q: str, limit=-1, kv: KV = None):
+        op = OP_SearchTimeline
+        kv = {
+            "rawQuery": q,
+            "count": 20,
+            "product": "Latest",
+            "querySource": "typed_query",
+            **(kv or {}),
+        }
+        async with aclosing(self._gql_items(op, kv, limit=limit)) as gen:
+            async for x in gen:
+                yield x
+
+    async def search(self, q: str, limit=-1, kv: KV = None):
+        async with aclosing(self.search_raw(q, limit=limit, kv=kv)) as gen:
+            async for rep in gen:
+                for x in parse_tweets(rep.json(), limit):
+                    yield x
+
+    async def search_user(self, q: str, limit=-1, kv: KV = None):
+        kv = {"product": "People", **(kv or {})}
+        async with aclosing(self.search_raw(q, limit=limit, kv=kv)) as gen:
+            async for rep in gen:
+                for x in parse_users(rep.json(), limit):
+                    yield x
+
+    # user_by_id
+
+    async def user_by_id_raw(self, uid: int, kv: KV = None):
+        op = OP_UserByRestId
+        kv = {"userId": str(uid), "withSafetyModeUserFields": True, **(kv or {})}
+        ft = {
+            "hidden_profile_likes_enabled": True,
+            "highlights_tweets_tab_ui_enabled": True,
+            "creator_subscriptions_tweet_preview_api_enabled": True,
+            "hidden_profile_subscriptions_enabled": True,
+            "responsive_web_twitter_article_notes_tab_enabled": False,
+            "subscriptions_feature_can_gift_premium": False,
+            "profile_label_improvements_pcf_label_in_post_enabled": False,
+        }
+        return await self._gql_item(op, kv, ft)
+
+    async def user_by_id(self, uid: int, kv: KV = None) -> User | None:
+        rep = await self.user_by_id_raw(uid, kv=kv)
+        return parse_user(rep) if rep else None
+
+    # user_by_login
+
+    async def user_by_login_raw(self, login: str, kv: KV = None):
+        op = OP_UserByScreenName
+        kv = {"screen_name": login, "withSafetyModeUserFields": True, **(kv or {})}
+        ft = {
+            "highlights_tweets_tab_ui_enabled": True,
+            "hidden_profile_likes_enabled": True,
+            "creator_subscriptions_tweet_preview_api_enabled": True,
+            "hidden_profile_subscriptions_enabled": True,
+            "subscriptions_verification_info_verified_since_enabled": True,
+            "subscriptions_verification_info_is_identity_verified_enabled": False,
+            "responsive_web_twitter_article_notes_tab_enabled": False,
+            "subscriptions_feature_can_gift_premium": False,
+            "profile_label_improvements_pcf_label_in_post_enabled": False,
+        }
+        return await self._gql_item(op, kv, ft)
+
+    async def user_by_login(self, login: str, kv: KV = None) -> User | None:
+        rep = await self.user_by_login_raw(login, kv=kv)
+        return parse_user(rep) if rep else None
+
+    # tweet_details
+
+    async def tweet_details_raw(self, twid: int, kv: KV = None):
+        op = OP_TweetDetail
+        kv = {
+            "focalTweetId": str(twid),
+            "with_rux_injections": True,
+            "includePromotedContent": True,
+            "withCommunity": True,
+            "withQuickPromoteEligibilityTweetFields": True,
+            "withBirdwatchNotes": True,
+            "withVoice": True,
+            "withV2Timeline": True,
+            **(kv or {}),
+        }
+        return await self._gql_item(op, kv)
+
+    async def tweet_details(self, twid: int, kv: KV = None) -> Tweet | None:
+        rep = await self.tweet_details_raw(twid, kv=kv)
+        return parse_tweet(rep, twid) if rep else None
+
+    # tweet_replies
+    # note: uses same op as tweet_details, see: https://github.com/vladkens/twscrape/issues/104
+
+    async def tweet_replies_raw(self, twid: int, limit=-1, kv: KV = None):
+        op = OP_TweetDetail
+        kv = {
+            "focalTweetId": str(twid),
+            "referrer": "tweet",
+            "with_rux_injections": True,
+            "includePromotedContent": True,
+            "withCommunity": True,
+            "withQuickPromoteEligibilityTweetFields": True,
+            "withBirdwatchNotes": True,
+            "withVoice": True,
+            "withV2Timeline": True,
+            **(kv or {}),
+        }
+        async with aclosing(
+            self._gql_items(op, kv, limit=limit, cursor_type="ShowMoreThreads")
+        ) as gen:
+            async for x in gen:
+                yield x
+
+    async def tweet_replies(self, twid: int, limit=-1, kv: KV = None):
+        async with aclosing(self.tweet_replies_raw(twid, limit=limit, kv=kv)) as gen:
+            async for rep in gen:
+                for x in parse_tweets(rep.json(), limit):
+                    if x.inReplyToTweetId == twid:
+                        yield x
+
+    # followers
+
+    async def followers_raw(self, uid: int, limit=-1, kv: KV = None):
+        op = OP_Followers
+        kv = {"userId": str(uid), "count": 20, "includePromotedContent": False, **(kv or {})}
+        ft = {"responsive_web_twitter_article_notes_tab_enabled": False}
+        async with aclosing(self._gql_items(op, kv, limit=limit, ft=ft)) as gen:
+            async for x in gen:
+                yield x
+
+    async def followers(self, uid: int, limit=-1, kv: KV = None):
+        async with aclosing(self.followers_raw(uid, limit=limit, kv=kv)) as gen:
+            async for rep in gen:
+                for x in parse_users(rep.json(), limit):
+                    yield x
+
+    # verified_followers
+
+    async def verified_followers_raw(self, uid: int, limit=-1, kv: KV = None):
+        op = OP_BlueVerifiedFollowers
+        kv = {"userId": str(uid), "count": 20, "includePromotedContent": False, **(kv or {})}
+        ft = {
+            "responsive_web_twitter_article_notes_tab_enabled": True,
+        }
+        async with aclosing(self._gql_items(op, kv, limit=limit, ft=ft)) as gen:
+            async for x in gen:
+                yield x
+
+    async def verified_followers(self, uid: int, limit=-1, kv: KV = None):
+        async with aclosing(self.verified_followers_raw(uid, limit=limit, kv=kv)) as gen:
+            async for rep in gen:
+                for x in parse_users(rep.json(), limit):
+                    yield x
+
+    # following
+
+    async def following_raw(self, uid: int, limit=-1, kv: KV = None):
+        op = OP_Following
+        kv = {"userId": str(uid), "count": 20, "includePromotedContent": False, **(kv or {})}
+        async with aclosing(self._gql_items(op, kv, limit=limit)) as gen:
+            async for x in gen:
+                yield x
+
+    async def following(self, uid: int, limit=-1, kv: KV = None):
+        async with aclosing(self.following_raw(uid, limit=limit, kv=kv)) as gen:
+            async for rep in gen:
+                for x in parse_users(rep.json(), limit):
+                    yield x
+
+    # subscriptions
+
+    async def subscriptions_raw(self, uid: int, limit=-1, kv: KV = None):
+        op = OP_UserCreatorSubscriptions
+        kv = {"userId": str(uid), "count": 20, "includePromotedContent": False, **(kv or {})}
+        async with aclosing(self._gql_items(op, kv, limit=limit)) as gen:
+            async for x in gen:
+                yield x
+
+    async def subscriptions(self, uid: int, limit=-1, kv: KV = None):
+        async with aclosing(self.subscriptions_raw(uid, limit=limit, kv=kv)) as gen:
+            async for rep in gen:
+                for x in parse_users(rep.json(), limit):
+                    yield x
+
+    # retweeters
+
+    async def retweeters_raw(self, twid: int, limit=-1, kv: KV = None):
+        op = OP_Retweeters
+        kv = {"tweetId": str(twid), "count": 20, "includePromotedContent": True, **(kv or {})}
+        async with aclosing(self._gql_items(op, kv, limit=limit)) as gen:
+            async for x in gen:
+                yield x
+
+    async def retweeters(self, twid: int, limit=-1, kv: KV = None):
+        async with aclosing(self.retweeters_raw(twid, limit=limit, kv=kv)) as gen:
+            async for rep in gen:
+                for x in parse_users(rep.json(), limit):
+                    yield x
+
+    # user_tweets
+
+    async def user_tweets_raw(self, uid: int, limit=-1, kv: KV = None):
+        op = OP_UserTweets
+        kv = {
+            "userId": str(uid),
+            "count": 40,
+            "includePromotedContent": True,
+            "withQuickPromoteEligibilityTweetFields": True,
+            "withVoice": True,
+            "withV2Timeline": True,
+            **(kv or {}),
+        }
+        async with aclosing(self._gql_items(op, kv, limit=limit)) as gen:
+            async for x in gen:
+                yield x
+
+    async def user_tweets(self, uid: int, limit=-1, kv: KV = None):
+        async with aclosing(self.user_tweets_raw(uid, limit=limit, kv=kv)) as gen:
+            async for rep in gen:
+                for x in parse_tweets(rep.json(), limit):
+                    yield x
+
+    # user_tweets_and_replies
+
+    async def user_tweets_and_replies_raw(self, uid: int, limit=-1, kv: KV = None):
+        op = OP_UserTweetsAndReplies
+        kv = {
+            "userId": str(uid),
+            "count": 40,
+            "includePromotedContent": True,
+            "withCommunity": True,
+            "withVoice": True,
+            "withV2Timeline": True,
+            **(kv or {}),
+        }
+        async with aclosing(self._gql_items(op, kv, limit=limit)) as gen:
+            async for x in gen:
+                yield x
+
+    async def user_tweets_and_replies(self, uid: int, limit=-1, kv: KV = None):
+        async with aclosing(self.user_tweets_and_replies_raw(uid, limit=limit, kv=kv)) as gen:
+            async for rep in gen:
+                for x in parse_tweets(rep.json(), limit):
+                    yield x
+
+    # user_media
+
+    async def user_media_raw(self, uid: int, limit=-1, kv: KV = None):
+        op = OP_UserMedia
+        kv = {
+            "userId": str(uid),
+            "count": 40,
+            "includePromotedContent": False,
+            "withClientEventToken": False,
+            "withBirdwatchNotes": False,
+            "withVoice": True,
+            "withV2Timeline": True,
+            **(kv or {}),
+        }
+
+        async with aclosing(self._gql_items(op, kv, limit=limit)) as gen:
+            async for x in gen:
+                yield x
+
+    async def user_media(self, uid: int, limit=-1, kv: KV = None):
+        async with aclosing(self.user_media_raw(uid, limit=limit, kv=kv)) as gen:
+            async for rep in gen:
+                for x in parse_tweets(rep, limit):
+                    # sometimes some tweets without media, so skip them
+                    media_count = (
+                        len(x.media.photos) + len(x.media.videos) + len(x.media.animated)
+                        if x.media
+                        else 0
+                    )
+
+                    if media_count > 0:
+                        yield x
+
+    # list_timeline
+
+    async def list_timeline_raw(self, list_id: int, limit=-1, kv: KV = None):
+        op = OP_ListLatestTweetsTimeline
+        kv = {"listId": str(list_id), "count": 20, **(kv or {})}
+        async with aclosing(self._gql_items(op, kv, limit=limit)) as gen:
+            async for x in gen:
+                yield x
+
+    async def list_timeline(self, list_id: int, limit=-1, kv: KV = None):
+        async with aclosing(self.list_timeline_raw(list_id, limit=limit, kv=kv)) as gen:
+            async for rep in gen:
+                for x in parse_tweets(rep, limit):
+                    yield x
+
+    # trends
+
+    async def trends_raw(self, trend_id: TrendId, limit=-1, kv: KV = None):
+        map = {
+            "trending": "VGltZWxpbmU6DAC2CwABAAAACHRyZW5kaW5nAAA",
+            "news": "VGltZWxpbmU6DAC2CwABAAAABG5ld3MAAA",
+            "sport": "VGltZWxpbmU6DAC2CwABAAAABnNwb3J0cwAA",
+            "entertainment": "VGltZWxpbmU6DAC2CwABAAAADWVudGVydGFpbm1lbnQAAA",
+        }
+        trend_id = map.get(trend_id, trend_id)
+
+        op = OP_GenericTimelineById
+        kv = {
+            "timelineId": trend_id,
+            "count": 20,
+            "withQuickPromoteEligibilityTweetFields": True,
+            **(kv or {}),
+        }
+        async with aclosing(self._gql_items(op, kv, limit=limit)) as gen:
+            async for x in gen:
+                yield x
+
+    async def trends(self, trend_id: TrendId, limit=-1, kv: KV = None):
+        async with aclosing(self.trends_raw(trend_id, limit=limit, kv=kv)) as gen:
+            async for rep in gen:
+                for x in parse_trends(rep, limit):
+                    yield x
+
+    async def search_trend(self, q: str, limit=-1, kv: KV = None):
+        kv = {
+            "querySource": "trend_click",
+            **(kv or {}),
+        }
+        async with aclosing(self.search_raw(q, limit=limit, kv=kv)) as gen:
+            async for rep in gen:
+                for x in parse_tweets(rep.json(), limit):
+                    yield x
+
+    # Get current user bookmarks
+
+    async def bookmarks_raw(self, limit=-1, kv: KV = None):
+        op = OP_Bookmarks
+        kv = {
+            "count": 20,
+            "includePromotedContent": False,
+            "withClientEventToken": False,
+            "withBirdwatchNotes": False,
+            "withVoice": True,
+            "withV2Timeline": True,
+            **(kv or {}),
+        }
+        ft = {
+            "graphql_timeline_v2_bookmark_timeline": True,
+        }
+        async with aclosing(self._gql_items(op, kv, ft, limit=limit)) as gen:
+            async for x in gen:
+                yield x
+
+    async def bookmarks(self, limit=-1, kv: KV = None):
+        async with aclosing(self.bookmarks_raw(limit=limit, kv=kv)) as gen:
+            async for rep in gen:
+                for x in parse_tweets(rep.json(), limit):
+                    yield x

apps/twscrape/twscrape/cli.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+
+import argparse
+import asyncio
+import io
+import json
+import sqlite3
+from importlib.metadata import version
+
+import httpx
+
+from .api import API, AccountsPool
+from .db import get_sqlite_version
+from .logger import logger, set_log_level
+from .login import LoginConfig
+from .models import Tweet, User
+from .utils import print_table
+
+
+class CustomHelpFormatter(argparse.HelpFormatter):
+    def __init__(self, prog):
+        super().__init__(prog, max_help_position=30, width=120)
+
+
+def get_fn_arg(args):
+    names = ["query", "tweet_id", "user_id", "username", "list_id", "trend_id"]
+    for name in names:
+        if name in args:
+            return name, getattr(args, name)
+
+    logger.error(f"Missing argument: {names}")
+    exit(1)
+
+
+def to_str(doc: httpx.Response | Tweet | User | None) -> str:
+    if doc is None:
+        return "Not Found. See --raw for more details."
+
+    tmp = doc.json()
+    return tmp if isinstance(tmp, str) else json.dumps(tmp, default=str)
+
+
+async def main(args):
+    if args.debug:
+        set_log_level("DEBUG")
+
+    if args.command == "version":
+        print(f"twscrape: {version('twscrape')}")
+        print(f"SQLite runtime: {sqlite3.sqlite_version} ({await get_sqlite_version()})")
+        return
+
+    login_config = LoginConfig(getattr(args, "email_first", False), getattr(args, "manual", False))
+    pool = AccountsPool(args.db, login_config=login_config)
+    api = API(pool, debug=args.debug)
+
+    if args.command == "accounts":
+        print_table([dict(x) for x in await pool.accounts_info()])
+        return
+
+    if args.command == "stats":
+        rep = await pool.stats()
+        total, active, inactive = rep["total"], rep["active"], rep["inactive"]
+
+        res = []
+        for k, v in rep.items():
+            if not k.startswith("locked") or v == 0:
+                continue
+            res.append({"queue": k, "locked": v, "available": max(active - v, 0)})
+
+        res = sorted(res, key=lambda x: x["locked"], reverse=True)
+        print_table(res, hr_after=True)
+        print(f"Total: {total} - Active: {active} - Inactive: {inactive}")
+        return
+
+    if args.command == "add_accounts":
+        await pool.load_from_file(args.file_path, args.line_format)
+        print("\nNow run:\ntwscrape login_accounts")
+        return
+
+    if args.command == "del_accounts":
+        await pool.delete_accounts(args.usernames)
+        return
+
+    if args.command == "login_accounts":
+        stats = await pool.login_all()
+        print(stats)
+        return
+
+    if args.command == "relogin_failed":
+        await pool.relogin_failed()
+        return
+
+    if args.command == "relogin":
+        await pool.relogin(args.usernames)
+        return
+
+    if args.command == "reset_locks":
+        await pool.reset_locks()
+        return
+
+    if args.command == "delete_inactive":
+        await pool.delete_inactive()
+        return
+
+    fn = args.command + "_raw" if args.raw else args.command
+    fn = getattr(api, fn, None)
+    if fn is None:
+        logger.error(f"Unknown command: {args.command}")
+        exit(1)
+
+    _, val = get_fn_arg(args)
+
+    if "limit" in args:
+        async for doc in fn(val, limit=args.limit):
+            print(to_str(doc))
+    else:
+        doc = await fn(val)
+        print(to_str(doc))
+
+
+def custom_help(p):
+    buffer = io.StringIO()
+    p.print_help(buffer)
+    msg = buffer.getvalue()
+
+    cmd = msg.split("positional arguments:")[1].strip().split("\n")[0]
+    msg = msg.replace("positional arguments:", "commands:")
+    msg = [x for x in msg.split("\n") if cmd not in x and "..." not in x]
+    msg[0] = f"{msg[0]} <command> [...]"
+
+    i = 0
+    for i, line in enumerate(msg):
+        if line.strip().startswith("search"):
+            break
+
+    msg.insert(i, "")
+    msg.insert(i + 1, "search commands:")
+
+    print("\n".join(msg))
+
+
+def run():
+    p = argparse.ArgumentParser(add_help=False, formatter_class=CustomHelpFormatter)
+    p.add_argument("--db", default="accounts.db", help="Accounts database file")
+    p.add_argument("--debug", action="store_true", help="Enable debug mode")
+    subparsers = p.add_subparsers(dest="command")
+
+    def c_one(name: str, msg: str, a_name: str, a_msg: str, a_type: type = str):
+        p = subparsers.add_parser(name, help=msg)
+        p.add_argument(a_name, help=a_msg, type=a_type)
+        p.add_argument("--raw", action="store_true", help="Print raw response")
+        return p
+
+    def c_lim(name: str, msg: str, a_name: str, a_msg: str, a_type: type = str):
+        p = c_one(name, msg, a_name, a_msg, a_type)
+        p.add_argument("--limit", type=int, default=-1, help="Max tweets to retrieve")
+        return p
+
+    subparsers.add_parser("version", help="Show version")
+    subparsers.add_parser("accounts", help="List all accounts")
+    subparsers.add_parser("stats", help="Get current usage stats")
+
+    add_accounts = subparsers.add_parser("add_accounts", help="Add accounts")
+    add_accounts.add_argument("file_path", help="File with accounts")
+    add_accounts.add_argument("line_format", help="args of Pool.add_account splited by same delim")
+
+    del_accounts = subparsers.add_parser("del_accounts", help="Delete accounts")
+    del_accounts.add_argument("usernames", nargs="+", default=[], help="Usernames to delete")
+
+    login_cmd = subparsers.add_parser("login_accounts", help="Login accounts")
+    relogin = subparsers.add_parser("relogin", help="Re-login selected accounts")
+    relogin.add_argument("usernames", nargs="+", default=[], help="Usernames to re-login")
+    re_failed = subparsers.add_parser("relogin_failed", help="Retry login for failed accounts")
+
+    login_commands = [login_cmd, relogin, re_failed]
+    for cmd in login_commands:
+        cmd.add_argument("--email-first", action="store_true", help="Check email first")
+        cmd.add_argument("--manual", action="store_true", help="Enter email code manually")
+
+    subparsers.add_parser("reset_locks", help="Reset all locks")
+    subparsers.add_parser("delete_inactive", help="Delete inactive accounts")
+
+    c_lim("search", "Search for tweets", "query", "Search query")
+    c_one("tweet_details", "Get tweet details", "tweet_id", "Tweet ID", int)
+    c_lim("tweet_replies", "Get replies  of a tweet", "tweet_id", "Tweet ID", int)
+    c_lim("retweeters", "Get retweeters of a tweet", "tweet_id", "Tweet ID", int)
+    c_one("user_by_id", "Get user data by ID", "user_id", "User ID", int)
+    c_one("user_by_login", "Get user data by username", "username", "Username")
+    c_lim("following", "Get user following", "user_id", "User ID", int)
+    c_lim("followers", "Get user followers", "user_id", "User ID", int)
+    # https://x.com/xDaily/status/1701694747767648500
+    c_lim("verified_followers", "Get user verified followers", "user_id", "User ID", int)
+    c_lim("subscriptions", "Get user subscriptions", "user_id", "User ID", int)
+    c_lim("user_tweets", "Get user tweets", "user_id", "User ID", int)
+    c_lim("user_tweets_and_replies", "Get user tweets and replies", "user_id", "User ID", int)
+    c_lim("user_media", "Get user's media", "user_id", "User ID", int)
+    c_lim("list_timeline", "Get tweets from list", "list_id", "List ID", int)
+    c_lim("trends", "Get trends", "trend_id", "Trend ID or name", str)
+
+    args = p.parse_args()
+    if args.command is None:
+        return custom_help(p)
+
+    try:
+        asyncio.run(main(args))
+    except KeyboardInterrupt:
+        pass

apps/twscrape/twscrape/db.py

@@ -0,0 +1,158 @@
+import asyncio
+import random
+import sqlite3
+from collections import defaultdict
+
+import aiosqlite
+
+from .logger import logger
+
+MIN_SQLITE_VERSION = "3.24"
+
+_lock = asyncio.Lock()
+
+
+def lock_retry(max_retries=10):
+    # this lock decorator has double nature:
+    # 1. it uses asyncio lock in same process
+    # 2. it retries when db locked by other process (eg. two cli instances running)
+    def decorator(func):
+        async def wrapper(*args, **kwargs):
+            for i in range(max_retries):
+                try:
+                    async with _lock:
+                        return await func(*args, **kwargs)
+                except sqlite3.OperationalError as e:
+                    if i == max_retries - 1 or "database is locked" not in str(e):
+                        raise e
+
+                    await asyncio.sleep(random.uniform(0.5, 1.0))
+
+        return wrapper
+
+    return decorator
+
+
+async def get_sqlite_version():
+    async with aiosqlite.connect(":memory:") as db:
+        async with db.execute("SELECT SQLITE_VERSION()") as cur:
+            rs = await cur.fetchone()
+            return rs[0] if rs else "3.0.0"
+
+
+async def check_version():
+    ver = await get_sqlite_version()
+    ver = ".".join(ver.split(".")[:2])
+
+    try:
+        msg = f"SQLite version '{ver}' is too old, please upgrade to {MIN_SQLITE_VERSION}+"
+        if float(ver) < float(MIN_SQLITE_VERSION):
+            raise SystemError(msg)
+    except ValueError:
+        pass
+
+
+async def migrate(db: aiosqlite.Connection):
+    async with db.execute("PRAGMA user_version") as cur:
+        rs = await cur.fetchone()
+        uv = rs[0] if rs else 0
+
+    async def v1():
+        qs = """
+        CREATE TABLE IF NOT EXISTS accounts (
+            username TEXT PRIMARY KEY NOT NULL COLLATE NOCASE,
+            password TEXT NOT NULL,
+            email TEXT NOT NULL COLLATE NOCASE,
+            email_password TEXT NOT NULL,
+            user_agent TEXT NOT NULL,
+            active BOOLEAN DEFAULT FALSE NOT NULL,
+            locks TEXT DEFAULT '{}' NOT NULL,
+            headers TEXT DEFAULT '{}' NOT NULL,
+            cookies TEXT DEFAULT '{}' NOT NULL,
+            proxy TEXT DEFAULT NULL,
+            error_msg TEXT DEFAULT NULL
+        );"""
+        await db.execute(qs)
+
+    async def v2():
+        await db.execute("ALTER TABLE accounts ADD COLUMN stats TEXT DEFAULT '{}' NOT NULL")
+        await db.execute("ALTER TABLE accounts ADD COLUMN last_used TEXT DEFAULT NULL")
+
+    async def v3():
+        await db.execute("ALTER TABLE accounts ADD COLUMN _tx TEXT DEFAULT NULL")
+
+    async def v4():
+        await db.execute("ALTER TABLE accounts ADD COLUMN mfa_code TEXT DEFAULT NULL")
+
+    migrations = {
+        1: v1,
+        2: v2,
+        3: v3,
+        4: v4,
+    }
+
+    # logger.debug(f"Current migration v{uv} (latest v{len(migrations)})")
+    for i in range(uv + 1, len(migrations) + 1):
+        logger.info(f"Running migration to v{i}")
+        try:
+            await migrations[i]()
+        except sqlite3.OperationalError as e:
+            if "duplicate column name" not in str(e):
+                raise e
+
+        await db.execute(f"PRAGMA user_version = {i}")
+        await db.commit()
+
+
+class DB:
+    _init_queries: defaultdict[str, list[str]] = defaultdict(list)
+    _init_once: defaultdict[str, bool] = defaultdict(bool)
+
+    def __init__(self, db_path):
+        self.db_path = db_path
+        self.conn = None
+
+    async def __aenter__(self):
+        await check_version()
+        db = await aiosqlite.connect(self.db_path)
+        db.row_factory = aiosqlite.Row
+
+        if not self._init_once[self.db_path]:
+            await migrate(db)
+            self._init_once[self.db_path] = True
+
+        self.conn = db
+        return db
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self.conn:
+            await self.conn.commit()
+            await self.conn.close()
+
+
+@lock_retry()
+async def execute(db_path: str, qs: str, params: dict | None = None):
+    async with DB(db_path) as db:
+        await db.execute(qs, params)
+
+
+@lock_retry()
+async def fetchone(db_path: str, qs: str, params: dict | None = None):
+    async with DB(db_path) as db:
+        async with db.execute(qs, params) as cur:
+            row = await cur.fetchone()
+            return row
+
+
+@lock_retry()
+async def fetchall(db_path: str, qs: str, params: dict | None = None):
+    async with DB(db_path) as db:
+        async with db.execute(qs, params) as cur:
+            rows = await cur.fetchall()
+            return rows
+
+
+@lock_retry()
+async def executemany(db_path: str, qs: str, params: list[dict]):
+    async with DB(db_path) as db:
+        await db.executemany(qs, params)

apps/twscrape/twscrape/imap.py

@@ -0,0 +1,111 @@
+import asyncio
+import email as emaillib
+import imaplib
+import os
+import time
+from datetime import datetime
+
+from .logger import logger
+
+
+def env_int(key: str | list[str], default: int) -> int:
+    key = [key] if isinstance(key, str) else key
+    val = [os.getenv(k) for k in key]
+    val = [int(x) for x in val if x is not None]
+    return val[0] if val else default
+
+
+TWS_WAIT_EMAIL_CODE = env_int(["TWS_WAIT_EMAIL_CODE", "LOGIN_CODE_TIMEOUT"], 30)
+
+
+class EmailLoginError(Exception):
+    def __init__(self, message="Email login error"):
+        self.message = message
+        super().__init__(self.message)
+
+
+class EmailCodeTimeoutError(Exception):
+    def __init__(self, message="Email code timeout"):
+        self.message = message
+        super().__init__(self.message)
+
+
+IMAP_MAPPING: dict[str, str] = {
+    "yahoo.com": "imap.mail.yahoo.com",
+    "icloud.com": "imap.mail.me.com",
+    "outlook.com": "imap-mail.outlook.com",
+    "hotmail.com": "imap-mail.outlook.com",
+}
+
+
+def add_imap_mapping(email_domain: str, imap_domain: str):
+    IMAP_MAPPING[email_domain] = imap_domain
+
+
+def _get_imap_domain(email: str) -> str:
+    email_domain = email.split("@")[1]
+    if email_domain in IMAP_MAPPING:
+        return IMAP_MAPPING[email_domain]
+    return f"imap.{email_domain}"
+
+
+def _wait_email_code(imap: imaplib.IMAP4_SSL, count: int, min_t: datetime | None) -> str | None:
+    for i in range(count, 0, -1):
+        _, rep = imap.fetch(str(i), "(RFC822)")
+        for x in rep:
+            if isinstance(x, tuple):
+                msg = emaillib.message_from_bytes(x[1])
+
+                # https://www.ietf.org/rfc/rfc9051.html#section-6.3.12-13
+                msg_time = msg.get("Date", "").split("(")[0].strip()
+                msg_time = datetime.strptime(msg_time, "%a, %d %b %Y %H:%M:%S %z")
+
+                msg_from = str(msg.get("From", "")).lower()
+                msg_subj = str(msg.get("Subject", "")).lower()
+                logger.info(f"({i} of {count}) {msg_from} - {msg_time} - {msg_subj}")
+
+                if min_t is not None and msg_time < min_t:
+                    return None
+
+                if "info@x.com" in msg_from and "confirmation code is" in msg_subj:
+                    # eg. Your Twitter confirmation code is XXX
+                    return msg_subj.split(" ")[-1].strip()
+
+    return None
+
+
+async def imap_get_email_code(
+    imap: imaplib.IMAP4_SSL, email: str, min_t: datetime | None = None
+) -> str:
+    try:
+        logger.info(f"Waiting for confirmation code for {email}...")
+        start_time = time.time()
+        while True:
+            _, rep = imap.select("INBOX")
+            msg_count = int(rep[0].decode("utf-8")) if len(rep) > 0 and rep[0] is not None else 0
+            code = _wait_email_code(imap, msg_count, min_t)
+            if code is not None:
+                return code
+
+            if TWS_WAIT_EMAIL_CODE < time.time() - start_time:
+                raise EmailCodeTimeoutError(f"Email code timeout ({TWS_WAIT_EMAIL_CODE} sec)")
+
+            await asyncio.sleep(5)
+    except Exception as e:
+        imap.select("INBOX")
+        imap.close()
+        raise e
+
+
+async def imap_login(email: str, password: str):
+    domain = _get_imap_domain(email)
+    imap = imaplib.IMAP4_SSL(domain)
+
+    try:
+        imap.login(email, password)
+        imap.select("INBOX", readonly=True)
+    except imaplib.IMAP4.error as e:
+        logger.error(f"Error logging into {email} on {domain}: {e}")
+        raise EmailLoginError() from e
+
+    return imap

apps/twscrape/twscrape/logger.py

@@ -0,0 +1,20 @@
+import sys
+from typing import Literal
+
+from loguru import logger
+
+_LEVELS = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
+_LOG_LEVEL: _LEVELS = "INFO"
+
+
+def set_log_level(level: _LEVELS):
+    global _LOG_LEVEL
+    _LOG_LEVEL = level
+
+
+def _filter(r):
+    return r["level"].no >= logger.level(_LOG_LEVEL).no
+
+
+logger.remove()
+logger.add(sys.stderr, filter=_filter)

apps/twscrape/twscrape/login.py

@@ -0,0 +1,279 @@
+import imaplib
+from dataclasses import dataclass
+from datetime import timedelta
+from typing import Any
+
+import pyotp
+from httpx import AsyncClient, Response
+
+from .account import Account
+from .imap import imap_get_email_code, imap_login
+from .logger import logger
+from .utils import utc
+
+LOGIN_URL = "https://api.x.com/1.1/onboarding/task.json"
+
+
+@dataclass
+class LoginConfig:
+    email_first: bool = False
+    manual: bool = False
+
+
+@dataclass
+class TaskCtx:
+    client: AsyncClient
+    acc: Account
+    cfg: LoginConfig
+    prev: Any
+    imap: None | imaplib.IMAP4_SSL
+
+
+async def get_guest_token(client: AsyncClient):
+    rep = await client.post("https://api.x.com/1.1/guest/activate.json")
+    rep.raise_for_status()
+    return rep.json()["guest_token"]
+
+
+async def login_initiate(client: AsyncClient) -> Response:
+    payload = {
+        "input_flow_data": {
+            "flow_context": {"debug_overrides": {}, "start_location": {"location": "unknown"}}
+        },
+        "subtask_versions": {},
+    }
+
+    rep = await client.post(LOGIN_URL, params={"flow_name": "login"}, json=payload)
+    rep.raise_for_status()
+    return rep
+
+
+async def login_alternate_identifier(ctx: TaskCtx, *, username: str) -> Response:
+    payload = {
+        "flow_token": ctx.prev["flow_token"],
+        "subtask_inputs": [
+            {
+                "subtask_id": "LoginEnterAlternateIdentifierSubtask",
+                "enter_text": {"text": username, "link": "next_link"},
+            }
+        ],
+    }
+
+    rep = await ctx.client.post(LOGIN_URL, json=payload)
+    rep.raise_for_status()
+    return rep
+
+
+async def login_instrumentation(ctx: TaskCtx) -> Response:
+    payload = {
+        "flow_token": ctx.prev["flow_token"],
+        "subtask_inputs": [
+            {
+                "subtask_id": "LoginJsInstrumentationSubtask",
+                "js_instrumentation": {"response": "{}", "link": "next_link"},
+            }
+        ],
+    }
+
+    rep = await ctx.client.post(LOGIN_URL, json=payload)
+    rep.raise_for_status()
+    return rep
+
+
+async def login_enter_username(ctx: TaskCtx) -> Response:
+    payload = {
+        "flow_token": ctx.prev["flow_token"],
+        "subtask_inputs": [
+            {
+                "subtask_id": "LoginEnterUserIdentifierSSO",
+                "settings_list": {
+                    "setting_responses": [
+                        {
+                            "key": "user_identifier",
+                            "response_data": {"text_data": {"result": ctx.acc.username}},
+                        }
+                    ],
+                    "link": "next_link",
+                },
+            }
+        ],
+    }
+
+    rep = await ctx.client.post(LOGIN_URL, json=payload)
+    rep.raise_for_status()
+    return rep
+
+
+async def login_enter_password(ctx: TaskCtx) -> Response:
+    payload = {
+        "flow_token": ctx.prev["flow_token"],
+        "subtask_inputs": [
+            {
+                "subtask_id": "LoginEnterPassword",
+                "enter_password": {"password": ctx.acc.password, "link": "next_link"},
+            }
+        ],
+    }
+
+    rep = await ctx.client.post(LOGIN_URL, json=payload)
+    rep.raise_for_status()
+    return rep
+
+
+async def login_two_factor_auth_challenge(ctx: TaskCtx) -> Response:
+    if ctx.acc.mfa_code is None:
+        raise ValueError("MFA code is required")
+
+    totp = pyotp.TOTP(ctx.acc.mfa_code)
+    payload = {
+        "flow_token": ctx.prev["flow_token"],
+        "subtask_inputs": [
+            {
+                "subtask_id": "LoginTwoFactorAuthChallenge",
+                "enter_text": {"text": totp.now(), "link": "next_link"},
+            }
+        ],
+    }
+
+    rep = await ctx.client.post(LOGIN_URL, json=payload)
+    rep.raise_for_status()
+    return rep
+
+
+async def login_duplication_check(ctx: TaskCtx) -> Response:
+    payload = {
+        "flow_token": ctx.prev["flow_token"],
+        "subtask_inputs": [
+            {
+                "subtask_id": "AccountDuplicationCheck",
+                "check_logged_in_account": {"link": "AccountDuplicationCheck_false"},
+            }
+        ],
+    }
+
+    rep = await ctx.client.post(LOGIN_URL, json=payload)
+    rep.raise_for_status()
+    return rep
+
+
+async def login_confirm_email(ctx: TaskCtx) -> Response:
+    payload = {
+        "flow_token": ctx.prev["flow_token"],
+        "subtask_inputs": [
+            {
+                "subtask_id": "LoginAcid",
+                "enter_text": {"text": ctx.acc.email, "link": "next_link"},
+            }
+        ],
+    }
+
+    rep = await ctx.client.post(LOGIN_URL, json=payload)
+    rep.raise_for_status()
+    return rep
+
+
+async def login_confirm_email_code(ctx: TaskCtx):
+    if ctx.cfg.manual:
+        print(f"Enter email code for {ctx.acc.username} / {ctx.acc.email}")
+        value = input("Code: ")
+        value = value.strip()
+    else:
+        if not ctx.imap:
+            ctx.imap = await imap_login(ctx.acc.email, ctx.acc.email_password)
+
+        now_time = utc.now() - timedelta(seconds=30)
+        value = await imap_get_email_code(ctx.imap, ctx.acc.email, now_time)
+
+    payload = {
+        "flow_token": ctx.prev["flow_token"],
+        "subtask_inputs": [
+            {
+                "subtask_id": "LoginAcid",
+                "enter_text": {"text": value, "link": "next_link"},
+            }
+        ],
+    }
+
+    rep = await ctx.client.post(LOGIN_URL, json=payload)
+    rep.raise_for_status()
+    return rep
+
+
+async def login_success(ctx: TaskCtx) -> Response:
+    payload = {
+        "flow_token": ctx.prev["flow_token"],
+        "subtask_inputs": [],
+    }
+
+    rep = await ctx.client.post(LOGIN_URL, json=payload)
+    rep.raise_for_status()
+    return rep
+
+
+async def next_login_task(ctx: TaskCtx, rep: Response):
+    ct0 = ctx.client.cookies.get("ct0", None)
+    if ct0:
+        ctx.client.headers["x-csrf-token"] = ct0
+        ctx.client.headers["x-twitter-auth-type"] = "OAuth2Session"
+
+    ctx.prev = rep.json()
+    assert "flow_token" in ctx.prev, f"flow_token not in {rep.text}"
+
+    for x in ctx.prev["subtasks"]:
+        task_id = x["subtask_id"]
+
+        try:
+            if task_id == "LoginSuccessSubtask":
+                return await login_success(ctx)
+            if task_id == "LoginAcid":
+                is_code = x["enter_text"]["hint_text"].lower() == "confirmation code"
+                fn = login_confirm_email_code if is_code else login_confirm_email
+                return await fn(ctx)
+            if task_id == "AccountDuplicationCheck":
+                return await login_duplication_check(ctx)
+            if task_id == "LoginEnterPassword":
+                return await login_enter_password(ctx)
+            if task_id == "LoginTwoFactorAuthChallenge":
+                return await login_two_factor_auth_challenge(ctx)
+            if task_id == "LoginEnterUserIdentifierSSO":
+                return await login_enter_username(ctx)
+            if task_id == "LoginJsInstrumentationSubtask":
+                return await login_instrumentation(ctx)
+            if task_id == "LoginEnterAlternateIdentifierSubtask":
+                return await login_alternate_identifier(ctx, username=ctx.acc.username)
+        except Exception as e:
+            ctx.acc.error_msg = f"login_step={task_id} err={e}"
+            raise e
+
+    return None
+
+
+async def login(acc: Account, cfg: LoginConfig | None = None) -> Account:
+    log_id = f"{acc.username} - {acc.email}"
+    if acc.active:
+        logger.info(f"account already active {log_id}")
+        return acc
+
+    cfg, imap = cfg or LoginConfig(), None
+    if cfg.email_first and not cfg.manual:
+        imap = await imap_login(acc.email, acc.email_password)
+
+    async with acc.make_client() as client:
+        guest_token = await get_guest_token(client)
+        client.headers["x-guest-token"] = guest_token
+
+        rep = await login_initiate(client)
+        ctx = TaskCtx(client, acc, cfg, None, imap)
+        while True:
+            rep = await next_login_task(ctx, rep)
+            if not rep:
+                break
+
+        assert "ct0" in client.cookies, "ct0 not in cookies (most likely ip ban)"
+        client.headers["x-csrf-token"] = client.cookies["ct0"]
+        client.headers["x-twitter-auth-type"] = "OAuth2Session"
+
+        acc.active = True
+        acc.headers = {k: v for k, v in client.headers.items()}
+        acc.cookies = {k: v for k, v in client.cookies.items()}
+        return acc

apps/twscrape/twscrape/models.py

@@ -0,0 +1,780 @@
+import email.utils
+import json
+import os
+import random
+import re
+import string
+import sys
+import traceback
+from dataclasses import asdict, dataclass, field
+from datetime import datetime
+from typing import Generator, Optional, Union
+
+import httpx
+
+from .logger import logger
+from .utils import find_item, get_or, int_or, to_old_rep, utc
+
+
+@dataclass
+class JSONTrait:
+    def dict(self):
+        return asdict(self)
+
+    def json(self):
+        return json.dumps(self.dict(), default=str)
+
+
+@dataclass
+class Coordinates(JSONTrait):
+    longitude: float
+    latitude: float
+
+    @staticmethod
+    def parse(tw_obj: dict):
+        if tw_obj.get("coordinates"):
+            coords = tw_obj["coordinates"]["coordinates"]
+            return Coordinates(coords[0], coords[1])
+        if tw_obj.get("geo"):
+            coords = tw_obj["geo"]["coordinates"]
+            return Coordinates(coords[1], coords[0])
+        return None
+
+
+@dataclass
+class Place(JSONTrait):
+    id: str
+    fullName: str
+    name: str
+    type: str
+    country: str
+    countryCode: str
+
+    @staticmethod
+    def parse(obj: dict):
+        return Place(
+            id=obj["id"],
+            fullName=obj["full_name"],
+            name=obj["name"],
+            type=obj["place_type"],
+            country=obj["country"],
+            countryCode=obj["country_code"],
+        )
+
+
+@dataclass
+class TextLink(JSONTrait):
+    url: str
+    text: str | None
+    tcourl: str | None
+
+    @staticmethod
+    def parse(obj: dict):
+        tmp = TextLink(
+            url=obj.get("expanded_url", None),
+            text=obj.get("display_url", None),
+            tcourl=obj.get("url", None),
+        )
+
+        if tmp.url is None or tmp.tcourl is None:
+            return None
+
+        return tmp
+
+
+@dataclass
+class UserRef(JSONTrait):
+    id: int
+    id_str: str
+    username: str
+    displayname: str
+    _type: str = "snscrape.modules.twitter.UserRef"
+
+    @staticmethod
+    def parse(obj: dict):
+        return UserRef(
+            id=int(obj["id_str"]),
+            id_str=obj["id_str"],
+            username=obj["screen_name"],
+            displayname=obj["name"],
+        )
+
+
+@dataclass
+class User(JSONTrait):
+    id: int
+    id_str: str
+    url: str
+    username: str
+    displayname: str
+    rawDescription: str
+    created: datetime
+    followersCount: int
+    friendsCount: int
+    statusesCount: int
+    favouritesCount: int
+    listedCount: int
+    mediaCount: int
+    location: str
+    profileImageUrl: str
+    profileBannerUrl: str | None = None
+    protected: bool | None = None
+    verified: bool | None = None
+    blue: bool | None = None
+    blueType: str | None = None
+    descriptionLinks: list[TextLink] = field(default_factory=list)
+    pinnedIds: list[int] = field(default_factory=list)
+    _type: str = "snscrape.modules.twitter.User"
+
+    # todo:
+    # link: typing.Optional[TextLink] = None
+    # label: typing.Optional["UserLabel"] = None
+
+    @staticmethod
+    def parse(obj: dict, res=None):
+        return User(
+            id=int(obj["id_str"]),
+            id_str=obj["id_str"],
+            url=f"https://x.com/{obj['screen_name']}",
+            username=obj["screen_name"],
+            displayname=obj["name"],
+            rawDescription=obj["description"],
+            created=email.utils.parsedate_to_datetime(obj["created_at"]),
+            followersCount=obj["followers_count"],
+            friendsCount=obj["friends_count"],
+            statusesCount=obj["statuses_count"],
+            favouritesCount=obj["favourites_count"],
+            listedCount=obj["listed_count"],
+            mediaCount=obj["media_count"],
+            location=obj["location"],
+            profileImageUrl=obj["profile_image_url_https"],
+            profileBannerUrl=obj.get("profile_banner_url"),
+            verified=obj.get("verified"),
+            blue=obj.get("is_blue_verified"),
+            blueType=obj.get("verified_type"),
+            protected=obj.get("protected"),
+            descriptionLinks=_parse_links(obj, ["entities.description.urls", "entities.url.urls"]),
+            pinnedIds=[int(x) for x in obj.get("pinned_tweet_ids_str", [])],
+        )
+
+
+@dataclass
+class Tweet(JSONTrait):
+    id: int
+    id_str: str
+    url: str
+    date: datetime
+    user: User
+    lang: str
+    rawContent: str
+    replyCount: int
+    retweetCount: int
+    likeCount: int
+    quoteCount: int
+    bookmarkedCount: int
+    conversationId: int
+    conversationIdStr: str
+    hashtags: list[str]
+    cashtags: list[str]
+    mentionedUsers: list[UserRef]
+    links: list[TextLink]
+    media: "Media"
+    viewCount: int | None = None
+    retweetedTweet: Optional["Tweet"] = None
+    quotedTweet: Optional["Tweet"] = None
+    place: Optional[Place] = None
+    coordinates: Optional[Coordinates] = None
+    inReplyToTweetId: int | None = None
+    inReplyToTweetIdStr: str | None = None
+    inReplyToUser: UserRef | None = None
+    source: str | None = None
+    sourceUrl: str | None = None
+    sourceLabel: str | None = None
+    card: Union[None, "SummaryCard", "PollCard", "BroadcastCard", "AudiospaceCard"] = None
+    possibly_sensitive: bool | None = None
+    _type: str = "snscrape.modules.twitter.Tweet"
+
+    # todo:
+    # renderedContent: str
+    # vibe: Optional["Vibe"] = None
+
+    @staticmethod
+    def parse(obj: dict, res: dict):
+        tw_usr = User.parse(res["users"][obj["user_id_str"]])
+
+        rt_id_path = [
+            "retweeted_status_id_str",
+            "retweeted_status_result.result.rest_id",
+            "retweeted_status_result.result.tweet.rest_id",
+        ]
+
+        qt_id_path = [
+            "quoted_status_id_str",
+            "quoted_status_result.result.rest_id",
+            "quoted_status_result.result.tweet.rest_id",
+        ]
+
+        rt_obj = get_or(res, f"tweets.{_first(obj, rt_id_path)}")
+        qt_obj = get_or(res, f"tweets.{_first(obj, qt_id_path)}")
+
+        url = f"https://x.com/{tw_usr.username}/status/{obj['id_str']}"
+        doc = Tweet(
+            id=int(obj["id_str"]),
+            id_str=obj["id_str"],
+            url=url,
+            date=email.utils.parsedate_to_datetime(obj["created_at"]),
+            user=tw_usr,
+            lang=obj["lang"],
+            rawContent=get_or(obj, "note_tweet.note_tweet_results.result.text", obj["full_text"]),
+            replyCount=obj["reply_count"],
+            retweetCount=obj["retweet_count"],
+            likeCount=obj["favorite_count"],
+            quoteCount=obj["quote_count"],
+            bookmarkedCount=get_or(obj, "bookmark_count", 0),
+            conversationId=int(obj["conversation_id_str"]),
+            conversationIdStr=obj["conversation_id_str"],
+            hashtags=[x["text"] for x in get_or(obj, "entities.hashtags", [])],
+            cashtags=[x["text"] for x in get_or(obj, "entities.symbols", [])],
+            mentionedUsers=[UserRef.parse(x) for x in get_or(obj, "entities.user_mentions", [])],
+            links=_parse_links(
+                obj, ["entities.urls", "note_tweet.note_tweet_results.result.entity_set.urls"]
+            ),
+            viewCount=_get_views(obj, rt_obj or {}),
+            retweetedTweet=Tweet.parse(rt_obj, res) if rt_obj else None,
+            quotedTweet=Tweet.parse(qt_obj, res) if qt_obj else None,
+            place=Place.parse(obj["place"]) if obj.get("place") else None,
+            coordinates=Coordinates.parse(obj),
+            inReplyToTweetId=int_or(obj, "in_reply_to_status_id_str"),
+            inReplyToTweetIdStr=get_or(obj, "in_reply_to_status_id_str"),
+            inReplyToUser=_get_reply_user(obj, res),
+            source=obj.get("source", None),
+            sourceUrl=_get_source_url(obj),
+            sourceLabel=_get_source_label(obj),
+            media=Media.parse(obj),
+            card=_parse_card(obj, url),
+            possibly_sensitive=obj.get("possibly_sensitive", None),
+        )
+
+        # issue #42 – restore full rt text
+        rt = doc.retweetedTweet
+        if rt is not None and rt.user is not None and doc.rawContent.endswith("…"):
+            rt_msg = f"RT @{rt.user.username}: {rt.rawContent}"
+            if doc.rawContent != rt_msg:
+                doc.rawContent = rt_msg
+
+        return doc
+
+
+@dataclass
+class MediaPhoto(JSONTrait):
+    url: str
+
+    @staticmethod
+    def parse(obj: dict):
+        return MediaPhoto(url=obj["media_url_https"])
+
+
+@dataclass
+class MediaVideo(JSONTrait):
+    thumbnailUrl: str
+    variants: list["MediaVideoVariant"]
+    duration: int
+    views: int | None = None
+
+    @staticmethod
+    def parse(obj: dict):
+        return MediaVideo(
+            thumbnailUrl=obj["media_url_https"],
+            variants=[
+                MediaVideoVariant.parse(x) for x in obj["video_info"]["variants"] if "bitrate" in x
+            ],
+            duration=obj["video_info"]["duration_millis"],
+            views=int_or(obj, "mediaStats.viewCount"),
+        )
+
+
+@dataclass
+class MediaAnimated(JSONTrait):
+    thumbnailUrl: str
+    videoUrl: str
+
+    @staticmethod
+    def parse(obj: dict):
+        try:
+            return MediaAnimated(
+                thumbnailUrl=obj["media_url_https"],
+                videoUrl=obj["video_info"]["variants"][0]["url"],
+            )
+        except KeyError:
+            return None
+
+
+@dataclass
+class MediaVideoVariant(JSONTrait):
+    contentType: str
+    bitrate: int
+    url: str
+
+    @staticmethod
+    def parse(obj: dict):
+        return MediaVideoVariant(
+            contentType=obj["content_type"],
+            bitrate=obj["bitrate"],
+            url=obj["url"],
+        )
+
+
+@dataclass
+class Media(JSONTrait):
+    photos: list[MediaPhoto] = field(default_factory=list)
+    videos: list[MediaVideo] = field(default_factory=list)
+    animated: list[MediaAnimated] = field(default_factory=list)
+
+    @staticmethod
+    def parse(obj: dict):
+        photos: list[MediaPhoto] = []
+        videos: list[MediaVideo] = []
+        animated: list[MediaAnimated] = []
+
+        for x in get_or(obj, "extended_entities.media", []):
+            if x["type"] == "video":
+                if video := MediaVideo.parse(x):
+                    videos.append(video)
+                continue
+
+            if x["type"] == "photo":
+                if photo := MediaPhoto.parse(x):
+                    photos.append(photo)
+                continue
+
+            if x["type"] == "animated_gif":
+                if animated_gif := MediaAnimated.parse(x):
+                    animated.append(animated_gif)
+                continue
+
+            logger.warning(f"Unknown media type: {x['type']}: {json.dumps(x)}")
+
+        return Media(photos=photos, videos=videos, animated=animated)
+
+
+@dataclass
+class Card(JSONTrait):
+    pass
+
+
+@dataclass
+class SummaryCard(Card):
+    title: str
+    description: str
+    vanityUrl: str
+    url: str
+    photo: MediaPhoto | None = None
+    video: MediaVideo | None = None
+    _type: str = "summary"
+
+
+@dataclass
+class PollOption(JSONTrait):
+    label: str
+    votesCount: int
+
+
+@dataclass
+class PollCard(Card):
+    options: list[PollOption]
+    finished: bool
+    _type: str = "poll"
+
+
+@dataclass
+class BroadcastCard(Card):
+    title: str
+    url: str
+    photo: MediaPhoto | None = None
+    _type: str = "broadcast"
+
+
+@dataclass
+class AudiospaceCard(Card):
+    url: str
+    _type: str = "audiospace"
+
+
+@dataclass
+class RequestParam(JSONTrait):
+    key: str
+    value: str
+
+
+@dataclass
+class TrendUrl(JSONTrait):
+    url: str
+    urlType: str
+    urlEndpointOptions: list[RequestParam]
+
+    @staticmethod
+    def parse(obj: dict):
+        return TrendUrl(
+            url=obj["url"],
+            urlType=obj["urlType"],
+            urlEndpointOptions=[
+                RequestParam(key=x["key"], value=x["value"])
+                for x in obj["urtEndpointOptions"]["requestParams"]
+            ],
+        )
+
+
+@dataclass
+class TrendMetadata(JSONTrait):
+    domain_context: str
+    meta_description: str
+    url: TrendUrl
+
+    @staticmethod
+    def parse(obj: dict):
+        return TrendMetadata(
+            domain_context=obj["domain_context"],
+            meta_description=obj["meta_description"],
+            url=TrendUrl.parse(obj["url"]),
+        )
+
+
+@dataclass
+class GroupedTrend(JSONTrait):
+    name: str
+    url: TrendUrl
+
+    @staticmethod
+    def parse(obj: dict):
+        return GroupedTrend(name=obj["name"], url=TrendUrl.parse(obj["url"]))
+
+
+@dataclass
+class Trend(JSONTrait):
+    id: Optional[str]
+    rank: Optional[str | int]
+    name: str
+    trend_url: TrendUrl
+    trend_metadata: TrendMetadata
+    grouped_trends: list[GroupedTrend] = field(default_factory=list)
+    _type: str = "timelinetrend"
+
+    @staticmethod
+    def parse(obj: dict, res=None):
+        grouped_trends = [GroupedTrend.parse(x) for x in obj.get("grouped_trends", [])]
+        return Trend(
+            id=f"trend-{obj['name']}",
+            name=obj["name"],
+            rank=int(obj["rank"]) if "rank" in obj else None,
+            trend_url=TrendUrl.parse(obj["trend_url"]),
+            trend_metadata=TrendMetadata.parse(obj["trend_metadata"]),
+            grouped_trends=grouped_trends,
+        )
+
+
+def _parse_card_get_bool(values: list[dict], key: str):
+    for x in values:
+        if x["key"] == key:
+            return x["value"]["boolean_value"]
+    return False
+
+
+def _parse_card_get_str(values: list[dict], key: str, defaultVal=None) -> str | None:
+    for x in values:
+        if x["key"] == key:
+            return x["value"]["string_value"]
+    return defaultVal
+
+
+def _parse_card_extract_str(values: list[dict], key: str):
+    pretenders = [x["value"]["string_value"] for x in values if x["key"] == key]
+    new_values = [x for x in values if x["key"] != key]
+    return pretenders[0] if pretenders else "", new_values
+
+
+def _parse_card_extract_title(values: list[dict]):
+    new_values, pretenders = [], []
+    # title is trimmed to 70 chars, so try to find the longest text in alt_text
+    for x in values:
+        k = x["key"]
+        if k == "title" or k.endswith("_alt_text"):
+            pretenders.append(x["value"]["string_value"])
+        else:
+            new_values.append(x)
+
+    pretenders = sorted(pretenders, key=lambda x: len(x), reverse=True)
+    return pretenders[0] if pretenders else "", new_values
+
+
+def _parse_card_extract_largest_photo(values: list[dict]):
+    photos = [x for x in values if x["value"]["type"] == "IMAGE"]
+    photos = sorted(photos, key=lambda x: x["value"]["image_value"]["height"], reverse=True)
+    values = [x for x in values if x["value"]["type"] != "IMAGE"]
+    if photos:
+        return MediaPhoto(url=photos[0]["value"]["image_value"]["url"]), values
+    else:
+        return None, values
+
+
+def _parse_card_prepare_values(obj: dict):
+    values = get_or(obj, "card.legacy.binding_values", [])
+    # values = sorted(values, key=lambda x: x["key"])
+    # values = [x for x in values if x["key"] not in {"domain", "creator", "site"}]
+    values = [x for x in values if x["value"]["type"] != "IMAGE_COLOR"]
+    return values
+
+
+def _parse_card(obj: dict, url: str):
+    name = get_or(obj, "card.legacy.name", None)
+    if not name:
+        return None
+
+    if name in {"summary", "summary_large_image", "player"}:
+        val = _parse_card_prepare_values(obj)
+        title, val = _parse_card_extract_title(val)
+        description, val = _parse_card_extract_str(val, "description")
+        vanity_url, val = _parse_card_extract_str(val, "vanity_url")
+        url, val = _parse_card_extract_str(val, "card_url")
+        photo, val = _parse_card_extract_largest_photo(val)
+
+        return SummaryCard(
+            title=title,
+            description=description,
+            vanityUrl=vanity_url,
+            url=url,
+            photo=photo,
+        )
+
+    if name == "unified_card":
+        val = _parse_card_prepare_values(obj)
+        val = [x for x in val if x["key"] == "unified_card"][0]["value"]["string_value"]
+        val = json.loads(val)
+
+        co = get_or(val, "component_objects", {})
+        do = get_or(val, "destination_objects", {})
+        me = list(get_or(val, "media_entities", {}).values())
+        if len(me) > 1:
+            logger.debug(f"[Card] Multiple media entities: {json.dumps(me, indent=2)}")
+
+        me = me[0] if me else {}
+
+        title = get_or(co, "details_1.data.title.content", "")
+        description = get_or(co, "details_1.data.subtitle.content", "")
+        vanity_url = get_or(do, "browser_with_docked_media_1.data.url_data.vanity", "")
+        url = get_or(do, "browser_with_docked_media_1.data.url_data.url", "")
+        video = MediaVideo.parse(me) if me and me["type"] == "video" else None
+        photo = MediaPhoto.parse(me) if me and me["type"] == "photo" else None
+
+        return SummaryCard(
+            title=title,
+            description=description,
+            vanityUrl=vanity_url,
+            url=url,
+            photo=photo,
+            video=video,
+        )
+
+    if re.match(r"poll\d+choice_text_only", name):
+        val = _parse_card_prepare_values(obj)
+
+        options = []
+        for x in range(20):
+            label = _parse_card_get_str(val, f"choice{x + 1}_label")
+            votes = _parse_card_get_str(val, f"choice{x + 1}_count")
+            if label is None or votes is None:
+                break
+
+            options.append(PollOption(label=label, votesCount=int(votes)))
+
+        finished = _parse_card_get_bool(val, "counts_are_final")
+        # duration_minutes = int(_parse_card_get_str(val, "duration_minutes") or "0")
+        # end_datetime_utc = _parse_card_get_str(val, "end_datetime_utc")
+        # print(json.dumps(val, indent=2))
+        return PollCard(options=options, finished=finished)
+
+    if name == "745291183405076480:broadcast":
+        val = _parse_card_prepare_values(obj)
+        card_url = _parse_card_get_str(val, "broadcast_url")
+        card_title = _parse_card_get_str(val, "broadcast_title")
+        photo, _ = _parse_card_extract_largest_photo(val)
+        if card_url is None or card_title is None:
+            return None
+
+        return BroadcastCard(title=card_title, url=card_url, photo=photo)
+
+    if name == "3691233323:audiospace":
+        # no more data in this object, possible extra api call needed to get card info
+        val = _parse_card_prepare_values(obj)
+        card_url = _parse_card_get_str(val, "card_url")
+        if card_url is None:
+            return None
+
+        # print(json.dumps(val, indent=2))
+        return AudiospaceCard(url=card_url)
+
+    logger.warning(f"Unknown card type '{name}' on {url}")
+    if "PYTEST_CURRENT_TEST" in os.environ:  # help debugging tests
+        print(f"Unknown card type '{name}' on {url}", file=sys.stderr)
+        # print(json.dumps(obj["card"]["legacy"], indent=2))
+    return None
+
+
+# internal helpers
+
+
+def _get_reply_user(tw_obj: dict, res: dict):
+    user_id = tw_obj.get("in_reply_to_user_id_str", None)
+    if user_id is None:
+        return None
+
+    if user_id in res["users"]:
+        return UserRef.parse(res["users"][user_id])
+
+    mentions = get_or(tw_obj, "entities.user_mentions", [])
+    mention = find_item(mentions, lambda x: x["id_str"] == tw_obj["in_reply_to_user_id_str"])
+    if mention:
+        return UserRef.parse(mention)
+
+    # todo: user not found in reply (probably deleted or hidden)
+    return None
+
+
+def _get_source_url(tw_obj: dict):
+    source = tw_obj.get("source", None)
+    if source and (match := re.search(r'href=[\'"]?([^\'" >]+)', source)):
+        return str(match.group(1))
+    return None
+
+
+def _get_source_label(tw_obj: dict):
+    source = tw_obj.get("source", None)
+    if source and (match := re.search(r">([^<]*)<", source)):
+        return str(match.group(1))
+    return None
+
+
+def _parse_links(obj: dict, paths: list[str]):
+    links = []
+    for x in paths:
+        links.extend(get_or(obj, x, []))
+
+    links = [TextLink.parse(x) for x in links]
+    links = [x for x in links if x is not None]
+
+    return links
+
+
+def _first(obj: dict, paths: list[str]):
+    for x in paths:
+        cid = get_or(obj, x, None)
+        if cid is not None:
+            return cid
+    return None
+
+
+def _get_views(obj: dict, rt_obj: dict):
+    for x in [obj, rt_obj]:
+        for y in ["ext_views.count", "views.count"]:
+            k = int_or(x, y)
+            if k is not None:
+                return k
+    return None
+
+
+def _write_dump(kind: str, e: Exception, x: dict, obj: dict):
+    uniq = "".join(random.choice(string.ascii_lowercase) for _ in range(5))
+    time = utc.now().strftime("%Y-%m-%d_%H-%M-%S")
+    dumpfile = f"/tmp/twscrape/twscrape_parse_error_{time}_{uniq}.txt"
+    os.makedirs(os.path.dirname(dumpfile), exist_ok=True)
+
+    with open(dumpfile, "w") as fp:
+        msg = [
+            f"Error parsing {kind}. Error: {type(e)}",
+            traceback.format_exc(),
+            json.dumps(x, default=str),
+            json.dumps(obj, default=str),
+        ]
+        fp.write("\n\n".join(msg))
+
+    logger.error(f"Failed to parse response of {kind}, writing dump to {dumpfile}")
+
+
+def _parse_items(rep: httpx.Response, kind: str, limit: int = -1):
+    if kind == "user":
+        Cls, key = User, "users"
+    elif kind == "tweet":
+        Cls, key = Tweet, "tweets"
+    elif kind == "trends":
+        Cls, key = Trend, "trends"
+    else:
+        raise ValueError(f"Invalid kind: {kind}")
+
+    # check for dict, because httpx.Response can be mocked in tests with different type
+    res = rep if isinstance(rep, dict) else rep.json()
+    obj = to_old_rep(res)
+
+    ids = set()
+    for x in obj[key].values():
+        if limit != -1 and len(ids) >= limit:
+            # todo: move somewhere in configuration like force_limit
+            # https://github.com/vladkens/twscrape/issues/26#issuecomment-1656875132
+            # break
+            pass
+
+        try:
+            tmp = Cls.parse(x, obj)
+            if tmp.id not in ids:
+                ids.add(tmp.id)
+                yield tmp
+        except Exception as e:
+            _write_dump(kind, e, x, obj)
+            continue
+
+
+# public helpers
+
+
+def parse_tweet(rep: httpx.Response, twid: int) -> Tweet | None:
+    try:
+        docs = list(parse_tweets(rep))
+        for x in docs:
+            if x.id == twid:
+                return x
+        return None
+    except Exception as e:
+        logger.error(f"Failed to parse tweet {twid} - {type(e)}:\n{traceback.format_exc()}")
+        return None
+
+
+def parse_user(rep: httpx.Response) -> User | None:
+    try:
+        docs = list(parse_users(rep))
+        if len(docs) == 1:
+            return docs[0]
+        return None
+    except Exception as e:
+        logger.error(f"Failed to parse user - {type(e)}:\n{traceback.format_exc()}")
+        return None
+
+
+def parse_trend(rep: httpx.Response) -> Trend | None:
+    try:
+        docs = list(parse_trends(rep))
+        if len(docs) == 1:
+            return docs[0]
+        return None
+    except Exception as e:
+        logger.error(f"Failed to parse trend - {type(e)}:\n{traceback.format_exc()}")
+        return None
+
+
+def parse_tweets(rep: httpx.Response, limit: int = -1) -> Generator[Tweet, None, None]:
+    return _parse_items(rep, "tweet", limit)  # type: ignore
+
+
+def parse_users(rep: httpx.Response, limit: int = -1) -> Generator[User, None, None]:
+    return _parse_items(rep, "user", limit)  # type: ignore
+
+
+def parse_trends(rep: httpx.Response, limit: int = -1) -> Generator[Trend, None, None]:
+    return _parse_items(rep, kind="trends", limit=limit)  # type: ignore

apps/twscrape/twscrape/queue_client.py

@@ -0,0 +1,244 @@
+import json
+import os
+from typing import Any
+
+import httpx
+from httpx import AsyncClient, Response
+
+from .accounts_pool import Account, AccountsPool
+from .logger import logger
+from .utils import utc
+
+ReqParams = dict[str, str | int] | None
+TMP_TS = utc.now().isoformat().split(".")[0].replace("T", "_").replace(":", "-")[0:16]
+
+
+class Ctx:
+    def __init__(self, acc: Account, clt: AsyncClient):
+        self.acc = acc
+        self.clt = clt
+        self.req_count = 0
+
+
+class HandledError(Exception):
+    pass
+
+
+class AbortReqError(Exception):
+    pass
+
+
+def req_id(rep: Response):
+    lr = str(rep.headers.get("x-rate-limit-remaining", -1))
+    ll = str(rep.headers.get("x-rate-limit-limit", -1))
+    sz = max(len(lr), len(ll))
+    lr, ll = lr.rjust(sz), ll.rjust(sz)
+
+    username = getattr(rep, "__username", "<UNKNOWN>")
+    return f"{lr}/{ll} - {username}"
+
+
+def dump_rep(rep: Response):
+    count = getattr(dump_rep, "__count", -1) + 1
+    setattr(dump_rep, "__count", count)
+
+    acc = getattr(rep, "__username", "<unknown>")
+    outfile = f"{count:05d}_{rep.status_code}_{acc}.txt"
+    outfile = f"/tmp/twscrape-{TMP_TS}/{outfile}"
+    os.makedirs(os.path.dirname(outfile), exist_ok=True)
+
+    msg = []
+    msg.append(f"{count:,d} - {req_id(rep)}")
+    msg.append(f"{rep.status_code} {rep.request.method} {rep.request.url}")
+    msg.append("\n")
+    # msg.append("\n".join([str(x) for x in list(rep.request.headers.items())]))
+    msg.append("\n".join([str(x) for x in list(rep.headers.items())]))
+    msg.append("\n")
+
+    try:
+        msg.append(json.dumps(rep.json(), indent=2))
+    except json.JSONDecodeError:
+        msg.append(rep.text)
+
+    txt = "\n".join(msg)
+    with open(outfile, "w") as f:
+        f.write(txt)
+
+
+class QueueClient:
+    def __init__(self, pool: AccountsPool, queue: str, debug=False, proxy: str | None = None):
+        self.pool = pool
+        self.queue = queue
+        self.debug = debug
+        self.ctx: Ctx | None = None
+        self.proxy = proxy
+
+    async def __aenter__(self):
+        await self._get_ctx()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self._close_ctx()
+
+    async def _close_ctx(self, reset_at=-1, inactive=False, msg: str | None = None):
+        if self.ctx is None:
+            return
+
+        ctx, self.ctx, self.req_count = self.ctx, None, 0
+        username = ctx.acc.username
+        await ctx.clt.aclose()
+
+        if inactive:
+            await self.pool.mark_inactive(username, msg)
+            return
+
+        if reset_at > 0:
+            await self.pool.lock_until(ctx.acc.username, self.queue, reset_at, ctx.req_count)
+            return
+
+        await self.pool.unlock(ctx.acc.username, self.queue, ctx.req_count)
+
+    async def _get_ctx(self):
+        if self.ctx:
+            return self.ctx
+
+        acc = await self.pool.get_for_queue_or_wait(self.queue)
+        if acc is None:
+            return None
+
+        clt = acc.make_client(proxy=self.proxy)
+        self.ctx = Ctx(acc, clt)
+        return self.ctx
+
+    async def _check_rep(self, rep: Response) -> None:
+        """
+        This function can raise Exception and request will be retried or aborted
+        Or if None is returned, response will passed to api parser as is
+        """
+
+        if self.debug:
+            dump_rep(rep)
+
+        try:
+            res = rep.json()
+        except json.JSONDecodeError:
+            res: Any = {"_raw": rep.text}
+
+        limit_remaining = int(rep.headers.get("x-rate-limit-remaining", -1))
+        limit_reset = int(rep.headers.get("x-rate-limit-reset", -1))
+        # limit_max = int(rep.headers.get("x-rate-limit-limit", -1))
+
+        err_msg = "OK"
+        if "errors" in res:
+            err_msg = set([f"({x.get('code', -1)}) {x['message']}" for x in res["errors"]])
+            err_msg = "; ".join(list(err_msg))
+
+        log_msg = f"{rep.status_code:3d} - {req_id(rep)} - {err_msg}"
+        logger.trace(log_msg)
+
+        # for dev: need to add some features in api.py
+        if err_msg.startswith("(336) The following features cannot be null"):
+            logger.error(f"[DEV] Update required: {err_msg}")
+            exit(1)
+
+        # general api rate limit
+        if limit_remaining == 0 and limit_reset > 0:
+            logger.debug(f"Rate limited: {log_msg}")
+            await self._close_ctx(limit_reset)
+            raise HandledError()
+
+        # no way to check is account banned in direct way, but this check should work
+        if err_msg.startswith("(88) Rate limit exceeded") and limit_remaining > 0:
+            logger.warning(f"Ban detected: {log_msg}")
+            await self._close_ctx(-1, inactive=True, msg=err_msg)
+            raise HandledError()
+
+        if err_msg.startswith("(326) Authorization: Denied by access control"):
+            logger.warning(f"Ban detected: {log_msg}")
+            await self._close_ctx(-1, inactive=True, msg=err_msg)
+            raise HandledError()
+
+        if err_msg.startswith("(32) Could not authenticate you"):
+            logger.warning(f"Session expired or banned: {log_msg}")
+            await self._close_ctx(-1, inactive=True, msg=err_msg)
+            raise HandledError()
+
+        if err_msg == "OK" and rep.status_code == 403:
+            logger.warning(f"Session expired or banned: {log_msg}")
+            await self._close_ctx(-1, inactive=True, msg=None)
+            raise HandledError()
+
+        # something from twitter side - abort all queries, see: https://github.com/vladkens/twscrape/pull/80
+        if err_msg.startswith("(131) Dependency: Internal error"):
+            # looks like when data exists, we can ignore this error
+            # https://github.com/vladkens/twscrape/issues/166
+            if rep.status_code == 200 and "data" in res and "user" in res["data"]:
+                err_msg = "OK"
+            else:
+                logger.warning(f"Dependency error (request skipped): {err_msg}")
+                raise AbortReqError()
+
+        # content not found
+        if rep.status_code == 200 and "_Missing: No status found with that ID" in err_msg:
+            return  # ignore this error
+
+        # something from twitter side - just ignore it, see: https://github.com/vladkens/twscrape/pull/95
+        if rep.status_code == 200 and "Authorization" in err_msg:
+            logger.warning(f"Authorization unknown error: {log_msg}")
+            return
+
+        if err_msg != "OK":
+            logger.warning(f"API unknown error: {log_msg}")
+            return  # ignore any other unknown errors
+
+        try:
+            rep.raise_for_status()
+        except httpx.HTTPStatusError:
+            logger.error(f"Unhandled API response code: {log_msg}")
+            await self._close_ctx(utc.ts() + 60 * 15)  # 15 minutes
+            raise HandledError()
+
+    async def get(self, url: str, params: ReqParams = None) -> Response | None:
+        return await self.req("GET", url, params=params)
+
+    async def req(self, method: str, url: str, params: ReqParams = None) -> Response | None:
+        unknown_retry, connection_retry = 0, 0
+
+        while True:
+            ctx = await self._get_ctx()  # not need to close client, class implements __aexit__
+            if ctx is None:
+                return None
+
+            try:
+                rep = await ctx.clt.request(method, url, params=params)
+                setattr(rep, "__username", ctx.acc.username)
+                await self._check_rep(rep)
+
+                ctx.req_count += 1  # count only successful
+                unknown_retry, connection_retry = 0, 0
+                return rep
+            except AbortReqError:
+                # abort all queries
+                return
+            except HandledError:
+                # retry with new account
+                continue
+            except (httpx.ReadTimeout, httpx.ProxyError):
+                # http transport failed, just retry with same account
+                continue
+            except (httpx.ConnectError, httpx.ConnectTimeout) as e:
+                # if proxy missconfigured or ???
+                connection_retry += 1
+                if connection_retry >= 3:
+                    raise e
+            except Exception as e:
+                unknown_retry += 1
+                if unknown_retry >= 3:
+                    msg = [
+                        "Unknown error. Account timeouted for 15 minutes.",
+                        "Create issue please: https://github.com/vladkens/twscrape/issues",
+                        f"If it mistake, you can unlock accounts with `twscrape reset_locks`. Err: {type(e)}: {e}",
+                    ]
+
+                    logger.warning(" ".join(msg))
+                    await self._close_ctx(utc.ts() + 60 * 15)  # 15 minutes

apps/twscrape/twscrape/utils.py

@@ -0,0 +1,219 @@
+import base64
+import json
+import os
+from collections import defaultdict
+from datetime import datetime, timezone
+from typing import Any, AsyncGenerator, Callable, TypeVar
+
+T = TypeVar("T")
+
+
+class utc:
+    @staticmethod
+    def now() -> datetime:
+        return datetime.now(timezone.utc)
+
+    @staticmethod
+    def from_iso(iso: str) -> datetime:
+        return datetime.fromisoformat(iso).replace(tzinfo=timezone.utc)
+
+    @staticmethod
+    def ts() -> int:
+        return int(utc.now().timestamp())
+
+
+async def gather(gen: AsyncGenerator[T, None]) -> list[T]:
+    items = []
+    async for x in gen:
+        items.append(x)
+    return items
+
+
+def encode_params(obj: dict):
+    res = {}
+    for k, v in obj.items():
+        if isinstance(v, dict):
+            v = {a: b for a, b in v.items() if b is not None}
+            v = json.dumps(v, separators=(",", ":"))
+
+        res[k] = str(v)
+
+    return res
+
+
+def get_or(obj: dict, key: str, default_value: T = None) -> Any | T:
+    for part in key.split("."):
+        if part not in obj:
+            return default_value
+        obj = obj[part]
+    return obj
+
+
+def int_or(obj: dict, key: str, default_value: int | None = None):
+    try:
+        val = get_or(obj, key)
+        return int(val) if val is not None else default_value
+    except Exception:
+        return default_value
+
+
+# https://stackoverflow.com/a/43184871
+def get_by_path(obj: dict, key: str, default=None):
+    stack = [iter(obj.items())]
+    while stack:
+        for k, v in stack[-1]:
+            if k == key:
+                return v
+            elif isinstance(v, dict):
+                stack.append(iter(v.items()))
+                break
+            elif isinstance(v, list):
+                stack.append(iter(enumerate(v)))
+                break
+        else:
+            stack.pop()
+    return default
+
+
+def find_item(lst: list[T], fn: Callable[[T], bool]) -> T | None:
+    for item in lst:
+        if fn(item):
+            return item
+    return None
+
+
+def find_or_fail(lst: list[T], fn: Callable[[T], bool]) -> T:
+    item = find_item(lst, fn)
+    if item is None:
+        raise ValueError()
+    return item
+
+
+def find_obj(obj: dict, fn: Callable[[dict], bool]) -> Any | None:
+    if not isinstance(obj, dict):
+        return None
+
+    if fn(obj):
+        return obj
+
+    for _, v in obj.items():
+        if isinstance(v, dict):
+            if res := find_obj(v, fn):
+                return res
+        elif isinstance(v, list):
+            for x in v:
+                if res := find_obj(x, fn):
+                    return res
+
+    return None
+
+
+def get_typed_object(obj: dict, res: defaultdict[str, list]):
+    obj_type = obj.get("__typename", None)
+    if obj_type is not None:
+        res[obj_type].append(obj)
+
+    for _, v in obj.items():
+        if isinstance(v, dict):
+            get_typed_object(v, res)
+        elif isinstance(v, list):
+            for x in v:
+                if isinstance(x, dict):
+                    get_typed_object(x, res)
+
+    return res
+
+
+def to_old_obj(obj: dict):
+    return {
+        **obj,
+        **obj["legacy"],
+        "id_str": str(obj["rest_id"]),
+        "id": int(obj["rest_id"]),
+        "legacy": None,
+    }
+
+
+def to_old_rep(obj: dict) -> dict[str, dict]:
+    tmp = get_typed_object(obj, defaultdict(list))
+
+    tw1 = [x for x in tmp.get("Tweet", []) if "legacy" in x]
+    tw1 = {str(x["rest_id"]): to_old_obj(x) for x in tw1}
+
+    # https://github.com/vladkens/twscrape/issues/53
+    tw2 = [x["tweet"] for x in tmp.get("TweetWithVisibilityResults", []) if "legacy" in x["tweet"]]
+    tw2 = {str(x["rest_id"]): to_old_obj(x) for x in tw2}
+
+    users = [x for x in tmp.get("User", []) if "legacy" in x and "id" in x]
+    users = {str(x["rest_id"]): to_old_obj(x) for x in users}
+
+    trends = [x for x in tmp.get("TimelineTrend", [])]
+    trends = {x["name"]: x for x in trends}
+
+    return {"tweets": {**tw1, **tw2}, "users": users, "trends": trends}
+
+
+def print_table(rows: list[dict], hr_after=False):
+    if not rows:
+        return
+
+    def prt(x):
+        if isinstance(x, str):
+            return x
+
+        if isinstance(x, int):
+            return f"{x:,}"
+
+        if isinstance(x, datetime):
+            return x.isoformat().split("+")[0].replace("T", " ")
+
+        return str(x)
+
+    keys = list(rows[0].keys())
+    rows = [{k: k for k in keys}, *[{k: prt(x.get(k, "")) for k in keys} for x in rows]]
+    colw = [max(len(x[k]) for x in rows) + 1 for k in keys]
+
+    lines = []
+    for row in rows:
+        line = [f"{row[k]:<{colw[i]}}" for i, k in enumerate(keys)]
+        lines.append(" ".join(line))
+
+    max_len = max(len(x) for x in lines)
+    # lines.insert(1, "─" * max_len)
+    # lines.insert(0, "─" * max_len)
+    print("\n".join(lines))
+    if hr_after:
+        print("-" * max_len)
+
+
+def parse_cookies(val: str) -> dict[str, str]:
+    try:
+        val = base64.b64decode(val).decode()
+    except Exception:
+        pass
+
+    try:
+        try:
+            res = json.loads(val)
+            if isinstance(res, dict) and "cookies" in res:
+                res = res["cookies"]
+
+            if isinstance(res, list):
+                return {x["name"]: x["value"] for x in res}
+            if isinstance(res, dict):
+                return res
+        except json.JSONDecodeError:
+            res = val.split("; ")
+            res = [x.split("=") for x in res]
+            return {x[0]: x[1] for x in res}
+    except Exception:
+        pass
+
+    raise ValueError(f"Invalid cookie value: {val}")
+
+
+def get_env_bool(key: str, default_val: bool = False) -> bool:
+    val = os.getenv(key)
+    if val is None:
+        return default_val
+    return val.lower() in ("1", "true", "yes")