add socials tests

apps/bright/lib/bright/oban_workers/scrape_posts.ex

@@ -0,0 +1,64 @@
+# defmodule Bright.ObanWorkers.ScrapeX do
+#   alias Bright.Vtubers.Vtuber
+#   use Oban.Worker, queue: :default, max_attempts: 3
+
+#   alias Bright.Streams.Vod
+
+#   alias Bright.{
+#     Repo,
+#     Downloader,
+#     B2,
+#     Images
+#   }
+
+#   require Logger
+
+#   @impl Oban.Worker
+#   def perform(%Oban.Job{args: %{"vtuber_id" => vtuber_id}}) do
+#     Logger.info(">>>> Scrape X is performing. with vtuber_id=#{vtuber_id}")
+
+#     # @todo get vtuber from db
+#     # @todo get vtuber's X account
+#     # @todo get nitter URL
+#     # @todo get X posts
+#     # @todo queue posts we haven't yet processed
+#     # @todo parse posts in the queue to find CB/Fansly/OF invite links
+#     # @todo for each post with an invite, create a stream in the db
+
+#     # case Repo.get(Vtuber, vtuber_id) do
+
+#     #   nil ->
+#     #     Logger.error("Vtuber id #{vtuber_id} not found.")
+#     #     {:error, "Vtuber not found"}
+
+#     #   %Vtuber{} = vtuber ->
+#     #     with {:ok, } <-
+
+#     # end
+
+#     case Repo.get(Vod, vod_id) do
+#       nil ->
+#         Logger.error("VOD ID #{vod_id} not found")
+#         {:error, "VOD not found"}
+
+#       %Vod{origin_temp_input_url: origin_temp_input_url} = vod ->
+#         with {:ok, local_filename} <- Downloader.get(origin_temp_input_url),
+#              {:ok, thumbnail_filename} <- Images.create_thumbnail(local_filename),
+#              {:ok, s3Asset} <- B2.put(thumbnail_filename) do
+#           update_vod_with_thumbnail_url(vod, s3Asset.cdn_url)
+#         else
+#           {:error, reason} ->
+#             Logger.error("Failed to create thumbnail for VOD ID #{vod_id}: #{inspect(reason)}")
+#             {:error, reason}
+#         end
+#     end
+#   end
+
+#   # defp generate_thumbnail_url(basename), do: "#{Application.get_env(:bright, :public_s3_endpoint)}/#{basename}"
+#   defp update_vod_with_thumbnail_url(vod, thumbnail_url) do
+#     case Repo.update(vod |> Ecto.Changeset.change(thumbnail_url: thumbnail_url)) do
+#       {:ok, updated_vod} -> {:ok, updated_vod}
+#       {:error, changeset} -> {:error, changeset}
+#     end
+#   end
+# end

apps/bright/lib/bright/platforms/x.ex

@@ -1,22 +0,0 @@
-defmodule Bright.Platforms.XPost do
-  use Ecto.Schema
-  import Ecto.Changeset
-
-  schema "x_posts" do
-    field :raw, :string
-    field :url, :string
-    field :date, :utc_datetime
-    field :is_invitation, :boolean
-
-    belongs_to :vtuber, Bright.Vtubers.Vtuber
-
-    timestamps(type: :utc_datetime)
-  end
-
-  @doc false
-  def changeset(platform, attrs) do
-    platform
-    |> cast(attrs, [:raw, :url, :date])
-    |> validate_required([:raw, :url, :date])
-  end
-end

apps/bright/lib/bright/shopping_cart/cart.ex

@@ -1,19 +0,0 @@
-defmodule Bright.ShoppingCart.Cart do
-  use Ecto.Schema
-  import Ecto.Changeset
-
-  schema "carts" do
-    field :user_uuid, Ecto.UUID
-    has_many :items, Bright.ShoppingCart.CartItem
-
-    timestamps(type: :utc_datetime)
-  end
-
-  @doc false
-  def changeset(cart, attrs) do
-    cart
-    |> cast(attrs, [:user_uuid])
-    |> validate_required([:user_uuid])
-    |> unique_constraint(:user_uuid)
-  end
-end

apps/bright/lib/bright/shopping_cart/cart_item.ex

@@ -1,21 +0,0 @@
-defmodule Bright.ShoppingCart.CartItem do
-  use Ecto.Schema
-  import Ecto.Changeset
-
-  schema "cart_items" do
-    field :price_when_carted, :decimal
-    field :quantity, :integer
-    belongs_to :cart, Bright.ShoppingCart.Cart
-    belongs_to :product, Bright.Catalog.Product
-
-    timestamps(type: :utc_datetime)
-  end
-
-  @doc false
-  def changeset(cart_item, attrs) do
-    cart_item
-    |> cast(attrs, [:price_when_carted, :quantity])
-    |> validate_required([:price_when_carted, :quantity])
-    |> validate_number(:quantity, greater_than_or_equal_to: 0, less_than: 100)
-  end
-end

apps/bright/lib/bright/socials.ex

@@ -0,0 +1,7 @@
+defmodule Bright.Socials do
+  @moduledoc """
+    Socials context, for functions for interacting with social media platforms
+  """
+
+
+end

apps/bright/lib/bright/socials/rss_parser.ex

@@ -0,0 +1,49 @@
+
+defmodule Bright.Socials.RSSParser do
+  @moduledoc """
+  Module to parse X RSS feeds and extract URLs, datestamps, and text content from items.
+  """
+
+  @doc """
+  Extracts URLs, datestamps, and text content from the RSS feed data.
+
+  # Example usage:
+  # Assuming `data` is the parsed RSS feed data you provided.
+
+  ```elixir
+  item_details = RSSParser.extract_item_details(data)
+  IO.inspect(item_details)
+  ```
+
+  """
+  def extract_item_details(data) do
+    data
+    |> find_items()
+    |> Enum.map(fn item ->
+      %{
+        url: extract_value(item, :link),
+        datestamp: extract_value(item, :pubDate),
+        text: extract_value(item, :title)
+      }
+    end)
+  end
+
+  # Helper function to find all :item elements in the nested structure
+  defp find_items(data) do
+    data
+    |> List.wrap()  # Ensure data is treated as a list
+    |> Enum.flat_map(fn
+      %{name: :item} = item -> [item]  # If it's an item, return it
+      %{value: children} -> find_items(children)  # Recursively search children
+      _ -> []  # Skip anything else
+    end)
+  end
+
+  # Helper function to extract the value for a given key from an item
+  defp extract_value(item, key) do
+    case Enum.find(item[:value], fn %{name: name} -> name == key end) do
+      %{value: [value]} -> value  # Extract the value if found
+      _ -> nil  # Return nil if the key is not found
+    end
+  end
+end

apps/bright/lib/bright/socials/x_post.ex

@@ -0,0 +1,84 @@
+defmodule Bright.Socials.XPost do
+  use Ecto.Schema
+  import Ecto.Changeset
+  alias Bright.Vtubers.Vtuber
+  alias Bright.Socials.RSSParser
+  alias Quinn
+  require Logger
+
+  ## @todo this needs to be pulled from the database
+  ## @todo rss.app doesn't scale.
+  ## @todo we need a better way to get data from vtuber's X accounts.
+  @sample_rss_feed "https://rss.app/feeds/FhPetvUY036xiFau.xml"
+
+  @doc """
+  We cache the posts in the db so it's clear which tweets we've read and which ones we haven't.
+  We only parse tweets which haven't been cached.
+  """
+  schema "x_posts" do
+    field :raw, :string
+    field :url, :string
+    field :date, :utc_datetime
+    field :is_invitation, :boolean
+
+    belongs_to :vtuber, Bright.Vtubers.Vtuber
+
+    timestamps(type: :utc_datetime)
+  end
+
+  @doc false
+  def changeset(post, attrs) do
+    post
+    |> cast(attrs, [:raw, :url, :date])
+    |> validate_required([:raw, :url, :date])
+  end
+
+
+  # def get_posts(vtuber_id) do
+  #   case Repo.get(Vtuber, vtuber_id) do
+  #     nil ->
+  #       Logger.error("Vtuber id #{vtuber_id} not found.")
+  #       {:error, "Vtuber not found"}
+
+  #     %Vtuber{} = vtuber ->
+  #       with {:ok, posts} <- get_posts(@sample_rss_feed) # @todo This feed URL needs to be dynamically looked up
+  #       do
+  #         save_posts(posts)
+  #       end
+  #   end
+
+
+  # end
+
+
+  @doc """
+    We read X posts via RSS URL.
+  """
+  def get_posts(feed_url) do
+    case HTTPoison.get(feed_url) do
+      {:ok, %HTTPoison.Response{ body: body }} ->
+
+        data = Quinn.parse(body)
+        extract = RSSParser.extract_item_details(data)
+        Logger.debug("we GETted a rss feed. Parsed data=#{inspect(data)}")
+        Logger.debug("we parsed the rss feed using RSSParser. parsed=#{inspect(extract)}")
+
+
+        {:ok, extract}
+
+      {:error, reason} ->
+        Logger.debug("failed to get_posts. reason=#{inspect(reason)}")
+        {:error, reason}
+    end
+  end
+
+
+  @doc """
+  save the posts to the db
+  """
+  def save_posts(posts) do
+    Logger.debug("@todo implement save_posts()")
+  end
+
+
+end

apps/bright/mix.exs

@@ -73,17 +73,8 @@ defmodule Bright.MixProject do
       {:atomex, "~> 0.3.0"},
       {:bento, "~> 1.0"},
       {:identicon_svg, "~> 0.9"},
-      {:excoveralls, "~> 0.18", only: :test}
-      # {:membrane_core, "~> 1.0"},
-      # {:membrane_mpeg_ts_plugin, "~> 1.0.3"},
-      # {:membrane_file_plugin, "~> 0.17.2"},
-      # {:membrane_mp4_plugin, "~> 0.35.2"},
-      # {:membrane_http_adaptive_stream_plugin, "> 0.0.0"},
-      # {:membrane_h264_ffmpeg_plugin, "~> 0.32.5"},
-      # {:membrane_aac_plugin, "~> 0.11.0"},
-      # {:membrane_hackney_plugin, "~> 0.6.0"}, # incompatible with membrane_core 1.1.2
-      # {:membrane_mpegts_plugin, "~> 0.4.0"}, # official module is 4 years outdated
-      # {:membrane_mpegts_plugin, path: "/home/cj/Documents/membrane_mpegts_plugin"},
+      {:excoveralls, "~> 0.18", only: :test},
+      {:quinn, "~> 1.1.3"},
     ]
   end
 

apps/bright/mix.lock

@@ -31,6 +31,7 @@
   "ex_m3u8": {:hex, :ex_m3u8, "0.14.2", "3eb17f936e2ca2fdcde11664f3a543e75a94814d928098e050bda5b1e149c021", [:mix], [{:nimble_parsec, "~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:typed_struct, "~> 0.3.0", [hex: :typed_struct, repo: "hexpm", optional: false]}], "hexpm", "d2a1fb4382a521cce7f966502ecce6187f286ca2852dbb0dcc25dea72f8ba039"},
   "excoveralls": {:hex, :excoveralls, "0.18.5", "e229d0a65982613332ec30f07940038fe451a2e5b29bce2a5022165f0c9b157e", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "523fe8a15603f86d64852aab2abe8ddbd78e68579c8525ae765facc5eae01562"},
   "expo": {:hex, :expo, "1.1.0", "f7b9ed7fb5745ebe1eeedf3d6f29226c5dd52897ac67c0f8af62a07e661e5c75", [:mix], [], "hexpm", "fbadf93f4700fb44c331362177bdca9eeb8097e8b0ef525c9cc501cb9917c960"},
+  "fast_rss": {:hex, :fast_rss, "0.5.0", "2e79c7142ff7843243635fc3f067999eb26beb0154699f2fa5a39d687629a0c9", [:mix], [{:rustler, "~> 0.29.0", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.6", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}], "hexpm", "dafb3eccc024d7366a0ef78521a26cfaf2075be534cfe1ec70d72903eabdd44d"},
   "ffmpex": {:hex, :ffmpex, "0.11.0", "70d2e211a70e1d8cc1a81d73208d5efedda59d82db4c91160c79e5461529d291", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:rambo, "~> 0.3.0", [hex: :rambo, repo: "hexpm", optional: false]}], "hexpm", "2429d67badc91957ace572b9169615619740904a58791289ba54d99e57a164eb"},
   "file_system": {:hex, :file_system, "1.1.0", "08d232062284546c6c34426997dd7ef6ec9f8bbd090eb91780283c9016840e8f", [:mix], [], "hexpm", "bfcf81244f416871f2a2e15c1b515287faa5db9c6bcf290222206d120b3d43f6"},
   "finch": {:hex, :finch, "0.19.0", "c644641491ea854fc5c1bbaef36bfc764e3f08e7185e1f084e35e0672241b76d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc5324ce209125d1e2fa0fcd2634601c52a787aff1cd33ee833664a5af4ea2b6"},
@@ -95,10 +96,12 @@
   "postgrex": {:hex, :postgrex, "0.19.3", "a0bda6e3bc75ec07fca5b0a89bffd242ca209a4822a9533e7d3e84ee80707e19", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "d31c28053655b78f47f948c85bb1cf86a9c1f8ead346ba1aa0d0df017fa05b61"},
   "pythonx": {:hex, :pythonx, "0.2.5", "05660dc8548a4ab5da5b7f7977c6a5a3fa16eefadbe54077f9a176c9d386be27", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.8", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.3", [hex: :req, repo: "hexpm", optional: false]}], "hexpm", "66d2179e37be527cbecf47097c15aea28a1dbcb2c6e965407c89ca1e1ac74d17"},
   "qex": {:hex, :qex, "0.5.1", "0d82c0f008551d24fffb99d97f8299afcb8ea9cf99582b770bd004ed5af63fd6", [:mix], [], "hexpm", "935a39fdaf2445834b95951456559e9dc2063d0a055742c558a99987b38d6bab"},
+  "quinn": {:hex, :quinn, "1.1.3", "83df8a990d898b7756dce0471439328994396d9cc407763f6c313f632e4571e3", [:mix], [], "hexpm", "685c92ad76f7398ffb86b7a3ba9de244ff66988d497e60b14360878d57a255d1"},
   "rambo": {:hex, :rambo, "0.3.4", "8962ac3bd1a633ee9d0e8b44373c7913e3ce3d875b4151dcd060886092d2dce7", [:mix], [], "hexpm", "0cc54ed089fbbc84b65f4b8a774224ebfe60e5c80186fafc7910b3e379ad58f1"},
   "ratio": {:hex, :ratio, "4.0.1", "3044166f2fc6890aa53d3aef0c336f84b2bebb889dc57d5f95cc540daa1912f8", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:numbers, "~> 5.2.0", [hex: :numbers, repo: "hexpm", optional: false]}], "hexpm", "c60cbb3ccdff9ffa56e7d6d1654b5c70d9f90f4d753ab3a43a6bf40855b881ce"},
   "redirect": {:hex, :redirect, "0.4.0", "98b46053504ee517bc3ad2fd04c064b64b48d339e1e18266355b30c4f8bb52b0", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.3 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "dfa29a8ecbad066ed0b73b34611cf24c78101719737f37bdf750f39197d67b97"},
   "req": {:hex, :req, "0.5.8", "50d8d65279d6e343a5e46980ac2a70e97136182950833a1968b371e753f6a662", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "d7fc5898a566477e174f26887821a3c5082b243885520ee4b45555f5d53f40ef"},
+  "rustler_precompiled": {:hex, :rustler_precompiled, "0.8.2", "5f25cbe220a8fac3e7ad62e6f950fcdca5a5a5f8501835d2823e8c74bf4268d5", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "63d1bd5f8e23096d1ff851839923162096364bac8656a4a3c00d1fff8e83ee0a"},
   "secure_random": {:hex, :secure_random, "0.5.1", "c5532b37c89d175c328f5196a0c2a5680b15ebce3e654da37129a9fe40ebf51b", [:mix], [], "hexpm", "1b9754f15e3940a143baafd19da12293f100044df69ea12db5d72878312ae6ab"},
   "shmex": {:hex, :shmex, "0.5.1", "81dd209093416bf6608e66882cb7e676089307448a1afd4fc906c1f7e5b94cf4", [:mix], [{:bunch_native, "~> 0.5.0", [hex: :bunch_native, repo: "hexpm", optional: false]}, {:bundlex, "~> 1.0", [hex: :bundlex, repo: "hexpm", optional: false]}], "hexpm", "c29f8286891252f64c4e1dac40b217d960f7d58def597c4e606ff8fbe71ceb80"},
   "squircle": {:hex, :squircle, "0.1.1", "2e82410f9cf80de1023e19f5cb75e3e74ffa12ef0d2dabee81b73604447f7d3d", [:mix], [], "hexpm", "fe1214a99c94aea157da106d56c360a7c52861f7608650fbdadf12a7fd60e170"},

apps/bright/test/bright/socials/rss_parser_test.exs

@@ -0,0 +1,60 @@
+defmodule Bright.RSSParserTest do
+  use Bright.DataCase
+
+  alias Bright.Socials.RSSParser
+
+  @sample_data %{
+    name: :rss,
+    value: [
+      %{
+        name: :channel,
+        value: [
+          %{
+            name: :item,
+            value: [
+              %{name: :title, value: ["Test Post 1"], attr: []},
+              %{name: :link, value: ["https://example.com/post/1"], attr: []},
+              %{name: :pubDate, value: ["Wed, 12 Oct 2022 12:00:00 GMT"], attr: []}
+            ],
+            attr: []
+          },
+          %{
+            name: :item,
+            value: [
+              %{name: :title, value: ["Test Post 2"], attr: []},
+              %{name: :link, value: ["https://example.com/post/2"], attr: []},
+              %{name: :pubDate, value: ["Wed, 12 Oct 2022 13:00:00 GMT"], attr: []}
+            ],
+            attr: []
+          }
+        ]
+      }
+    ]
+  }
+
+  describe "extract_item_details/1" do
+    @tag :unit
+    test "extracts URLs, datestamps, and text content from RSS data" do
+      result = RSSParser.extract_item_details(@sample_data)
+
+      assert result == [
+               %{
+                 url: "https://example.com/post/1",
+                 datestamp: "Wed, 12 Oct 2022 12:00:00 GMT",
+                 text: "Test Post 1"
+               },
+               %{
+                 url: "https://example.com/post/2",
+                 datestamp: "Wed, 12 Oct 2022 13:00:00 GMT",
+                 text: "Test Post 2"
+               }
+             ]
+    end
+
+    @tag :unit
+    test "returns an empty list if no items are found" do
+      empty_data = %{name: :rss, value: [%{name: :channel, value: []}]}
+      assert RSSParser.extract_item_details(empty_data) == []
+    end
+  end
+end

apps/bright/test/bright/socials/x_test.exs

@@ -0,0 +1,17 @@
+defmodule Bright.XPostTest do
+  use Bright.DataCase
+
+  alias Bright.Socials.XPost
+
+  @sample_feed "https://rss.app/feeds/FhPetvUY036xiFau.xml"
+
+  describe "x_posts" do
+    # import Bright.SocialsFixtures
+
+    @tag :integration
+    test "get_posts/1" do
+      {:ok, posts} = XPost.get_posts(@sample_feed)
+      assert length(posts) > 0
+    end
+  end
+end