57 lines
1.5 KiB
Elixir
57 lines
1.5 KiB
Elixir
defmodule Bright.Downloader do
|
|
@moduledoc """
|
|
Downloader functions
|
|
"""
|
|
|
|
require Logger
|
|
|
|
def get(url) do
|
|
filename = Bright.Cache.generate_filename(url)
|
|
Logger.debug("Downloader getting url=#{inspect(url)}")
|
|
|
|
try do
|
|
{download!(url, filename), filename}
|
|
rescue
|
|
exception ->
|
|
{:error, Exception.message(exception)}
|
|
end
|
|
end
|
|
|
|
# greets https://elixirforum.com/t/how-to-download-big-files/9173/4
|
|
def download!(file_url, filename) do
|
|
Logger.debug("Downloader downloading file_url=#{file_url} to filename=#{filename}")
|
|
|
|
file =
|
|
if File.exists?(filename) do
|
|
File.open!(filename, [:append])
|
|
else
|
|
File.touch!(filename)
|
|
File.open!(filename, [:append])
|
|
end
|
|
|
|
%HTTPoison.AsyncResponse{id: ref} = HTTPoison.get!(file_url, %{}, stream_to: self())
|
|
|
|
append_loop(ref, file)
|
|
end
|
|
|
|
defp append_loop(ref, file) do
|
|
receive do
|
|
%HTTPoison.AsyncChunk{chunk: chunk, id: ^ref} ->
|
|
IO.binwrite(file, chunk)
|
|
append_loop(ref, file)
|
|
|
|
%HTTPoison.AsyncEnd{id: ^ref} ->
|
|
File.close(file)
|
|
|
|
# need something to handle errors like request timeout and such
|
|
# otherwise it will loop forever
|
|
# don't know what httpoison returns in case of an error ...
|
|
# you can inspect `_other` below to find out
|
|
# and match on the error to exit the loop early
|
|
_ ->
|
|
Logger.debug("recursively downloading #{inspect(ref)} #{inspect(file)}")
|
|
append_loop(ref, file)
|
|
end
|
|
end
|
|
end
|