A service that uploads files to AWS S3.
It periodically looks for new files in a specificed directory, uploads them to S3, then moves them to an archive directory structure named by date.
It parses the date from the file name to determine where to archive them.
Files look like traffic-20260108-2049.log.
If available in Hex, the package can be installed
by adding s3_uploader to your list of dependencies in mix.exs:
def deps do
[
{:s3_uploader, "~> 0.7.0"}
]
endConfigure the uploader in config.exs:
config :ex_aws,
hackney_options: [
pool: :ex_aws_pool,
# Max time to wait for a connection (in ms)
checkout_timeout: 8_000,
# Max time to wait for a response (in ms)
recv_timeout: 15_000
],
access_key_id: [{:system, "AWS_ACCESS_KEY_ID"}, :instance_role],
secret_access_key: [{:system, "AWS_SECRET_ACCESS_KEY"}, :instance_role],
region: {:system, "AWS_REGION"}
config :foo, :s3_uploader,
sources: [
traffic: [
max_concurrency: 10,
batch_size: 50,
in_dir: "/var/log/foo/traffic/archive",
archive_dir: "/var/log/foo/traffic/archive",
file_pattern: "traffic-\\d{8}-\\d{4}\\.log$",
datetime_pattern: "traffic-(?<year>\\d{4})(?<month>\\d{2})(?<day>\\d{2})",
bucket: "foo-prod-traffic-logs",
bucket_prefix: "traffic",
bucket_region: "eu-central-1"
]
]Add it to your application supervision tree:
@app :foo
@impl true
def init(args) do
children =
List.flatten([
s3_uploader_spec(args)
])
Supervisor.init(children, strategy: :one_for_one, max_restarts: 1000, max_seconds: 300)
end
defp s3_uploader_spec(args) do
s3_uploder_config = Application.get_env(@app, :s3_uploader)
if s3_uploder_config do
sources = s3_uploder_config[:sources] || []
# Use a separate hackney pool for S3 uploads to avoid blocking other HTTP requests in the app.
# The pool name should match the one used in ExAws config
:ok = :hackney_pool.start_pool(:ex_aws_pool, timeout: 15_000, max_connections: 40)
# [{Hackney.Pool, name: :ex_aws_pool, max_connections: 40, timeout: 15_000}] ++
for {name, config} <- sources do
id = String.to_atom("s3_uploader_#{name}")
config = Keyword.put(config, :hostname, args[:hostname])
Logger.info("Starting S3 Uploader #{name}: #{inspect(config)}")
%{id: id, start: {S3Uploader, :start_link, [config]}}
end
else
[]
end
endDocumentation can be generated with ExDoc and published on HexDocs. Once published, the docs can be found at https://hexdocs.pm/s3_uploader.