Procesando millones de

imgenes
David Padilla
@dabit

Carrierwave

image.recreate_versions!

Image.each do |i|
i.file.recreate_versions!
end

2,700,000

1 segundo

2,700,000 60
=

45,000 minutos

45,000 60
=

750 horas

750 24
=

31.25 das

MAX_WORKERS = 20
Image.find_each(batch_size: 1200) do |batch|
queue = Queue.new
batch.each {|o| queue.push o}
workers = (0..MAX_WORKERS).map do |w|
Thread.new do
while image = queue.pop(true)
image.file.recreate_versions!
end
end
end
workers.map(&:join)
end

MAX_WORKERS = 20
Image.find_each(batch_size: 1200) do |batch|
queue = Queue.new
batch.each {|o| queue.push o}
workers = (0..MAX_WORKERS).map do |w|
Thread.new do
while image = queue.pop(true)
image.file.recreate_versions!
end
end
end
workers.map(&:join)
end

MAX_WORKERS = 20
Image.find_each(batch_size: 1200) do |batch|
queue = Queue.new
batch.each {|o| queue.push o}
workers = (0..MAX_WORKERS).map do |w|
Thread.new do
while image = queue.pop(true)
image.file.recreate_versions!
end
end
end
workers.map(&:join)
end

MAX_WORKERS = 20
Image.find_each(batch_size: 1200) do |batch|
queue = Queue.new
batch.each {|o| queue.push o}
workers = (0..MAX_WORKERS).map do |w|
Thread.new do
while image = queue.pop(true)
image.file.recreate_versions!
end
end
end
workers.map(&:join)
end

MAX_WORKERS = 20
Image.find_each(batch_size: 1200) do |batch|
queue = Queue.new
batch.each {|o| queue.push o}
workers = (0..MAX_WORKERS).map do |w|
Thread.new do
while image = queue.pop(true)
image.file.recreate_versions!
end
end
end
workers.map(&:join)
end

array = []
5.times.map do
Thread.new do
1000.times do
array << nil
end
end
end.each(&:join)
puts array.size

$ ruby pushing_nil.rb
5000
$ jruby pushing_nil.rb
4446
$ rbx pushing_nil.rb
3088

array = []
5.times.map do
Thread.new do
1000.times do
array << nil
end
end
end.each(&:join)
puts array.size

1 segundo

0.6 segundos

18 das

Elixir

class Code
def method(*args)
code = code + code
while(true) do
something_awesome
end
end
def something_awesome
add = 1 + 2
o = Object.new
o.save_to_database
end
end

Exception

Crear aplicacin
Traer registros de la Base de Datos
Bajar de S3 imagen original
Crear dos tamaos
Subir a S3

Crear aplicacin

$ mix new images

Traer registros de la Base

de Datos

use Mix.Config
config :images, Images.Repo,
adapter: Ecto.Adapters.MySQL,
database: "images_database",
username: "root",
password: "",
hostname: "localhost"

defmodule Images.PropertyImage do
use Ecto.Model
schema "images" do
field :file, :string
end
end

defmodule Images.Image do
use Ecto.Model
def main_query do
from i in Images.Image,
select: i
end
def find(image_id) do
from i in main_query
where: id = ^image_id
end
def paged(offset, limit) do
from i in main_query,
limit: ^limit,
offset: ^offset
end
def all do
Images.Repo.all main_query
end
end

Bajar de S3 la imagen
original

def download_original(filename, id) do

file
= temp_filename(filename, id)
ibrowse = [save_response_to_file: String.to_char_list(file)]
s3_url(filename, id)
|> HTTPotion.get([ibrowse: ibrowse])
file
end
def s3_url(file, id) do
"#{s3_path}/#{id}/#{file}"
end

def download_original(filename, id) do

file
= temp_filename(filename, id)
ibrowse = [save_response_to_file: String.to_char_list(file)]
s3_url(filename, id)
|> HTTPotion.get([ibrowse: ibrowse])
file
end
def s3_url(file, id) do
"#{s3_path}/#{id}/#{file}"
end

def download_original(filename, id) do

file
= temp_filename(filename, id)
ibrowse = [save_response_to_file: String.to_char_list(file)]
HTTPotion.get(s3_url(filename, id), [ibrowse: ibrowse])
file
end
def s3_url(file, id) do
"#{s3_path}/#{id}/#{file}"
end

def download_original(filename, id) do

file
= temp_filename(filename, id)
ibrowse = [save_response_to_file: String.to_char_list(file)]
s3_url(filename, id)
|> HTTPotion.get([ibrowse: ibrowse])
file
end
def s3_url(file, id) do
"#{s3_path}/#{id}/#{file}"
end

Crear dos tamaos

def generate_medium(file, filename, id) do

result = Path.join(System.tmp_dir, size_name(filename, :medium))
Mogrify.open(file)
|> Mogrify.copy
|> Mogrify.resize_to_fill("450x300")
|> Mogrify.save(result)
end

Subir a S3

{status, _} = System.cmd("s3cmd", ["-P", "put", filename, s3_name])

{_, file} = File.read(result)

Application.get_env(:images, :s3_bucket)
|> String.to_char_list
|> :erlcloud_s3.put_object(s3_name, file, [], [{#}])

y la concurrencia?

Traer registros de la Base de Datos

Bajar de S3 imagen original
Crear dos tamaos
Subir a S3

:poolboy

defmodule Images.PropertyImageWorker do
use GenServer
def start_link([]) do
:gen_server.start_link(__MODULE__, [], [])
end
def init(state) do
secret = to_char_list(Application.get_env(:images, :aws_secret_key))
Application.get_env(:images, :aws_access_key)
|> to_char_list
|> :erlcloud_s3.configure(secret)
{:ok, state}
end
def handle_call(image, from, state) do
result = Images.PropertyImage.process(image)
{:reply, [result], state}
end
end

defmodule Images.PropertyImageWorker do
use GenServer
def start_link([]) do
:gen_server.start_link(__MODULE__, [], [])
end
def init(state) do
secret = to_char_list(Application.get_env(:images, :aws_secret_key))
Application.get_env(:images, :aws_access_key)
|> to_char_list
|> :erlcloud_s3.configure(secret)
{:ok, state}
end
def handle_call(image, from, state) do
result = Images.PropertyImage.process(image)
{:reply, [result], state}
end
end

defmodule Images.ImagesSupervisor do
use Supervisor
def start_link do
:supervisor.start_link(__MODULE__, [])
end
def init([]) do
poolboy_config = [
{:name, {:local, pool_name()}},
{:worker_module, Images.PropertyImageWorker},
{:size, 20},
{:max_overflow, 0}
]
children = [
:poolboy.child_spec(pool_name(), poolboy_config, []),
worker(Images.Repo, [])
]
supervise(children, strategy: :one_for_one)
end
def pool_name do
:property_images
end
end

defmodule Images.ImagesSupervisor do
use Supervisor
def start_link do
:supervisor.start_link(__MODULE__, [])
end
def init([]) do
poolboy_config = [
{:name, {:local, pool_name()}},
{:worker_module, Images.PropertyImageWorker},
{:size, 20},
{:max_overflow, 0}
]
children = [
:poolboy.child_spec(pool_name(), poolboy_config, []),
worker(Images.Repo, [])
]
supervise(children, strategy: :one_for_one)
end
def pool_name do
:property_images
end
end

defmodule Images do
def start(_type, _args) do
supervisor = Images.ImagesSupervisor.start_link
enqueue
supervisor
end
def enqueue do
Images.PropertyImage.all
|> Enum.each fn(r) -> spawn(fn() -> pool_image(r) end) end
end
def pool_image(image) do
:poolboy.transaction(
Images.ImagesSupervisor.pool_name,
fn(pid) -> :gen_server.call(pid, image) end,
:infinity
)
end
end

defmodule Images do
def start(_type, _args) do
supervisor = Images.ImagesSupervisor.start_link
enqueue
supervisor
end
def enqueue do
Images.PropertyImage.all
|> Enum.each fn(r) -> spawn(fn() -> pool_image(r) end) end
end
def pool_image(image) do
:poolboy.transaction(
Images.ImagesSupervisor.pool_name,
fn(pid) -> :gen_server.call(pid, image) end,
:infinity
)
end
end

defmodule Images do
def start(_type, _args) do
supervisor = Images.ImagesSupervisor.start_link
enqueue
supervisor
end
def enqueue do
Images.PropertyImage.all
|> Enum.each fn(r) -> spawn(fn() -> pool_image(r) end) end
end
def pool_image(image) do
:poolboy.transaction(
Images.ImagesSupervisor.pool_name,
fn(pid) -> :gen_server.call(pid, image) end,
:infinity
)
end
end

defmodule Images do
def start(_type, _args) do
supervisor = Images.ImagesSupervisor.start_link
enqueue
supervisor
end
def enqueue do
Images.PropertyImage.all
|> Enum.each fn(r) -> spawn(fn() -> pool_image(r) end) end
end
def pool_image(image) do
:poolboy.transaction(
Images.ImagesSupervisor.pool_name,
fn(pid) -> :gen_server.call(pid, image) end,
:infinity
)
end
end

BEAM

Conclusin

4 das

4 x 24 = 96
96 x 60 = 5,760
5,760 x 60 = 345,600
345,600 / 2,700,000 = 0.128s

Conclusin II

Fin