From 93ec479bb0599a5d3dcaa2593af00e1b1bc51a37 Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Sun, 26 Feb 2017 11:38:24 +0100 Subject: [PATCH] Generate PNG files on worker pool --- resources/asciinema/system.edn | 10 +++-- src/asciinema/boundary/executor.clj | 4 ++ .../component/fixed_thread_executor.clj | 39 +++++++++++++++++++ src/asciinema/endpoint/asciicasts.clj | 31 ++++++++------- 4 files changed, 67 insertions(+), 17 deletions(-) create mode 100644 src/asciinema/boundary/executor.clj create mode 100644 src/asciinema/component/fixed_thread_executor.clj diff --git a/resources/asciinema/system.edn b/resources/asciinema/system.edn index 3614a53..eaebc66 100644 --- a/resources/asciinema/system.edn +++ b/resources/asciinema/system.edn @@ -4,14 +4,15 @@ :db #var asciinema.component.db/hikaricp :ragtime #var duct.component.ragtime/ragtime :file-store #var asciinema.component.s3-file-store/s3-file-store - :exp-set #var asciinema.component.redis-client/redis-client} + :exp-set #var asciinema.component.redis-client/redis-client + :executor #var asciinema.component.fixed-thread-executor/fixed-thread-executor} :endpoints {:asciicasts #var asciinema.endpoint.asciicasts/asciicasts-endpoint} :dependencies {:http [:app] :app [:asciicasts] :ragtime [:db] - :asciicasts [:db :file-store :exp-set]} + :asciicasts [:db :file-store :exp-set :executor]} :config {:app {:middleware @@ -64,4 +65,7 @@ :path-prefix "uploads/"} :exp-set {:host redis-host - :port redis-port}}} + :port redis-port} + :executor + {:threads 1 + :queue-length 1}}} diff --git a/src/asciinema/boundary/executor.clj b/src/asciinema/boundary/executor.clj new file mode 100644 index 0000000..51360a1 --- /dev/null +++ b/src/asciinema/boundary/executor.clj @@ -0,0 +1,4 @@ +(ns asciinema.boundary.executor) + +(defprotocol Executor + (execute [this f])) diff --git a/src/asciinema/component/fixed_thread_executor.clj b/src/asciinema/component/fixed_thread_executor.clj new file mode 100644 index 0000000..fb7255a --- /dev/null +++ b/src/asciinema/component/fixed_thread_executor.clj @@ -0,0 +1,39 @@ +(ns asciinema.component.fixed-thread-executor + (:require [aleph.flow :as flow] + [asciinema.boundary.executor :as executor] + [com.stuartsierra.component :as component] + [manifold.deferred :as d]) + (:import [java.util.concurrent + ExecutorService + RejectedExecutionException + TimeUnit])) + +(defrecord FixedThreadExecutor [threads queue-length] + executor/Executor + (execute [{:keys [^ExecutorService executor]} f] + (try + (let [result (d/deferred) + f (fn [] + (try + (d/success! result (f)) + (catch Exception e + (d/error! result e))))] + (.execute executor f) + result) + (catch RejectedExecutionException _ + {:status 503 :headers {"Retry-After" "5"} :body "

503

"}))) + + component/Lifecycle + (start [{:keys [threads queue-length] :as component}] + (let [executor (flow/fixed-thread-executor threads {:onto? false + :initial-thread-count threads + :queue-length queue-length})] + (assoc component :executor executor))) + (stop [{:keys [^ExecutorService executor] :as component}] + (.shutdown executor) + (when-not (.awaitTermination executor 1000 TimeUnit/MILLISECONDS) + (.shutdownNow executor)) + (assoc component :executor nil))) + +(defn fixed-thread-executor [{:keys [threads queue-length]}] + (->FixedThreadExecutor threads queue-length)) diff --git a/src/asciinema/endpoint/asciicasts.clj b/src/asciinema/endpoint/asciicasts.clj index 70af37d..eec30e4 100644 --- a/src/asciinema/endpoint/asciicasts.clj +++ b/src/asciinema/endpoint/asciicasts.clj @@ -3,6 +3,7 @@ [asciicast-database :as adb] [expiring-set :as exp-set] [file-store :as fstore] + [executor :as executor] [user-database :as udb]] [asciinema.model.asciicast :as asciicast] [asciinema.util.io :refer [with-tmp-dir]] @@ -37,7 +38,7 @@ (def png-ttl-days 7) -(defn asciicasts-endpoint [{:keys [db file-store exp-set]}] +(defn asciicasts-endpoint [{:keys [db file-store exp-set executor]}] (api {:exceptions {:handlers {:compojure.api.exception/default exception-handler}}} (context @@ -63,17 +64,19 @@ theme (assoc :theme theme) scale (assoc :scale (Integer/parseInt scale))) png-store-path (asciicast/png-store-path asciicast png-params)] - (when-not (exp-set/contains? exp-set png-store-path) - (with-tmp-dir [dir "asciinema-png-"] - (let [json-store-path (asciicast/json-store-path asciicast) - json-local-path (str dir "/asciicast.json") - png-local-path (str dir "/asciicast.png") - expires (-> png-ttl-days t/days t/from-now)] - (with-open [in (fstore/input-stream file-store json-store-path)] - (let [out (io/file json-local-path)] - (io/copy in out))) - (a2png json-local-path png-local-path png-params) - (fstore/put-file file-store (io/file png-local-path) png-store-path) - (exp-set/conj! exp-set png-store-path expires)))) - (fstore/serve-file file-store png-store-path {})) + (if (exp-set/contains? exp-set png-store-path) + (fstore/serve-file file-store png-store-path {}) + (executor/execute executor (fn [] + (with-tmp-dir [dir "asciinema-png-"] + (let [json-store-path (asciicast/json-store-path asciicast) + json-local-path (str dir "/asciicast.json") + png-local-path (str dir "/asciicast.png") + expires (-> png-ttl-days t/days t/from-now)] + (with-open [in (fstore/input-stream file-store json-store-path)] + (let [out (io/file json-local-path)] + (io/copy in out))) + (a2png json-local-path png-local-path png-params) + (fstore/put-file file-store (io/file png-local-path) png-store-path) + (exp-set/conj! exp-set png-store-path expires))) + (fstore/serve-file file-store png-store-path {}))))) (response/not-found))))))