Auto-expire png files after 7 days

dev/resources/dev.edn

@@ -1,5 +1,6 @@
 {:components
- {:file-store #var asciinema.component.local-file-store/local-file-store}
+ {:file-store #var asciinema.component.local-file-store/local-file-store
+  :exp-set #var asciinema.component.mem-expiring-set/mem-expiring-set}
  :config
  {:app
   {:middleware

project.clj

@@ -20,6 +20,7 @@
                  [clj-aws-s3 "0.3.10" :exclusions [joda-time com.fasterxml.jackson.core/jackson-core com.fasterxml.jackson.core/jackson-annotations]]
                  [aleph "0.4.1"]
                  [pandect "0.6.1"]
+                 [com.taoensso/carmine "2.15.1"]
                  [org.slf4j/slf4j-nop "1.7.21"]
                  [org.webjars/normalize.css "3.0.2"]
                  [duct/hikaricp-component "0.1.0"]

resources/asciinema/system.edn

@@ -3,14 +3,15 @@
   :http #var asciinema.component.aleph/aleph-server
   :db   #var asciinema.component.db/hikaricp
   :ragtime #var duct.component.ragtime/ragtime
-  :file-store #var asciinema.component.s3-file-store/s3-file-store}
+  :file-store #var asciinema.component.s3-file-store/s3-file-store
+  :exp-set #var asciinema.component.redis-client/redis-client}
  :endpoints
  {:asciicasts #var asciinema.endpoint.asciicasts/asciicasts-endpoint}
  :dependencies
  {:http [:app]
   :app  [:asciicasts]
   :ragtime [:db]
-  :asciicasts [:db :file-store]}
+  :asciicasts [:db :file-store :exp-set]}
  :config
  {:app
   {:middleware
@@ -60,4 +61,7 @@
   {:cred {:access-key s3-access-key
           :secret-key s3-secret-key}
    :bucket s3-bucket
-   :path-prefix "uploads/"}}}
+   :path-prefix "uploads/"}
+  :exp-set
+  {:host redis-host
+   :port redis-port}}}

src/asciinema/boundary/expiring_set.clj

@@ -0,0 +1,6 @@
+(ns asciinema.boundary.expiring-set
+  (:refer-clojure :exclude [conj! contains?]))
+
+(defprotocol ExpiringSet
+  (conj! [this value expires-at])
+  (contains? [this value]))

src/asciinema/component/mem_expiring_set.clj

@@ -0,0 +1,14 @@
+(ns asciinema.component.mem-expiring-set
+  (:require [asciinema.boundary.expiring-set :as exp-set]))
+
+(defrecord MemExpiringSet [store]
+  exp-set/ExpiringSet
+
+  (conj! [this value _expires-at]
+    (swap! store conj value))
+
+  (contains? [this value]
+    (contains? @store value)))
+
+(defn mem-expiring-set [{:keys [store]}]
+  (->MemExpiringSet (or store (atom #{}))))

src/asciinema/component/redis_client.clj

@@ -0,0 +1,28 @@
+(ns asciinema.component.redis-client
+  (:require [asciinema.boundary.expiring-set :as exp-set]
+            [clj-time.core :as t]
+            [clj-time.local :as tl]
+            [com.stuartsierra.component :as component]
+            [taoensso.carmine :as car]))
+
+(defrecord RedisClient [host port]
+  component/Lifecycle
+  (start [component]
+    (if (:listener component)
+      component
+      (let [conn {:pool {} :spec {:host host :port port}}]
+        (assoc component :conn conn))))
+  (stop [component]
+    (if (:conn component)
+      (dissoc component :conn)
+      component))
+
+  exp-set/ExpiringSet
+  (conj! [this value expires-at]
+    (let [seconds (t/in-seconds (t/interval (tl/local-now) expires-at))]
+      (car/wcar (:conn this) (car/setex value seconds true))))
+  (contains? [this value]
+    (car/as-bool (car/wcar (:conn this) (car/exists value)))))
+
+(defn redis-client [{:keys [host port]}]
+  (->RedisClient host port))

src/asciinema/endpoint/asciicasts.clj

@@ -1,17 +1,19 @@
 (ns asciinema.endpoint.asciicasts
   (:require [asciinema.boundary
              [asciicast-database :as adb]
+             [expiring-set :as exp-set]
              [file-store :as fstore]
              [user-database :as udb]]
             [asciinema.model.asciicast :as asciicast]
             [asciinema.util.io :refer [with-tmp-dir]]
+            [clj-time.core :as t]
             [clojure.java.io :as io]
             [clojure.java.shell :as shell]
+            [clojure.string :as str]
             [compojure.api.sweet :refer :all]
             [environ.core :refer [env]]
             [ring.util.http-response :as response]
-            [schema.core :as s]
+            [schema.core :as s]))
-            [clojure.string :as str]))
 
 (defn exception-handler [^Exception e data request]
   (throw e))
@@ -33,7 +35,9 @@
 
 (def Theme (apply s/enum asciicast/themes))
 
-(defn asciicasts-endpoint [{:keys [db file-store]}]
+(def png-ttl-days 7)
+
+(defn asciicasts-endpoint [{:keys [db file-store exp-set]}]
   (api
    {:exceptions {:handlers {:compojure.api.exception/default exception-handler}}}
    (context
@@ -58,15 +62,18 @@
                               time (assoc :snapshot-at time)
                               theme (assoc :theme theme)
                               scale (assoc :scale (Integer/parseInt scale)))
-                 json-store-path (asciicast/json-store-path asciicast)
                  png-store-path (asciicast/png-store-path asciicast png-params)]
-             (with-tmp-dir [dir "asciinema-png-"]
+             (when-not (exp-set/contains? exp-set png-store-path)
-               (let [json-local-path (str dir "/asciicast.json")
+               (with-tmp-dir [dir "asciinema-png-"]
-                     png-local-path (str dir "/asciicast.png")]
+                 (let [json-store-path (asciicast/json-store-path asciicast)
-                 (with-open [in (fstore/input-stream file-store json-store-path)]
+                       json-local-path (str dir "/asciicast.json")
-                   (let [out (io/file json-local-path)]
+                       png-local-path (str dir "/asciicast.png")
-                     (io/copy in out)))
+                       expires (-> png-ttl-days t/days t/from-now)]
-                 (a2png json-local-path png-local-path png-params)
+                   (with-open [in (fstore/input-stream file-store json-store-path)]
-                 (fstore/put-file file-store (io/file png-local-path) png-store-path)))
+                     (let [out (io/file json-local-path)]
+                       (io/copy in out)))
+                   (a2png json-local-path png-local-path png-params)
+                   (fstore/put-file file-store (io/file png-local-path) png-store-path)
+                   (exp-set/conj! exp-set png-store-path expires))))
              (fstore/serve-file file-store png-store-path {}))
            (response/not-found))))))

src/asciinema/main.clj

@@ -14,7 +14,9 @@
                   'bugsnag-key (:bugsnag-key env)
                   's3-bucket (:s3-bucket env)
                   's3-access-key (:s3-access-key env)
-                  's3-secret-key (:s3-secret-key env)}
+                  's3-secret-key (:s3-secret-key env)
+                  'redis-host (:redis-host env "localhost")
+                  'redis-port (Integer/parseInt (:redis-port env "6379"))}
         system   (->> (load-system [(io/resource "asciinema/system.edn")] bindings)
                       (component/start))]
     (add-shutdown-hook ::stop-system #(component/stop system))