don't pigz -11, support unarchiving gzip-encoded trains

deploy.txt

@@ -1,4 +1,4 @@
-(ql:quickload '(drakma cl-conspack cxml cl-statsd cl-redis qbase64 archive cl-ansi-text))
+(ql:quickload '(drakma cl-conspack cxml cl-statsd cl-redis qbase64 archive cl-ansi-text gzip-stream))
 (load "trackernet.lisp")
 (sb-ext:save-lisp-and-die "./intertube-scraper" :toplevel #'trackernet::main :executable t) 
 

trackernet.lisp

@@ -612,7 +612,7 @@
               (handler-case
                 (uiop:wait-process
                  (uiop:launch-program
-                  `("pigz" "-11" ,filename)))
+                  `("pigz" ,filename)))
                 (error (e)
                   (format
                    *error-output*
@@ -664,25 +664,31 @@
      (namestring (archive::entry-pathname entry)))
     (t (archive::name entry))))
 
-(defun unarchive-trains-tar (path func &key match)
+(defun unarchive-trains-tar (path func &key match (gzip-encoded (uiop:string-suffix-p path "gz")))
   "Read trains from the archive at PATH, running FUNC on each decoded train object."
-  (ignore-errors ; The archive probably isn't going to be terminated
-   (archive:with-open-archive (tar path
-                               :direction :input)
-     (archive:do-archive-entries (entry tar)
-       (let ((buf (flexi-streams:make-in-memory-output-stream))
-             (filename (ignore-errors (archive-entry-filename entry))))
-         (when (or
-                (not match)
-                (cl-ppcre:scan match filename))
-           (archive::transfer-entry-data-to-stream tar entry buf)
-           (funcall
-            func
-            (cons
-             filename
-             (cpk:decode
-              (subseq
-               (flexi-streams:get-output-stream-sequence buf) 0))))))))))
+  (with-open-file (in path
+                      :direction :input
+                      :element-type '(unsigned-byte 8))
+    (archive:with-open-archive (tar
+                                (if gzip-encoded
+                                    (gzip-stream:make-gzip-input-stream in)
+                                    in)
+                                :direction :input)
+      (ignore-errors ; The archive probably isn't going to be terminated
+       (archive:do-archive-entries (entry tar)
+         (let ((buf (flexi-streams:make-in-memory-output-stream))
+               (filename (ignore-errors (archive-entry-filename entry))))
+           (when (or
+                  (not match)
+                  (cl-ppcre:scan match filename))
+             (archive::transfer-entry-data-to-stream tar entry buf)
+             (funcall
+              func
+              (cons
+               filename
+               (cpk:decode
+                (subseq
+                 (flexi-streams:get-output-stream-sequence buf) 0)))))))))))
 
 (defun archive-trains (keys)
   "Archive trains from KEYS, a list of Redis train sorted set keys."