Browse Source

don't pigz -11, support unarchiving gzip-encoded trains

master
eta 3 months ago
parent
commit
60fabe43b4
  1. 2
      deploy.txt
  2. 44
      trackernet.lisp

2
deploy.txt

@ -1,4 +1,4 @@
(ql:quickload '(drakma cl-conspack cxml cl-statsd cl-redis qbase64 archive cl-ansi-text))
(ql:quickload '(drakma cl-conspack cxml cl-statsd cl-redis qbase64 archive cl-ansi-text gzip-stream))
(load "trackernet.lisp")
(sb-ext:save-lisp-and-die "./intertube-scraper" :toplevel #'trackernet::main :executable t)

44
trackernet.lisp

@ -612,7 +612,7 @@
(handler-case
(uiop:wait-process
(uiop:launch-program
`("pigz" "-11" ,filename)))
`("pigz" ,filename)))
(error (e)
(format
*error-output*
@ -664,25 +664,31 @@
(namestring (archive::entry-pathname entry)))
(t (archive::name entry))))
(defun unarchive-trains-tar (path func &key match)
(defun unarchive-trains-tar (path func &key match (gzip-encoded (uiop:string-suffix-p path "gz")))
"Read trains from the archive at PATH, running FUNC on each decoded train object."
(ignore-errors ; The archive probably isn't going to be terminated
(archive:with-open-archive (tar path
:direction :input)
(archive:do-archive-entries (entry tar)
(let ((buf (flexi-streams:make-in-memory-output-stream))
(filename (ignore-errors (archive-entry-filename entry))))
(when (or
(not match)
(cl-ppcre:scan match filename))
(archive::transfer-entry-data-to-stream tar entry buf)
(funcall
func
(cons
filename
(cpk:decode
(subseq
(flexi-streams:get-output-stream-sequence buf) 0))))))))))
(with-open-file (in path
:direction :input
:element-type '(unsigned-byte 8))
(archive:with-open-archive (tar
(if gzip-encoded
(gzip-stream:make-gzip-input-stream in)
in)
:direction :input)
(ignore-errors ; The archive probably isn't going to be terminated
(archive:do-archive-entries (entry tar)
(let ((buf (flexi-streams:make-in-memory-output-stream))
(filename (ignore-errors (archive-entry-filename entry))))
(when (or
(not match)
(cl-ppcre:scan match filename))
(archive::transfer-entry-data-to-stream tar entry buf)
(funcall
func
(cons
filename
(cpk:decode
(subseq
(flexi-streams:get-output-stream-sequence buf) 0)))))))))))
(defun archive-trains (keys)
"Archive trains from KEYS, a list of Redis train sorted set keys."

Loading…
Cancel
Save