Browse Source

modify clickhouse DDL to make train ids strings, insert XR trains

master
eta 1 week ago
parent
commit
7a8e9519bd
  1. 5
      clickhouse_ddl.sql
  2. 13
      trackernet.lisp

5
clickhouse_ddl.sql

@ -11,7 +11,7 @@ CREATE TABLE default.trains @@ -11,7 +11,7 @@ CREATE TABLE default.trains
`intertube_archive_path` LowCardinality(String),
`train_id` UInt32,
`train_id` String,
`lcid` UInt32,
@ -54,5 +54,4 @@ CREATE TABLE default.trains @@ -54,5 +54,4 @@ CREATE TABLE default.trains
ENGINE = MergeTree
PARTITION BY line
ORDER BY observation_ts
SETTINGS index_granularity = 8192
SETTINGS index_granularity = 8192;

13
trackernet.lisp

@ -1229,12 +1229,11 @@ @@ -1229,12 +1229,11 @@
(statsd-inc "intertube.archived")
(statsd-counter "intertube.archived-records" records-archived)
(conspack-encode-to-archive tar-out filename archive-data)
(unless (string= line-code "X")
(handler-case
(clickhouse-insert-live clickhouse-data)
(serious-condition (e)
(log:error "clickhouse insert failed (adding to clickhouse-failures): err: ~A" e)
(red:sadd "clickhouse-failures" (cpk-base64 clickhouse-data)))))
(handler-case
(clickhouse-insert-live clickhouse-data)
(serious-condition (e)
(log:error "clickhouse insert failed (adding to clickhouse-failures): err: ~A" e)
(red:sadd "clickhouse-failures" (cpk-base64 clickhouse-data))))
(when (and ltid (string= line-code "X"))
;; delete XR live-trains immediately, or else they'll stick around
;; forever
@ -1815,7 +1814,7 @@ @@ -1815,7 +1814,7 @@
(- archived-universal-ts 2208988800)
filename
*clickhouse-file*
(nilify-zeros-parse train-id)
train-id
(nilify-zeros-parse lcid)
(nilify-zeros-parse set-no)
(nilify-zeros-parse trip-no)

Loading…
Cancel
Save