chatlogs exporter, can do biboumi sqlite databases

11 months ago · 810a6e7855
11 changed files with 373 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,4 @@
				@@ -0,0 +1,4 @@
+/target
+out/
+biboumi/
+*.sqlite*
--- a/.idea/.gitignore
+++ b/.idea/.gitignore
@ -0,0 +1,3 @@
				@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml
--- a/.idea/misc.xml
+++ b/.idea/misc.xml
@ -0,0 +1,6 @@
				@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager">
+    <output url="file://$PROJECT_DIR$/out" />
+  </component>
+</project>
--- a/.idea/modules.xml
+++ b/.idea/modules.xml
@ -0,0 +1,8 @@
				@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/chatlogs-converter.iml" filepath="$PROJECT_DIR$/chatlogs-converter.iml" />
+    </modules>
+  </component>
+</project>
--- a/.idea/vcs.xml
+++ b/.idea/vcs.xml
@ -0,0 +1,6 @@
				@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>
--- a/Cargo.lock
+++ b/Cargo.lock
@ -0,0 +1,218 @@
				@@ -0,0 +1,218 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "ahash"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
+dependencies = [
+ "getrandom",
+ "once_cell",
+ "version_check",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "chatlogs-converter"
+version = "0.1.0"
+dependencies = [
+ "chrono",
+ "rusqlite",
+]
+
+[[package]]
+name = "chrono"
+version = "0.4.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73"
+dependencies = [
+ "libc",
+ "num-integer",
+ "num-traits",
+ "time",
+ "winapi",
+]
+
+[[package]]
+name = "fallible-iterator"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+
+[[package]]
+name = "fallible-streaming-iterator"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
+
+[[package]]
+name = "getrandom"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+dependencies = [
+ "ahash",
+]
+
+[[package]]
+name = "hashlink"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf"
+dependencies = [
+ "hashbrown",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.124"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50"
+
+[[package]]
+name = "libsqlite3-sys"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14"
+dependencies = [
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
+name = "memchr"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
+
+[[package]]
+name = "num-integer"
+version = "0.1.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
+dependencies = [
+ "autocfg",
+ "num-traits",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
+
+[[package]]
+name = "pkg-config"
+version = "0.3.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
+
+[[package]]
+name = "rusqlite"
+version = "0.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85127183a999f7db96d1a976a309eebbfb6ea3b0b400ddd8340190129de6eb7a"
+dependencies = [
+ "bitflags",
+ "fallible-iterator",
+ "fallible-streaming-iterator",
+ "hashlink",
+ "libsqlite3-sys",
+ "memchr",
+ "smallvec",
+]
+
+[[package]]
+name = "smallvec"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
+
+[[package]]
+name = "time"
+version = "0.1.43"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
+[[package]]
+name = "version_check"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+
+[[package]]
+name = "wasi"
+version = "0.10.2+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -0,0 +1,14 @@
				@@ -0,0 +1,14 @@
+[package]
+name = "chatlogs-converter"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+rusqlite = "0.27"
+chrono = "0.4"
+
+[[bin]]
+name = "biboumi"
+path = "src/biboumi.rs"
--- a/chatlogs-converter.iml
+++ b/chatlogs-converter.iml
@ -0,0 +1,12 @@
				@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="RUST_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
+      <excludeFolder url="file://$MODULE_DIR$/target" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>
--- a/src/biboumi.rs
+++ b/src/biboumi.rs
@ -0,0 +1,100 @@
				@@ -0,0 +1,100 @@
+use chrono::{TimeZone, Utc};
+use rusqlite::types::ValueRef;
+use rusqlite::{params, Connection};
+use std::collections::BTreeMap;
+use std::fs;
+use std::fs::File;
+use std::io::Write;
+use std::path::Path;
+
+static DATABASE: &'static str = "./biboumi.sqlite";
+static OUTDIR: &'static str = "./out";
+
+fn main() {
+    if !Path::new(DATABASE).exists() {
+        panic!("{} does not exist", DATABASE);
+    }
+    let conn = Connection::open(DATABASE).unwrap();
+    // get a list of all buffers
+    let mut stmt = conn
+        .prepare("SELECT DISTINCT ircchanname_, ircservername_ FROM muclogline_")
+        .unwrap();
+    let rows = stmt
+        .query_map([], |row| {
+            Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
+        })
+        .unwrap();
+    let buffers = rows.collect::<Result<Vec<_>, _>>().unwrap();
+    eprintln!("found {} buffers in database", buffers.len());
+
+    // iterate through each buffer and print it out
+    for (chan, server) in buffers {
+        let path = format!("{}/{}/{}", OUTDIR, server, chan);
+        eprint!("writing {}...", path);
+        fs::create_dir_all(&path).unwrap();
+        // get all messages for this buffer
+        let mut stmt = conn.prepare("SELECT date_, nick_, body_ FROM muclogline_ WHERE ircchanname_ = $1 AND ircservername_ = $2 ORDER BY date_ ASC")
+            .unwrap();
+        let rows = stmt
+            .query_map(params![&chan, &server], |row| {
+                let text = match row.get_ref(2)? {
+                    ValueRef::Text(txt) => String::from_utf8_lossy(txt).into_owned(),
+                    x => panic!("{:?} is not text", x),
+                };
+                Ok((
+                    Utc.timestamp(row.get(0)?, 0),
+                    row.get::<_, String>(1)?,
+                    text,
+                ))
+            })
+            .unwrap();
+        let mut lines_by_date = BTreeMap::new();
+        for row in rows {
+            let (ts, nick, body) = row.unwrap();
+            let entry = lines_by_date.entry(ts.date()).or_insert(vec![]);
+            entry.push((ts, nick, body));
+        }
+        let lines_by_date = lines_by_date.into_iter().collect::<Vec<_>>();
+        for (idx, (date, lines)) in lines_by_date.iter().enumerate() {
+            let path = format!("{}/{}.txt", path, date.format("%Y-%m-%d"));
+            let mut file = File::create(path).unwrap();
+            // write a header invoking a MediaWiki template
+            let prevdate = if idx != 0 {
+                format!("|prevdate={}", lines_by_date[idx - 1].0.format("%Y-%m-%d"))
+            } else {
+                "".into()
+            };
+            let nextdate = if idx != lines_by_date.len() - 1 {
+                format!("|nextdate={}", lines_by_date[idx + 1].0.format("%Y-%m-%d"))
+            } else {
+                "".into()
+            };
+            file.write_all(
+                format!(
+                    "{{{{logpage|source=biboumi|server={}|channel={}|date={}{}{}}}}}<pre>\n",
+                    server,
+                    chan,
+                    date.format("%Y-%m-%d"),
+                    nextdate,
+                    prevdate
+                )
+                .as_bytes(),
+            )
+            .unwrap();
+            // write the actual log lines
+            for (ts, nick, body) in lines {
+                let ts = format!("{}", ts.format("%H:%M:%S"));
+                let line = if nick == "" {
+                    format!("{} {}\n", ts, body)
+                } else if body.starts_with("/me ") {
+                    format!("{} * {} {}\n", ts, nick, &body[4..])
+                } else {
+                    format!("{} <{}> {}\n", ts, nick, body)
+                };
+                file.write_all(line.as_bytes()).unwrap();
+            }
+            file.write_all("</pre>\n".as_bytes()).unwrap();
+        }
+        eprintln!("wrote {} files", lines_by_date.len());
+    }
+}
--- a/src/conversations.rs
+++ b/src/conversations.rs
--- a/src/lib.rs
+++ b/src/lib.rs
@ -0,0 +1,2 @@
				@@ -0,0 +1,2 @@
+mod biboumi;
+mod conversations;