Browse Source

chatlogs exporter, can do biboumi sqlite databases

master
eta 11 months ago
commit
810a6e7855
  1. 4
      .gitignore
  2. 3
      .idea/.gitignore
  3. 6
      .idea/misc.xml
  4. 8
      .idea/modules.xml
  5. 6
      .idea/vcs.xml
  6. 218
      Cargo.lock
  7. 14
      Cargo.toml
  8. 12
      chatlogs-converter.iml
  9. 100
      src/biboumi.rs
  10. 0
      src/conversations.rs
  11. 2
      src/lib.rs

4
.gitignore vendored

@ -0,0 +1,4 @@ @@ -0,0 +1,4 @@
/target
out/
biboumi/
*.sqlite*

3
.idea/.gitignore vendored

@ -0,0 +1,3 @@ @@ -0,0 +1,3 @@
# Default ignored files
/shelf/
/workspace.xml

6
.idea/misc.xml

@ -0,0 +1,6 @@ @@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>

8
.idea/modules.xml

@ -0,0 +1,8 @@ @@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/chatlogs-converter.iml" filepath="$PROJECT_DIR$/chatlogs-converter.iml" />
</modules>
</component>
</project>

6
.idea/vcs.xml

@ -0,0 +1,6 @@ @@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>

218
Cargo.lock generated

@ -0,0 +1,218 @@ @@ -0,0 +1,218 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "ahash"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
dependencies = [
"getrandom",
"once_cell",
"version_check",
]
[[package]]
name = "autocfg"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chatlogs-converter"
version = "0.1.0"
dependencies = [
"chrono",
"rusqlite",
]
[[package]]
name = "chrono"
version = "0.4.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73"
dependencies = [
"libc",
"num-integer",
"num-traits",
"time",
"winapi",
]
[[package]]
name = "fallible-iterator"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "getrandom"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "hashbrown"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
dependencies = [
"ahash",
]
[[package]]
name = "hashlink"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf"
dependencies = [
"hashbrown",
]
[[package]]
name = "libc"
version = "0.2.124"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50"
[[package]]
name = "libsqlite3-sys"
version = "0.24.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14"
dependencies = [
"pkg-config",
"vcpkg",
]
[[package]]
name = "memchr"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]]
name = "num-integer"
version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
dependencies = [
"autocfg",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
dependencies = [
"autocfg",
]
[[package]]
name = "once_cell"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
[[package]]
name = "pkg-config"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
[[package]]
name = "rusqlite"
version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85127183a999f7db96d1a976a309eebbfb6ea3b0b400ddd8340190129de6eb7a"
dependencies = [
"bitflags",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
"libsqlite3-sys",
"memchr",
"smallvec",
]
[[package]]
name = "smallvec"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
[[package]]
name = "time"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "version_check"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

14
Cargo.toml

@ -0,0 +1,14 @@ @@ -0,0 +1,14 @@
[package]
name = "chatlogs-converter"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
rusqlite = "0.27"
chrono = "0.4"
[[bin]]
name = "biboumi"
path = "src/biboumi.rs"

12
chatlogs-converter.iml

@ -0,0 +1,12 @@ @@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="RUST_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

100
src/biboumi.rs

@ -0,0 +1,100 @@ @@ -0,0 +1,100 @@
use chrono::{TimeZone, Utc};
use rusqlite::types::ValueRef;
use rusqlite::{params, Connection};
use std::collections::BTreeMap;
use std::fs;
use std::fs::File;
use std::io::Write;
use std::path::Path;
static DATABASE: &'static str = "./biboumi.sqlite";
static OUTDIR: &'static str = "./out";
fn main() {
if !Path::new(DATABASE).exists() {
panic!("{} does not exist", DATABASE);
}
let conn = Connection::open(DATABASE).unwrap();
// get a list of all buffers
let mut stmt = conn
.prepare("SELECT DISTINCT ircchanname_, ircservername_ FROM muclogline_")
.unwrap();
let rows = stmt
.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
})
.unwrap();
let buffers = rows.collect::<Result<Vec<_>, _>>().unwrap();
eprintln!("found {} buffers in database", buffers.len());
// iterate through each buffer and print it out
for (chan, server) in buffers {
let path = format!("{}/{}/{}", OUTDIR, server, chan);
eprint!("writing {}...", path);
fs::create_dir_all(&path).unwrap();
// get all messages for this buffer
let mut stmt = conn.prepare("SELECT date_, nick_, body_ FROM muclogline_ WHERE ircchanname_ = $1 AND ircservername_ = $2 ORDER BY date_ ASC")
.unwrap();
let rows = stmt
.query_map(params![&chan, &server], |row| {
let text = match row.get_ref(2)? {
ValueRef::Text(txt) => String::from_utf8_lossy(txt).into_owned(),
x => panic!("{:?} is not text", x),
};
Ok((
Utc.timestamp(row.get(0)?, 0),
row.get::<_, String>(1)?,
text,
))
})
.unwrap();
let mut lines_by_date = BTreeMap::new();
for row in rows {
let (ts, nick, body) = row.unwrap();
let entry = lines_by_date.entry(ts.date()).or_insert(vec![]);
entry.push((ts, nick, body));
}
let lines_by_date = lines_by_date.into_iter().collect::<Vec<_>>();
for (idx, (date, lines)) in lines_by_date.iter().enumerate() {
let path = format!("{}/{}.txt", path, date.format("%Y-%m-%d"));
let mut file = File::create(path).unwrap();
// write a header invoking a MediaWiki template
let prevdate = if idx != 0 {
format!("|prevdate={}", lines_by_date[idx - 1].0.format("%Y-%m-%d"))
} else {
"".into()
};
let nextdate = if idx != lines_by_date.len() - 1 {
format!("|nextdate={}", lines_by_date[idx + 1].0.format("%Y-%m-%d"))
} else {
"".into()
};
file.write_all(
format!(
"{{{{logpage|source=biboumi|server={}|channel={}|date={}{}{}}}}}<pre>\n",
server,
chan,
date.format("%Y-%m-%d"),
nextdate,
prevdate
)
.as_bytes(),
)
.unwrap();
// write the actual log lines
for (ts, nick, body) in lines {
let ts = format!("{}", ts.format("%H:%M:%S"));
let line = if nick == "" {
format!("{} {}\n", ts, body)
} else if body.starts_with("/me ") {
format!("{} * {} {}\n", ts, nick, &body[4..])
} else {
format!("{} <{}> {}\n", ts, nick, body)
};
file.write_all(line.as_bytes()).unwrap();
}
file.write_all("</pre>\n".as_bytes()).unwrap();
}
eprintln!("wrote {} files", lines_by_date.len());
}
}

0
src/conversations.rs

2
src/lib.rs

@ -0,0 +1,2 @@ @@ -0,0 +1,2 @@
mod biboumi;
mod conversations;
Loading…
Cancel
Save