From e5aec4d82ae051b61acc801add365b108b6d51d5 Mon Sep 17 00:00:00 2001 From: Aleteoryx Date: Fri, 22 Nov 2024 12:01:50 -0500 Subject: [PATCH] multiple algos, file reorg --- .../jetstream-stats.tcl | 0 config.ini => config.example.ini | 1 + main.tcl | 38 +++++++++++-------- src/algos.tcl | 9 +++++ src/algos/tclposting.tcl | 10 +++++ src/at.tcl | 15 ++++++++ config.tcl => src/config.tcl | 0 feed.tcl => src/feed.tcl | 27 ++++++++++--- httpd.tcl => src/httpd.tcl | 0 jetstream.tcl => src/jetstream.tcl | 17 +++++---- ws.tcl => src/ws.tcl | 0 11 files changed, 89 insertions(+), 28 deletions(-) rename jetstream-stats.tcl => bin/jetstream-stats.tcl (100%) rename config.ini => config.example.ini (75%) create mode 100644 src/algos.tcl create mode 100644 src/algos/tclposting.tcl create mode 100644 src/at.tcl rename config.tcl => src/config.tcl (100%) rename feed.tcl => src/feed.tcl (72%) rename httpd.tcl => src/httpd.tcl (100%) rename jetstream.tcl => src/jetstream.tcl (65%) rename ws.tcl => src/ws.tcl (100%) diff --git a/jetstream-stats.tcl b/bin/jetstream-stats.tcl similarity index 100% rename from jetstream-stats.tcl rename to bin/jetstream-stats.tcl diff --git a/config.ini b/config.example.ini similarity index 75% rename from config.ini rename to config.example.ini index 125695a..3121068 100644 --- a/config.ini +++ b/config.example.ini @@ -4,3 +4,4 @@ max_posts = 10000 [atproto] jetstream_host = jetstream2.us-east.bsky.network +publisher_did = ; your atproto did diff --git a/main.tcl b/main.tcl index d7ce83c..53e32c4 100755 --- a/main.tcl +++ b/main.tcl @@ -6,34 +6,42 @@ package require sqlite3 package require inifile package require sha1 -source config.tcl +source src/config.tcl -source ws.tcl -source jetstream.tcl -source httpd.tcl -source feed.tcl +source src/at.tcl +source src/ws.tcl +source src/jetstream.tcl +source src/httpd.tcl +source src/feed.tcl + +source src/algos.tcl set log [logger::init tclfeed] ${log}::info "tclfeed v0.0.1" -#::json::write indented 0 - ### DB STUFF ### sqlite3 db [::ini::value $config database path] -db eval {SELECT count(*) as n FROM sqlite_master WHERE name="posts";} values { - if ![set values(n)] { - ${log}::info "Initializing db!" - db eval {CREATE TABLE posts (uri TEXT NOT NULL PRIMARY KEY, ts TEXT NOT NULL);} + +foreach algo $::algos::list { + db eval {SELECT count(*) as n FROM sqlite_master WHERE name=$algo;} values { + if ![set values(n)] { + ${log}::info "Creating table \"$algo\"!" + if [info exists ::algos::${algo}::cols] { + db eval "CREATE TABLE $algo (uri TEXT NOT NULL PRIMARY KEY, ord TEXT NOT NULL, [set ::algos::${algo}::cols]);" + } else { + db eval "CREATE TABLE $algo (uri TEXT NOT NULL PRIMARY KEY, ord TEXT NOT NULL);" + } + } } } ### JETSTREAM ### -jetstream::listen [::ini::value $config atproto jetstream_host] db +jetstream::listen [::ini::value $config atproto jetstream_host] $::algos::list ### HTTPD ### -httpd::router main { - /xrpc/app.bsky.feed.getFeedSkeleton {::feed::serve_skeleton {tcl posts}} -} +httpd::router main [list \ + /xrpc/app.bsky.feed.getFeedSkeleton [list ::feed::serve_skeleton $::algos::list] \ +] httpd::listen main 3000 vwait nil diff --git a/src/algos.tcl b/src/algos.tcl new file mode 100644 index 0000000..e268842 --- /dev/null +++ b/src/algos.tcl @@ -0,0 +1,9 @@ +namespace eval ::algos { variable list {} } + +foreach file [glob src/algos/*.tcl] { + source $file +} + +foreach algo [namespace children ::algos] { + lappend ::algos::list [lindex [split $algo :] end] +} diff --git a/src/algos/tclposting.tcl b/src/algos/tclposting.tcl new file mode 100644 index 0000000..4b5e436 --- /dev/null +++ b/src/algos/tclposting.tcl @@ -0,0 +1,10 @@ +namespace eval ::algos::tclposting { + variable log [logger::init algos::tclposting] + proc intake {uri text data} { + variable log + if {[regexp -- {\s[Tt]cl(/[Tt]k)?(\.|,|\s)|^[Tt]cl(/[Tt]k)?(,|\s)|\s[Tt]cl(/[Tt]k)?$|\sTk(\.|,|\s)|^Tk(,|\s)|\sTk$|\.tcl|tclsh|tcl-lang|tcltk|tcllib|tklib|tcl\.tk|#tcllang} $text]} { + return [dict get $data commit record createdAt] + } + return {} + } +} diff --git a/src/at.tcl b/src/at.tcl new file mode 100644 index 0000000..cff3f70 --- /dev/null +++ b/src/at.tcl @@ -0,0 +1,15 @@ +namespace eval at { + # conformant enough, i hope + proc uri {uri} { + if {[string first "at://" $uri] != 0} return + set uri [string range $uri 5 end] + set uri [split $uri /] + + switch -- [llength $uri] { + 1 { return [list authority [lindex $uri 0]] } + 2 { return [list authority [lindex $uri 0] collection [lindex $uri 1]] } + 3 { return [list authority [lindex $uri 0] collection [lindex $uri 1] rkey [lindex $uri 2]] } + default { return {} } + } + } +} diff --git a/config.tcl b/src/config.tcl similarity index 100% rename from config.tcl rename to src/config.tcl diff --git a/feed.tcl b/src/feed.tcl similarity index 72% rename from feed.tcl rename to src/feed.tcl index b4686b6..392fb9d 100644 --- a/feed.tcl +++ b/src/feed.tcl @@ -1,5 +1,7 @@ namespace eval feed { proc ::feed::serve_skeleton {feeds _ sock path query headers} { + global config + if ![dict exists $query feed] { puts $sock "HTTP/1.0 400 Bad Request" puts $sock "Content-Type: text/plain" @@ -30,21 +32,34 @@ namespace eval feed { return } - if ![dict exists $feeds $feed] { - puts $sock "HTTP/1.0 404 Not Found" + set feed [::at::uri $feed] + + if {[llength $feed] != 6 || + [dict get $feed authority] != [::ini::value $config atproto publisher_did] || + [dict get $feed collection] != "app.bsky.feed.generator"} { + puts $sock "HTTP/1.0 400 Bad Request" puts $sock "Content-Type: text/plain" puts $sock "" - puts $sock "Not a feed." + puts $sock "Invalid at:// URI." puts $sock "" close $sock return } - set table [dict get $feeds $feed] + set feed [dict get $feed rkey] + if {[lsearch -exact $feeds $feed] == -1} { + puts $sock "HTTP/1.0 404 Not Found" + puts $sock "Content-Type: text/plain" + puts $sock "" + puts $sock "Not a feed." + puts $sock "" + close $sock + return + } set feed_data {} - foreach uri [db eval "SELECT uri FROM $table ORDER BY ts LIMIT $limit OFFSET $cursor;"] { + foreach uri [db eval "SELECT uri FROM $feed ORDER BY ord LIMIT $limit OFFSET $cursor;"] { lappend feed_data [::json::write object post [::json::write string $uri]] } set feed_data [::json::write array {*}$feed_data] @@ -54,7 +69,7 @@ namespace eval feed { puts $sock "Content-Type: application/json" puts $sock "Content-Length: [string length $feed_contents]" puts $sock "" - fconfigure $sock -translation lf + fconfigure $sock -translation binary puts -nonewline $sock $feed_contents close $sock return diff --git a/httpd.tcl b/src/httpd.tcl similarity index 100% rename from httpd.tcl rename to src/httpd.tcl diff --git a/jetstream.tcl b/src/jetstream.tcl similarity index 65% rename from jetstream.tcl rename to src/jetstream.tcl index 5582963..e8a9865 100644 --- a/jetstream.tcl +++ b/src/jetstream.tcl @@ -1,6 +1,6 @@ namespace eval ::jetstream { variable log [logger::init tclfeed::jetstream] - proc listen {host db} { + proc listen {host algos} { variable log ${log}::info "listening to jetstream on wss://$host/subscribe" @@ -10,7 +10,7 @@ namespace eval ::jetstream { list ::ws::c::connect $host 443 \ /subscribe?wantedCollections=app.bsky.feed.post on_ws_$postfix] - proc ::jetstream::on_ws_$postfix {sock mode data} [concat [list set db $db] \; [list set postfix $postfix] \; { + proc ::jetstream::on_ws_$postfix {sock mode data} [concat [list set postfix $postfix] \; [list set algos $algos] \; { variable log switch -- $mode { close { @@ -27,15 +27,18 @@ namespace eval ::jetstream { switch -- [dict get $data commit operation] { create { set text [dict get $data commit record text] - if {[regexp -- {\s[Tt]cl(/[Tt]k)?(\.|,|\s)|^[Tt]cl(/[Tt]k)?(,|\s)|\s[Tt]cl(/[Tt]k)?$|\sTk(\.|,|\s)|^Tk(,|\s)|\sTk$|\.tcl|tclsh|tcl-lang|tcltk|tcllib|tklib|tcl\.tk|#tcllang} $text]} { - set ts [dict get $data commit record createdAt] - if ![catch {$db eval {INSERT OR FAIL INTO posts (uri, ts) VALUES ($uri, $ts);}}] { - ${log}::info "new tclpost! https://bsky.app/profile/[dict get $data did]/post/[dict get $data commit rkey]" + foreach algo $algos { + if {[set ord [::algos::${algo}::intake $uri $text $data]] != {}} { + if ![catch {db eval "INSERT OR FAIL INTO $algo (uri, ord) VALUES (:uri, :ord);"}] { + ${log}::info "new post for feed \"$algo\"! https://bsky.app/profile/[dict get $data did]/post/[dict get $data commit rkey]" + } } } } delete { - $db eval {DELETE FROM posts WHERE uri = $uri;} + foreach algo $algos { + db eval "DELETE FROM $algo WHERE uri = :uri;" + } } } } diff --git a/ws.tcl b/src/ws.tcl similarity index 100% rename from ws.tcl rename to src/ws.tcl -- 2.45.2