From 44523daeb10158e5f081d9cd25a1fccf54d401e7 Mon Sep 17 00:00:00 2001 From: Aleteoryx Date: Fri, 22 Nov 2024 13:50:06 -0500 Subject: [PATCH] misc improvements --- config.example.ini | 3 +-- main.tcl | 27 +++++++++++++++++++++++++-- src/algos/tclposting.tcl | 29 ++++++++++++++++++++++++++--- src/config.tcl | 25 ++++++++++++++++++++++--- src/feed.tcl | 20 +++++++++++--------- 5 files changed, 85 insertions(+), 19 deletions(-) diff --git a/config.example.ini b/config.example.ini index 3121068..8c9400c 100644 --- a/config.example.ini +++ b/config.example.ini @@ -1,7 +1,6 @@ [database] path = posts.db -max_posts = 10000 [atproto] jetstream_host = jetstream2.us-east.bsky.network -publisher_did = ; your atproto did +; publisher_did = diff --git a/main.tcl b/main.tcl index 53e32c4..c633ef0 100755 --- a/main.tcl +++ b/main.tcl @@ -1,4 +1,7 @@ #!/bin/env tclsh + +cd [file dirname [dict get [info frame [info frame]] file]] + package require logger package require json package require json::write @@ -17,6 +20,7 @@ source src/feed.tcl source src/algos.tcl set log [logger::init tclfeed] +set logd [logger::init tclfeed::database] ${log}::info "tclfeed v0.0.1" ### DB STUFF ### @@ -35,14 +39,33 @@ foreach algo $::algos::list { } } +proc db_cleanup {interval max_posts} { + global ::algos::list logd + + ${logd}::info "Performing database cleanup!" + + foreach algo $::algos::list { + # slightly cursed + db eval "DELETE FROM $algo WHERE uri IN (SELECT uri FROM $algo ORDER BY ord DESC LIMIT -1 OFFSET :max_posts); REINDEX $algo;" + } + + db eval "VACUUM"; + + ${logd}::info "DB cleanup completed without error!" + ${logd}::debug "Scheduling next cleanup for [clock format [expr {[clock seconds] + $interval}]]." + + after [expr {$interval * 1000}] [list db_cleanup $interval $max_posts] +} +db_cleanup [::ini::value $config database cleanup_interval] [::ini::value $config database max_posts] + ### JETSTREAM ### jetstream::listen [::ini::value $config atproto jetstream_host] $::algos::list ### HTTPD ### httpd::router main [list \ - /xrpc/app.bsky.feed.getFeedSkeleton [list ::feed::serve_skeleton $::algos::list] \ + /xrpc/app.bsky.feed.getFeedSkeleton [list ::feed::serve_skeleton $::algos::list [::ini::value $config atproto publisher_did]] \ ] -httpd::listen main 3000 +httpd::listen main [::ini::value $config httpd port] vwait nil diff --git a/src/algos/tclposting.tcl b/src/algos/tclposting.tcl index 4b5e436..41b1b1b 100644 --- a/src/algos/tclposting.tcl +++ b/src/algos/tclposting.tcl @@ -2,9 +2,32 @@ namespace eval ::algos::tclposting { variable log [logger::init algos::tclposting] proc intake {uri text data} { variable log - if {[regexp -- {\s[Tt]cl(/[Tt]k)?(\.|,|\s)|^[Tt]cl(/[Tt]k)?(,|\s)|\s[Tt]cl(/[Tt]k)?$|\sTk(\.|,|\s)|^Tk(,|\s)|\sTk$|\.tcl|tclsh|tcl-lang|tcltk|tcllib|tklib|tcl\.tk|#tcllang} $text]} { - return [dict get $data commit record createdAt] + if {![regexp -- [join { + {\s[Tt]cl(/[Tt]k)?(\.|,|\s)} + {^[Tt]cl(/[Tt]k)?(,|\s)} + {\s[Tt]cl(/[Tt]k)?$} + {\sTk(\.|,|\s)} + {^Tk(,|\s)} + {\sTk$} + {\.tcl} + {tclsh} + {tcl-lang} + {tcltk} + {tcllib} + {tklib} + {tcl\.tk} + {#tcllang}} | ] $text]} \ + { + return {} } - return {} + + if {[regexp -nocase -- [join { + {Title Tk} + {Comic-Con Tk}} | ] $text]} \ + { + return {} + } + + return [dict get $data commit record createdAt] } } diff --git a/src/config.tcl b/src/config.tcl index b20b968..71f5d99 100644 --- a/src/config.tcl +++ b/src/config.tcl @@ -2,23 +2,42 @@ if {$argc != 1} { puts stderr "Usage: $argv0 " exit -1 } -if [catch {set config [::ini::open [lindex $argv 0]]} result] { +if [catch {set config [::ini::open [lindex $argv 0] r]} result] { puts stderr "Couldn't open config: $result" exit -1 } namespace eval ::config { + variable log [logger::init "tclfeed::config"] proc require {config section key} { - if ![ini::exists $config $section $key] { + variable log + if ![::ini::exists $config $section $key] { puts stderr "Missing [$section/$key] in config!" exit -2 } + ${log}::info "\[$section/$key\] = [::ini::value $config $section $key]" + } + proc optional {config section key value} { + variable log + if ![::ini::exists $config $section $key] { + ::ini::set $config $section $key $value + ${log}::info "\[$section/$key\] = [::ini::value $config $section $key] (default)" + } else { + ${log}::info "\[$section/$key\] = [::ini::value $config $section $key]" + } } proc check {config} { + variable log + ${log}::info "Loading config [::ini::filename $config]..." require $config database path - require $config database max_posts + optional $config database max_posts 10000 + optional $config database cleanup_interval 600 require $config atproto jetstream_host + require $config atproto publisher_did + + optional $config httpd port 3000 + ${log}::info "Config loaded!..." } } diff --git a/src/feed.tcl b/src/feed.tcl index 392fb9d..1a00a09 100644 --- a/src/feed.tcl +++ b/src/feed.tcl @@ -1,7 +1,5 @@ namespace eval feed { - proc ::feed::serve_skeleton {feeds _ sock path query headers} { - global config - + proc ::feed::serve_skeleton {feeds publisher _ sock path query headers} { if ![dict exists $query feed] { puts $sock "HTTP/1.0 400 Bad Request" puts $sock "Content-Type: text/plain" @@ -20,8 +18,8 @@ namespace eval feed { if [catch { set feed [dict get $query feed] - set cursor [expr {min(0, [dict get $query cursor])}] - set limit [expr {min(1, max(100, [dict get $query limit]))}] + set cursor [expr {max(0, [dict get $query cursor])}] + set limit [expr {max(1, min(100, [dict get $query limit]))}] }] { puts $sock "HTTP/1.0 400 Bad Request" puts $sock "Content-Type: text/plain" @@ -35,7 +33,7 @@ namespace eval feed { set feed [::at::uri $feed] if {[llength $feed] != 6 || - [dict get $feed authority] != [::ini::value $config atproto publisher_did] || + [dict get $feed authority] != $publisher || [dict get $feed collection] != "app.bsky.feed.generator"} { puts $sock "HTTP/1.0 400 Bad Request" puts $sock "Content-Type: text/plain" @@ -59,11 +57,11 @@ namespace eval feed { } set feed_data {} - foreach uri [db eval "SELECT uri FROM $feed ORDER BY ord LIMIT $limit OFFSET $cursor;"] { + foreach uri [db eval "SELECT uri FROM $feed ORDER BY ord DESC LIMIT $limit OFFSET $cursor;"] { lappend feed_data [::json::write object post [::json::write string $uri]] } - set feed_data [::json::write array {*}$feed_data] - set feed_contents [::json::write object cursor [expr {$cursor + $limit}] feed $feed_data] + set feed_data_arr [::json::write array {*}$feed_data] + set feed_contents [::json::write object cursor [expr {$cursor + min([llength $feed_data], $limit)}] feed $feed_data_arr] puts $sock "HTTP/1.0 200 OK" puts $sock "Content-Type: application/json" @@ -74,4 +72,8 @@ namespace eval feed { close $sock return } + + proc ::feed::serve_generator_description {feeds publisher _ sock path query headers} { + + } } -- 2.45.2