diff --git a/Gemfile b/Gemfile index 676a105..705561e 100644 --- a/Gemfile +++ b/Gemfile @@ -57,6 +57,8 @@ gem "onnxruntime" gem "mini_magick" gem "sentry-ruby" +gem "whenever", require: false + gem "warning" group :cli, :development do diff --git a/Gemfile.lock b/Gemfile.lock index 4062d96..1c80623 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -63,6 +63,7 @@ GEM sshkit (~> 1.3) capistrano-systemd-multiservice (0.1.0.beta13) capistrano (~> 3.7) + chronic (0.10.2) coderay (1.1.3) concurrent-ruby (1.2.3) connection_pool (2.4.1) @@ -482,6 +483,8 @@ GEM http (~> 5.0) indieweb-endpoints (~> 8.0) nokogiri (>= 1.13) + whenever (1.0.0) + chronic (>= 0.6.3) xml-simple (1.1.9) rexml xrpc (0.1.7.2) @@ -561,6 +564,7 @@ DEPENDENCIES timecop warning webmention + whenever RUBY VERSION ruby 3.3.0p0 diff --git a/Rakefile b/Rakefile index f3ea50c..27f3ba1 100644 --- a/Rakefile +++ b/Rakefile @@ -80,6 +80,14 @@ namespace :blog do user_repo = Admin::Container["repos.user_repo"] user_repo.create(id: SecureRandom.uuid, email: args[:email]) end + + task clean_webmentions: ["blog:load_environment"] do + require "hanami/prepare" + require "que" + + Que.connection = Adamantium::Container["persistence.db"] + Adamantium::Jobs::ArchiveDeletedWebmentions.enqueue + end end namespace :tailwind do diff --git a/config/schedule.rb b/config/schedule.rb new file mode 100644 index 0000000..a398b6b --- /dev/null +++ b/config/schedule.rb @@ -0,0 +1,22 @@ +# Use this file to easily define all of your cron jobs. +# +# It's helpful, but not entirely necessary to understand cron before proceeding. +# http://en.wikipedia.org/wiki/Cron + +# Example: +# +# set :output, "/path/to/my/cron_log.log" +# +# every 2.hours do +# command "/usr/bin/some_great_command" +# runner "MyModel.some_method" +# rake "some:great:rake:task" +# end +# +every 1.days do + rake "blog:clean_webmentions" + rake "blog:load_from_letterboxd" + rake "blog:scrobble_podcasts" +end + +# Learn more: http://github.com/javan/whenever diff --git a/db/migrate/20240210231532_add_retrieval_attempt_to_webmentions.rb b/db/migrate/20240210231532_add_retrieval_attempt_to_webmentions.rb new file mode 100644 index 0000000..adfa129 --- /dev/null +++ b/db/migrate/20240210231532_add_retrieval_attempt_to_webmentions.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +ROM::SQL.migration do + change do + alter_table :webmentions do + add_column :retrieval_attempts, :integer, default: 0 + end + end +end diff --git a/db/migrate/20240210232136_add_last_checked_at_to_webmentions.rb b/db/migrate/20240210232136_add_last_checked_at_to_webmentions.rb new file mode 100644 index 0000000..9f7e1ba --- /dev/null +++ b/db/migrate/20240210232136_add_last_checked_at_to_webmentions.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +ROM::SQL.migration do + change do + alter_table :webmentions do + add_column :last_checked_at, :timestamp + end + end +end diff --git a/lib/adamantium/jobs/archive_deleted_webmentions.rb b/lib/adamantium/jobs/archive_deleted_webmentions.rb new file mode 100644 index 0000000..2e6629d --- /dev/null +++ b/lib/adamantium/jobs/archive_deleted_webmentions.rb @@ -0,0 +1,29 @@ +require "httparty" +require "que" + +module Adamantium + module Jobs + class ArchiveDeletedWebmentions < Que::Job + def run + webmention_repo = Admin::Container["repos.webmention_repo"] + + webmentions = webmention_repo.list_all + + webmentions.each do |webmention| + code = HTTParty.get(webmention.source_url, follow_redirects: false).code + + # try and fetch the source at least 5 times, just in case a self-hosted server is down for a bit. + if code >= 400 + if webmention.retrieval_attempts >= 5 + webmention_repo.update(webmention.id, published_at: nil) + else + webmention_repo.update(webmention.id, retrieval_attempts: (webmention.retrieval_attempts + 1)) + end + end + + webmention_repo.update(webmention.id, last_checked_at: Time.now) + end + end + end + end +end