Archive webmentions if they have been removed from their source

Gemfile

@@ -57,6 +57,8 @@ gem "onnxruntime"
 gem "mini_magick"
 gem "sentry-ruby"
 
+gem "whenever", require: false
+
 gem "warning"
 
 group :cli, :development do

Gemfile.lock

@@ -63,6 +63,7 @@ GEM
       sshkit (~> 1.3)
     capistrano-systemd-multiservice (0.1.0.beta13)
       capistrano (~> 3.7)
+    chronic (0.10.2)
     coderay (1.1.3)
     concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
@@ -482,6 +483,8 @@ GEM
       http (~> 5.0)
       indieweb-endpoints (~> 8.0)
       nokogiri (>= 1.13)
+    whenever (1.0.0)
+      chronic (>= 0.6.3)
     xml-simple (1.1.9)
       rexml
     xrpc (0.1.7.2)
@@ -561,6 +564,7 @@ DEPENDENCIES
   timecop
   warning
   webmention
+  whenever
 
 RUBY VERSION
    ruby 3.3.0p0

Rakefile

@@ -80,6 +80,14 @@ namespace :blog do
     user_repo = Admin::Container["repos.user_repo"]
     user_repo.create(id: SecureRandom.uuid, email: args[:email])
   end
+
+  task clean_webmentions: ["blog:load_environment"] do
+    require "hanami/prepare"
+    require "que"
+
+    Que.connection = Adamantium::Container["persistence.db"]
+    Adamantium::Jobs::ArchiveDeletedWebmentions.enqueue
+  end
 end
 
 namespace :tailwind do

config/schedule.rb

@@ -0,0 +1,22 @@
+# Use this file to easily define all of your cron jobs.
+#
+# It's helpful, but not entirely necessary to understand cron before proceeding.
+# http://en.wikipedia.org/wiki/Cron
+
+# Example:
+#
+# set :output, "/path/to/my/cron_log.log"
+#
+# every 2.hours do
+#   command "/usr/bin/some_great_command"
+#   runner "MyModel.some_method"
+#   rake "some:great:rake:task"
+# end
+#
+every 1.days do
+  rake "blog:clean_webmentions"
+  rake "blog:load_from_letterboxd"
+  rake "blog:scrobble_podcasts"
+end
+
+# Learn more: http://github.com/javan/whenever

db/migrate/20240210231532_add_retrieval_attempt_to_webmentions.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+ROM::SQL.migration do
+  change do
+    alter_table :webmentions do
+      add_column :retrieval_attempts, :integer, default: 0
+    end
+  end
+end

db/migrate/20240210232136_add_last_checked_at_to_webmentions.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+ROM::SQL.migration do
+  change do
+    alter_table :webmentions do
+      add_column :last_checked_at, :timestamp
+    end
+  end
+end

lib/adamantium/jobs/archive_deleted_webmentions.rb

@@ -0,0 +1,29 @@
+require "httparty"
+require "que"
+
+module Adamantium
+  module Jobs
+    class ArchiveDeletedWebmentions < Que::Job
+      def run
+        webmention_repo = Admin::Container["repos.webmention_repo"]
+
+        webmentions = webmention_repo.list_all
+
+        webmentions.each do |webmention|
+          code = HTTParty.get(webmention.source_url, follow_redirects: false).code
+
+          # try and fetch the source at least 5 times, just in case a self-hosted server is down for a bit.
+          if code >= 400
+            if webmention.retrieval_attempts >= 5
+              webmention_repo.update(webmention.id, published_at: nil)
+            else
+              webmention_repo.update(webmention.id, retrieval_attempts: (webmention.retrieval_attempts + 1))
+            end
+          end
+
+          webmention_repo.update(webmention.id, last_checked_at: Time.now)
+        end
+      end
+    end
+  end
+end