From ebdf051b82336ba1974b587a65ed89d6ccdbb505 Mon Sep 17 00:00:00 2001 From: Daniel Nitsikopoulos Date: Mon, 8 May 2023 21:57:32 +1000 Subject: [PATCH] Add job to clean dead links --- Gemfile | 2 ++ Gemfile.lock | 4 ++++ config/environment.rb | 17 ++++++++++++++ config/routes.rb | 1 + db/migrate/20230508101336_add_que.rb | 15 +++++++++++++ lib/adamantium/jobs/remove_dead_bookmarks.rb | 21 ++++++++++++++++++ slices/admin/actions/bookmarks/clean.rb | 22 +++++++++++++++++++ slices/admin/repos/bookmark_repo.rb | 7 ++++++ .../admin/templates/bookmarks/index.html.slim | 3 +++ 9 files changed, 92 insertions(+) create mode 100644 config/environment.rb create mode 100644 db/migrate/20230508101336_add_que.rb create mode 100644 lib/adamantium/jobs/remove_dead_bookmarks.rb create mode 100644 slices/admin/actions/bookmarks/clean.rb diff --git a/Gemfile b/Gemfile index db7d9b6..15706b3 100644 --- a/Gemfile +++ b/Gemfile @@ -44,6 +44,8 @@ gem "sanitize" gem "time_math2", require: "time_math" gem "lastfm", "~> 1.27" gem "mail" +gem "que" +gem "connection_pool" group :cli, :development do gem "hanami-reloader" diff --git a/Gemfile.lock b/Gemfile.lock index 4c611ff..6de6207 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -67,6 +67,7 @@ GEM sshkit (~> 1.3) coderay (1.1.3) concurrent-ruby (1.2.2) + connection_pool (2.4.0) crass (1.0.6) database_cleaner-core (2.0.1) database_cleaner-sequel (2.0.2) @@ -299,6 +300,7 @@ GEM public_suffix (5.0.1) puma (6.2.2) nio4r (~> 2.0) + que (2.2.1) racc (1.6.2) rack (2.2.7) rack-test (2.1.0) @@ -438,6 +440,7 @@ DEPENDENCIES capistrano-bundler capistrano-rbenv (~> 2.2) capistrano3-puma! + connection_pool database_cleaner-sequel dotenv down @@ -463,6 +466,7 @@ DEPENDENCIES pg pinboard! puma + que rack-test rake redcarpet diff --git a/config/environment.rb b/config/environment.rb new file mode 100644 index 0000000..7bcd9b1 --- /dev/null +++ b/config/environment.rb @@ -0,0 +1,17 @@ +require "uri" +require "pg" +require "connection_pool" +require "que" +require "hanami/boot" + +uri = URI.parse(ENV['DATABASE_URL']) + +Que.connection = ConnectionPool.new(size: 10) do + PG::Connection.open( + host: uri.host, + user: uri.user, + password: uri.password, + port: uri.port || 5432, + dbname: uri.path[1..-1] + ) +end \ No newline at end of file diff --git a/config/routes.rb b/config/routes.rb index eae9be4..0510179 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -72,6 +72,7 @@ module Adamantium get "/bookmarks", to: "bookmarks.index" delete "/bookmarks/:id", to: "bookmarks.delete" + post "/bookmarks/clean", to: "bookmarks.clean" post "/bookmarks/cache/:id", to: "bookmarks.cache" post "/bookmarks/:id/archive", to: "bookmarks.archive" diff --git a/db/migrate/20230508101336_add_que.rb b/db/migrate/20230508101336_add_que.rb new file mode 100644 index 0000000..fd5dd37 --- /dev/null +++ b/db/migrate/20230508101336_add_que.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require "que" + +ROM::SQL.migration do + up do + Que.connection = self + Que.migrate!(version: 7) + end + + down do + Que.connection = self + Que.migrate!(version: 0) + end +end diff --git a/lib/adamantium/jobs/remove_dead_bookmarks.rb b/lib/adamantium/jobs/remove_dead_bookmarks.rb new file mode 100644 index 0000000..0c775e2 --- /dev/null +++ b/lib/adamantium/jobs/remove_dead_bookmarks.rb @@ -0,0 +1,21 @@ +require "httparty" +require "que" + +module Adamantium + module Jobs + class RemoveDeadBookmarks < Que::Job + def run + bookmark_repo = Admin::Container["repos.bookmark_repo"] + + bookmarks = bookmark_repo.list + + bookmarks.each do |bookmark| + code = HTTParty.get(bookmark.url, follow_redirects: false).code + if code >= 400 + bookmark_repo.archive(id: bookmark.id) + end + end + end + end + end +end \ No newline at end of file diff --git a/slices/admin/actions/bookmarks/clean.rb b/slices/admin/actions/bookmarks/clean.rb new file mode 100644 index 0000000..2fd388c --- /dev/null +++ b/slices/admin/actions/bookmarks/clean.rb @@ -0,0 +1,22 @@ +require "que" + +module Admin + module Actions + module Bookmarks + class Clean < Action + + def handle(req, res) + Que.connection = Adamantium::Container["persistence.db"] + + res.status = 200 + if Que.job_stats.any? { |job| job[:job_class] == Adamantium::Jobs::RemoveDeadBookmarks.name } + res.body = "Job already queued" + else + Adamantium::Jobs::RemoveDeadBookmarks.enqueue + res.body = "#{Que.job_stats.count} job queued" + end + end + end + end + end +end \ No newline at end of file diff --git a/slices/admin/repos/bookmark_repo.rb b/slices/admin/repos/bookmark_repo.rb index 4c2b6ed..dd71b40 100644 --- a/slices/admin/repos/bookmark_repo.rb +++ b/slices/admin/repos/bookmark_repo.rb @@ -7,6 +7,13 @@ module Admin .to_a end + def list_published + posts + .published + .where(post_type: "bookmark") + .to_a + end + def fetch(id:) posts.where(id: id).one end diff --git a/slices/admin/templates/bookmarks/index.html.slim b/slices/admin/templates/bookmarks/index.html.slim index ebb26b8..9970977 100644 --- a/slices/admin/templates/bookmarks/index.html.slim +++ b/slices/admin/templates/bookmarks/index.html.slim @@ -2,6 +2,9 @@ div class="mb-12 prose dark:prose-invert max-w-prose mx-auto text-gray-800 dark: h1 Admin // Bookmarks div class="max-w-prose mx-auto" + + button class="rounded bg-blue-100 hover:bg-blue-200 text-blue-600 px-2 hover:cursor-pointer" hx-post="/admin/bookmarks/clean" Check for dead links + table class="prose dark:prose-invert table-auto prose-a:text-blue-600 prose-a:no-underline" thead th Details