Setup Delayed::Job and create job. Resolves #23.

- Sets up Delayed::Job - Create a job to populated wasapi_files table from Archive-It WASAPI endpoint - Modify schema to handle large integers for `size` field - tweak RuboCop config to deal with new code. We should come back around and clean this up
archivesunleashed · Jan 9, 2018 · f5b39f95371575e93b102e09c71e1f97c751dab0 · f5b39f9
1 parent e3e9af0
commit f5b39f95371575e93b102e09c71e1f97c751dab0
diff --git a/.rubocop.yml b/.rubocop.yml
@@ -27,3 +27,15 @@ Style/PercentLiteralDelimiters:
 Style/ClassAndModuleChildren:
  Exclude:
    - test/test_helper.rb
+  
+
+  
+Metrics/AbcSize:
+  
+  Exclude:
+  
+    - app/jobs/wasapi_files_populate_job.rb
+  
+
+  
+Metrics/MethodLength:
+  
+  Exclude:
+  
+    - app/jobs/wasapi_files_populate_job.rb
+  
+
+  
+Style/GuardClause:
+  
+  Exclude:
+  
+    - app/jobs/wasapi_files_populate_job.rb
diff --git a/Gemfile b/Gemfile
@@ -62,6 +62,8 @@ gem 'attr_encrypted'
 gem 'bootstrap-sass'
 gem 'bootstrap_form'
 gem 'codecov', require: false, group: :test
+  
+gem 'delayed_job_active_record'
 gem 'font-awesome-rails'
+  
+gem 'http'
 gem 'omniauth-github'
 gem 'omniauth-twitter'
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -76,7 +76,14 @@ GEM
    coffee-script-source (1.12.2)
    concurrent-ruby (1.0.5)
    crass (1.0.2)
+  
+    delayed_job (4.1.3)
+  
+      activesupport (>= 3.0, < 5.2)
+  
+    delayed_job_active_record (4.1.2)
+  
+      activerecord (>= 3.0, < 5.2)
+  
+      delayed_job (>= 3.0, < 5)
    docile (1.1.5)
+  
+    domain_name (0.5.20170404)
+  
+      unf (>= 0.0.5, < 1.0.0)
    dotenv (2.2.1)
    dotenv-rails (2.2.1)
      dotenv (= 2.2.1)
@@ -92,6 +99,15 @@ GEM
    globalid (0.4.1)
      activesupport (>= 4.2.0)
    hashie (3.5.6)
+  
+    http (3.0.0)
+  
+      addressable (~> 2.3)
+  
+      http-cookie (~> 1.0)
+  
+      http-form_data (>= 2.0.0.pre.pre2, < 3)
+  
+      http_parser.rb (~> 0.6.0)
+  
+    http-cookie (1.0.3)
+  
+      domain_name (~> 0.5)
+  
+    http-form_data (2.0.0)
+  
+    http_parser.rb (0.6.0)
    i18n (0.9.0)
      concurrent-ruby (~> 1.0)
    jbuilder (2.7.0)
@@ -233,6 +249,9 @@ GEM
      thread_safe (~> 0.1)
    uglifier (3.2.0)
      execjs (>= 0.3.0, < 3)
+  
+    unf (0.1.4)
+  
+      unf_ext
+  
+    unf_ext (0.0.7.4)
    unicode-display_width (1.3.0)
    url (0.3.2)
    web-console (3.5.1)
@@ -258,8 +277,10 @@ DEPENDENCIES
  capybara (~> 2.13)
  codecov
  coffee-rails (~> 4.2)
+  
+  delayed_job_active_record
  dotenv-rails
  font-awesome-rails
+  
+  http
  jbuilder (~> 2.5)
  listen (>= 3.0.5, < 3.2)
  omniauth-github

diff --git a/app/controllers/users_controller.rb b/app/controllers/users_controller.rb
@@ -18,10 +18,9 @@ def update
    respond_to do |format|
      if @user.update(user_params)
        format.html { redirect_to @user, notice: 'Update successful.' }
-  
-        format.json { render :show, status: :ok, location: @user }
+  
+        WasapiFilesPopulateJob.perform_later(@user)
      else
        format.html { render :edit }
-  
-        format.json { render json: @user.errors, status: :unprocessable_entity }
      end
    end
  end

diff --git a/app/controllers/wasapi_files_controller.rb b/app/controllers/wasapi_files_controller.rb
@@ -0,0 +1,5 @@
+  
+# frozen_string_literal: true
+  
+
+  
+# WASAPI Files controller methods.
+  
+class WasapiFilesController < ApplicationController
+  
+end
diff --git a/app/jobs/wasapi_files_populate_job.rb b/app/jobs/wasapi_files_populate_job.rb
@@ -0,0 +1,57 @@
+  
+# frozen_string_literal: true
+  
+
+  
+# Methods for populating Wasapi Files.
+  
+class WasapiFilesPopulateJob < ApplicationJob
+  
+  queue_as :default
+  
+
+  
+  # Constants
+  
+  WASAPI_BASE_URL = 'https://partner.archive-it.org/wasapi/v1/webdata'
+  
+
+  
+  def perform(user)
+  
+    wasapi_request = HTTP.basic_auth(user: user.wasapi_username,
+  
+                                     pass: user.wasapi_password)
+  
+                         .get(WASAPI_BASE_URL)
+  
+    wasapi_results = JSON.parse(wasapi_request)
+  
+    wasapi_files = wasapi_results['files']
+  
+    wasapi_files.each do |file|
+  
+      WasapiFile.create!(filetype: file['filetype'],
+  
+                         size: file['size'],
+  
+                         filename: file['filename'],
+  
+                         crawl_time: file['crawl-time'],
+  
+                         crawl_start: file['crawl-start'],
+  
+                         crawl: file['crawl'],
+  
+                         account: file['account'],
+  
+                         collection_id: file['collection'],
+  
+                         location_archive_it: file['locations'][0],
+  
+                         location_internet_archive: file['locations'][1],
+  
+                         checksum_md5: file['checksums']['md5'],
+  
+                         checksum_sha1: file['checksums']['sha1'])
+  
+    end
+  
+    paginate = wasapi_results['next']
+  
+    if paginate.present?
+  
+      loop do
+  
+        wasapi_paged_request = HTTP.basic_auth(user: user.wasapi_username,
+  
+                                               pass: user.wasapi_password)
+  
+                                   .get(paginate)
+  
+        wasapi_paged_results = JSON.parse(wasapi_paged_request)
+  
+        wasapi_paged_files = wasapi_paged_results['files']
+  
+        paginate = wasapi_paged_results['next']
+  
+        wasapi_paged_files.each do |file|
+  
+          WasapiFile.create!(filetype: file['filetype'],
+  
+                             size: file['size'],
+  
+                             filename: file['filename'],
+  
+                             crawl_time: file['crawl-time'],
+  
+                             crawl_start: file['crawl-start'],
+  
+                             crawl: file['crawl'],
+  
+                             account: file['account'],
+  
+                             collection_id: file['collection'],
+  
+                             location_archive_it: file['locations'][0],
+  
+                             location_internet_archive: file['locations'][1],
+  
+                             checksum_md5: file['checksums']['md5'],
+  
+                             checksum_sha1: file['checksums']['sha1'])
+  
+        end
+  
+        break if paginate.blank?
+  
+      end
+  
+    end
+  
+  end
+  
+end
diff --git a/app/models/wasapi_file.rb b/app/models/wasapi_file.rb
@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+  
+# WASAPI File methods.
 class WasapiFile < ApplicationRecord
 end
diff --git a/bin/delayed_job b/bin/delayed_job
@@ -0,0 +1,5 @@
+  
+#!/usr/bin/env ruby
+  
+
+  
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment'))
+  
+require 'delayed/command'
+  
+Delayed::Command.new(ARGV).daemonize
diff --git a/config/application.rb b/config/application.rb
@@ -13,6 +13,7 @@ module Auk
  class Application < Rails::Application
    # Initialize configuration defaults for originally generated Rails version.
    config.load_defaults 5.1
+  
+    config.active_job.queue_adapter = :delayed_job
    # Settings in config/environments/* take precedence over those specified
    # here.

diff --git a/db/migrate/20171128135615_create_wasapi_files.rb b/db/migrate/20171128135615_create_wasapi_files.rb
@@ -4,13 +4,13 @@ def change
      t.string :checksum_md5
      t.string :checksum_sha1
      t.string :filetype
-  
-      t.integer :size
+  
+      t.integer :size, :limit => 8
      t.string :filename
      t.string :crawl_time
      t.string :crawl_start
      t.integer :crawl
      t.integer :account
-  
-      t.integer :collection
+  
+      t.integer :collection_id
      t.string :location_archive_it
      t.string :location_internet_archive

diff --git a/db/migrate/20180108203303_create_delayed_jobs.rb b/db/migrate/20180108203303_create_delayed_jobs.rb
@@ -0,0 +1,22 @@
+  
+class CreateDelayedJobs < ActiveRecord::Migration[5.1]
+  
+  def self.up
+  
+    create_table :delayed_jobs, force: true do |table|
+  
+      table.integer :priority, default: 0, null: false # Allows some jobs to jump to the front of the queue
+  
+      table.integer :attempts, default: 0, null: false # Provides for retries, but still fail eventually.
+  
+      table.text :handler,                 null: false # YAML-encoded string of the object that will do work
+  
+      table.text :last_error                           # reason for last failure (See Note below)
+  
+      table.datetime :run_at                           # When to run. Could be Time.zone.now for immediately, or sometime in the future.
+  
+      table.datetime :locked_at                        # Set when a client is working on this object
+  
+      table.datetime :failed_at                        # Set when all retries have failed (actually, by default, the record is deleted instead)
+  
+      table.string :locked_by                          # Who is working on this object (if locked)
+  
+      table.string :queue                              # The name of the queue this job is in
+  
+      table.timestamps null: true
+  
+    end
+  
+
+  
+    add_index :delayed_jobs, [:priority, :run_at], name: "delayed_jobs_priority"
+  
+  end
+  
+
+  
+  def self.down
+  
+    drop_table :delayed_jobs
+  
+  end
+  
+end
diff --git a/db/schema.rb b/db/schema.rb
@@ -10,7 +10,22 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-  
-ActiveRecord::Schema.define(version: 20171128135615) do
+  
+ActiveRecord::Schema.define(version: 20180108203303) do
+  
+
+  
+  create_table "delayed_jobs", force: :cascade do |t|
+  
+    t.integer "priority", default: 0, null: false
+  
+    t.integer "attempts", default: 0, null: false
+  
+    t.text "handler", null: false
+  
+    t.text "last_error"
+  
+    t.datetime "run_at"
+  
+    t.datetime "locked_at"
+  
+    t.datetime "failed_at"
+  
+    t.string "locked_by"
+  
+    t.string "queue"
+  
+    t.datetime "created_at"
+  
+    t.datetime "updated_at"
+  
+    t.index ["priority", "run_at"], name: "delayed_jobs_priority"
+  
+  end
  create_table "users", force: :cascade do |t|
    t.string "provider"
@@ -32,13 +47,13 @@
    t.string "checksum_md5"
    t.string "checksum_sha1"
    t.string "filetype"
-  
-    t.integer "size"
+  
+    t.integer "size", limit: 8
    t.string "filename"
    t.string "crawl_time"
    t.string "crawl_start"
    t.integer "crawl"
    t.integer "account"
-  
-    t.integer "collection"
+  
+    t.integer "collection_id"
    t.string "location_archive_it"
    t.string "location_internet_archive"
    t.datetime "created_at", null: false

diff --git a/test/fixtures/wasapi_files.yml b/test/fixtures/wasapi_files.yml
@@ -4,26 +4,26 @@ one:
  checksum_md5: MyString
  checksum_sha1: MyString
  filetype: MyString
-  
-  size: 1
+  
+  size: 2388654180
  filename: MyString
  crawl_time: MyString
  crawl_start: MyString
  crawl: 1
  account: 1
-  
-  collection: 1
+  
+  collection_id: 1
  location_archive_it: MyString
  location_internet_archive: MyString
 two:
  checksum_md5: MyString
  checksum_sha1: MyString
  filetype: MyString
-  
-  size: 1
+  
+  size: 2388654180
  filename: MyString
  crawl_time: MyString
  crawl_start: MyString
  crawl: 1
  account: 1
-  
-  collection: 1
+  
+  collection_id: 1
  location_archive_it: MyString
  location_internet_archive: MyString
diff --git a/test/jobs/wasapi_files_populate_job_test.rb b/test/jobs/wasapi_files_populate_job_test.rb
@@ -0,0 +1,9 @@
+  
+# frozen_string_literal: true
+  
+
+  
+require 'test_helper'
+  
+
+  
+class WasapiFilesPopulateJobTest < ActiveJob::TestCase
+  
+  # test "the truth" do
+  
+  #   assert true
+  
+  # end
+  
+end