Permalink
Browse files

Setup Delayed::Job and create job. Resolves #23.

- Sets up Delayed::Job
- Create a job to populated wasapi_files table from Archive-It WASAPI
  endpoint
- Modify schema to handle large integers for `size` field
- tweak RuboCop config to deal with new code. We should come back around
  and clean this up
  • Loading branch information...
ruebot committed Jan 9, 2018
1 parent e3e9af0 commit f5b39f95371575e93b102e09c71e1f97c751dab0
@@ -27,3 +27,15 @@ Style/PercentLiteralDelimiters:
Style/ClassAndModuleChildren:
Exclude:
- test/test_helper.rb

Metrics/AbcSize:
Exclude:
- app/jobs/wasapi_files_populate_job.rb

Metrics/MethodLength:
Exclude:
- app/jobs/wasapi_files_populate_job.rb

Style/GuardClause:
Exclude:
- app/jobs/wasapi_files_populate_job.rb
@@ -62,6 +62,8 @@ gem 'attr_encrypted'
gem 'bootstrap-sass'
gem 'bootstrap_form'
gem 'codecov', require: false, group: :test
gem 'delayed_job_active_record'
gem 'font-awesome-rails'
gem 'http'
gem 'omniauth-github'
gem 'omniauth-twitter'
@@ -76,7 +76,14 @@ GEM
coffee-script-source (1.12.2)
concurrent-ruby (1.0.5)
crass (1.0.2)
delayed_job (4.1.3)
activesupport (>= 3.0, < 5.2)
delayed_job_active_record (4.1.2)
activerecord (>= 3.0, < 5.2)
delayed_job (>= 3.0, < 5)
docile (1.1.5)
domain_name (0.5.20170404)
unf (>= 0.0.5, < 1.0.0)
dotenv (2.2.1)
dotenv-rails (2.2.1)
dotenv (= 2.2.1)
@@ -92,6 +99,15 @@ GEM
globalid (0.4.1)
activesupport (>= 4.2.0)
hashie (3.5.6)
http (3.0.0)
addressable (~> 2.3)
http-cookie (~> 1.0)
http-form_data (>= 2.0.0.pre.pre2, < 3)
http_parser.rb (~> 0.6.0)
http-cookie (1.0.3)
domain_name (~> 0.5)
http-form_data (2.0.0)
http_parser.rb (0.6.0)
i18n (0.9.0)
concurrent-ruby (~> 1.0)
jbuilder (2.7.0)
@@ -233,6 +249,9 @@ GEM
thread_safe (~> 0.1)
uglifier (3.2.0)
execjs (>= 0.3.0, < 3)
unf (0.1.4)
unf_ext
unf_ext (0.0.7.4)
unicode-display_width (1.3.0)
url (0.3.2)
web-console (3.5.1)
@@ -258,8 +277,10 @@ DEPENDENCIES
capybara (~> 2.13)
codecov
coffee-rails (~> 4.2)
delayed_job_active_record
dotenv-rails
font-awesome-rails
http
jbuilder (~> 2.5)
listen (>= 3.0.5, < 3.2)
omniauth-github
@@ -18,10 +18,9 @@ def update
respond_to do |format|
if @user.update(user_params)
format.html { redirect_to @user, notice: 'Update successful.' }
format.json { render :show, status: :ok, location: @user }
WasapiFilesPopulateJob.perform_later(@user)
else
format.html { render :edit }
format.json { render json: @user.errors, status: :unprocessable_entity }
end
end
end
@@ -0,0 +1,5 @@
# frozen_string_literal: true

# WASAPI Files controller methods.
class WasapiFilesController < ApplicationController
end
@@ -0,0 +1,57 @@
# frozen_string_literal: true

# Methods for populating Wasapi Files.
class WasapiFilesPopulateJob < ApplicationJob
queue_as :default

# Constants
WASAPI_BASE_URL = 'https://partner.archive-it.org/wasapi/v1/webdata'

def perform(user)
wasapi_request = HTTP.basic_auth(user: user.wasapi_username,
pass: user.wasapi_password)
.get(WASAPI_BASE_URL)
wasapi_results = JSON.parse(wasapi_request)
wasapi_files = wasapi_results['files']
wasapi_files.each do |file|
WasapiFile.create!(filetype: file['filetype'],
size: file['size'],
filename: file['filename'],
crawl_time: file['crawl-time'],
crawl_start: file['crawl-start'],
crawl: file['crawl'],
account: file['account'],
collection_id: file['collection'],
location_archive_it: file['locations'][0],
location_internet_archive: file['locations'][1],
checksum_md5: file['checksums']['md5'],
checksum_sha1: file['checksums']['sha1'])
end
paginate = wasapi_results['next']
if paginate.present?
loop do
wasapi_paged_request = HTTP.basic_auth(user: user.wasapi_username,
pass: user.wasapi_password)
.get(paginate)
wasapi_paged_results = JSON.parse(wasapi_paged_request)
wasapi_paged_files = wasapi_paged_results['files']
paginate = wasapi_paged_results['next']
wasapi_paged_files.each do |file|
WasapiFile.create!(filetype: file['filetype'],
size: file['size'],
filename: file['filename'],
crawl_time: file['crawl-time'],
crawl_start: file['crawl-start'],
crawl: file['crawl'],
account: file['account'],
collection_id: file['collection'],
location_archive_it: file['locations'][0],
location_internet_archive: file['locations'][1],
checksum_md5: file['checksums']['md5'],
checksum_sha1: file['checksums']['sha1'])
end
break if paginate.blank?
end
end
end
end
@@ -1,4 +1,5 @@
# frozen_string_literal: true

# WASAPI File methods.
class WasapiFile < ApplicationRecord
end
@@ -0,0 +1,5 @@
#!/usr/bin/env ruby

require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment'))
require 'delayed/command'
Delayed::Command.new(ARGV).daemonize
@@ -13,6 +13,7 @@ module Auk
class Application < Rails::Application
# Initialize configuration defaults for originally generated Rails version.
config.load_defaults 5.1
config.active_job.queue_adapter = :delayed_job

# Settings in config/environments/* take precedence over those specified
# here.
@@ -4,13 +4,13 @@ def change
t.string :checksum_md5
t.string :checksum_sha1
t.string :filetype
t.integer :size
t.integer :size, :limit => 8
t.string :filename
t.string :crawl_time
t.string :crawl_start
t.integer :crawl
t.integer :account
t.integer :collection
t.integer :collection_id
t.string :location_archive_it
t.string :location_internet_archive

@@ -0,0 +1,22 @@
class CreateDelayedJobs < ActiveRecord::Migration[5.1]
def self.up
create_table :delayed_jobs, force: true do |table|
table.integer :priority, default: 0, null: false # Allows some jobs to jump to the front of the queue
table.integer :attempts, default: 0, null: false # Provides for retries, but still fail eventually.
table.text :handler, null: false # YAML-encoded string of the object that will do work
table.text :last_error # reason for last failure (See Note below)
table.datetime :run_at # When to run. Could be Time.zone.now for immediately, or sometime in the future.
table.datetime :locked_at # Set when a client is working on this object
table.datetime :failed_at # Set when all retries have failed (actually, by default, the record is deleted instead)
table.string :locked_by # Who is working on this object (if locked)
table.string :queue # The name of the queue this job is in
table.timestamps null: true
end

add_index :delayed_jobs, [:priority, :run_at], name: "delayed_jobs_priority"
end

def self.down
drop_table :delayed_jobs
end
end
@@ -10,7 +10,22 @@
#
# It's strongly recommended that you check this file into your version control system.

ActiveRecord::Schema.define(version: 20171128135615) do
ActiveRecord::Schema.define(version: 20180108203303) do

create_table "delayed_jobs", force: :cascade do |t|
t.integer "priority", default: 0, null: false
t.integer "attempts", default: 0, null: false
t.text "handler", null: false
t.text "last_error"
t.datetime "run_at"
t.datetime "locked_at"
t.datetime "failed_at"
t.string "locked_by"
t.string "queue"
t.datetime "created_at"
t.datetime "updated_at"
t.index ["priority", "run_at"], name: "delayed_jobs_priority"
end

create_table "users", force: :cascade do |t|
t.string "provider"
@@ -32,13 +47,13 @@
t.string "checksum_md5"
t.string "checksum_sha1"
t.string "filetype"
t.integer "size"
t.integer "size", limit: 8
t.string "filename"
t.string "crawl_time"
t.string "crawl_start"
t.integer "crawl"
t.integer "account"
t.integer "collection"
t.integer "collection_id"
t.string "location_archive_it"
t.string "location_internet_archive"
t.datetime "created_at", null: false
@@ -4,26 +4,26 @@ one:
checksum_md5: MyString
checksum_sha1: MyString
filetype: MyString
size: 1
size: 2388654180
filename: MyString
crawl_time: MyString
crawl_start: MyString
crawl: 1
account: 1
collection: 1
collection_id: 1
location_archive_it: MyString
location_internet_archive: MyString

two:
checksum_md5: MyString
checksum_sha1: MyString
filetype: MyString
size: 1
size: 2388654180
filename: MyString
crawl_time: MyString
crawl_start: MyString
crawl: 1
account: 1
collection: 1
collection_id: 1
location_archive_it: MyString
location_internet_archive: MyString
@@ -0,0 +1,9 @@
# frozen_string_literal: true

require 'test_helper'

class WasapiFilesPopulateJobTest < ActiveJob::TestCase
# test "the truth" do
# assert true
# end
end

0 comments on commit f5b39f9

Please sign in to comment.