localhots
/
oldhaven
1
0
Fork 0
Code Releases Activity

Import refactoring

This commit is contained in:
Gregory Eremin 2011-09-14 20:03:21 +04:00
parent 8f56a84a32
commit 67b6899f9f
3 changed files with 62 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Gemfile.lock
View File

@ -70,7 +70,7 @@ GEM
treetop (~> 1.4.8) treetop (~> 1.4.8)
mime-types (1.16) mime-types (1.16)
multi_json (1.0.3) multi_json (1.0.3)
musicbrainz (0.4.1) musicbrainz (0.4.3)
nokogiri nokogiri
nokogiri (1.5.0) nokogiri (1.5.0)
pg (0.11.0) pg (0.11.0)
@ -131,7 +131,7 @@ DEPENDENCIES
haml haml
json json
lastfm! lastfm!
musicbrainz (~> 0.4.1) musicbrainz (~> 0.4.3)
pg pg
rails (= 3.1.0) rails (= 3.1.0)
sass sass

241
app/controllers/import_controller.rb
View File

@ -10,205 +10,71 @@ class ImportController < ApplicationController
# Initializing gems # Initializing gems
lastfm = Lastfm.new(@@lastfm_api_key, @@lastfm_secret) lastfm = Lastfm.new(@@lastfm_api_key, @@lastfm_secret)
artist = Artist.find_or_create_by_name(name) artist = Artist.find_by_name(name)
begin begin
lastfm_artist = lastfm.artist.get_info(name) lastfm_artist = lastfm.artist.get_info(name)
brainz_artist = MusicBrainz::Artist.find_by_name(name) brainz_artist = MusicBrainz::Artist.find_by_name(name)
rescue => e rescue => e
lastfm_artist = {
'bio' => { 'summary' => '' },
'image' => [ nil, nil, nil, { 'content' => '' } ]
}
ap e.message ap e.message
ap e.backtrace ap e.backtrace
end end
# Save artist
artist.desc = lastfm_artist['bio']['summary']
artist.pic_url = lastfm_artist['image'][3]['content']
artist.artist_type = brainz_artist.type
artist.mbid = brainz_artist.id
artist.save! unless dry_run
brainz_artist.release_groups.each do |release_group|
# Saving album
begin
album_lastfm = lastfm.album.get_info(lastfm_artist['name'], release_group.title)
album_image = album_lastfm['image'][3]['content']
rescue
album_image = ''
end
album = Album.new
album.name = release_group.title
album.year = release_group.first_release_date.year
album.artist_id = artist.id
album.mbid = release_group.id
album.album_type = release_group.type
album.pic_url = album_image
album.has_pic = (album_image != '' and not album_image.nil?)
dry_run ? ap(album) : album.save
# Preparing releases
release_group.releases.each do |release|
end
end
begin begin
# Get albums from MB # Save artist
release_groups_mb = brainz.release_group(nil, :artist => artist_mb_data[:mbid], :limit => 500) artist.desc = lastfm_artist['bio']['summary']
artist.pic_url = lastfm_artist['image'][3]['content']
unless release_groups_mb['release_group_list']['release_group'].nil? artist.artist_type = brainz_artist.type
artist.mbid = brainz_artist.id
if release_groups_mb['release_group_list']['release_group'].is_a? Hash dry_run ? ap(artist) : artist.save
release_groups_mb['release_group_list']['release_group'] = [release_groups_mb['release_group_list']['release_group']]
brainz_artist.release_groups.each do |brainz_release_group|
# Saving album
begin
album_lastfm = lastfm.album.get_info(lastfm_artist['name'], brainz_release_group.title)
album_image = album_lastfm['image'][3]['content']
rescue
album_image = ''
end end
album = Album.new
release_groups_mb['release_group_list']['release_group'].each do |mb_album| album.name = brainz_release_group.title
album.year = brainz_release_group.first_release_date.year
album_name = mb_album['title'] + (mb_album['disambiguation'].nil? ? '' : ' ('+ mb_album['disambiguation'] +')') album.artist_id = artist.id
album.mbid = brainz_release_group.id
# Checking if there is already an album with the same name album.album_type = brainz_release_group.type
albums_db = Album.where(:artist_id => artist.id, :name => album_name, :album_type => mb_album['type']) album.pic_url = album_image
album.has_pic = (album_image != '' and not album_image.nil?)
# Get album releases from MB dry_run ? ap(album) : album.save
releases_mb = brainz.release(nil, {'release-group' => mb_album['id']}) # Tracks from the first release
tracks_hashed = []
unless releases_mb['release_list']['release'].nil? or not albums_db.empty? brainz_release_group.releases.each_with_index do |brainz_release, i|
# Processing tracks
releases_mb = releases_mb['release_list']['release'] brainz_release.tracks.each do |brainz_track|
track_title = brainz_track.title.gsub(/\s\/\s\[.*?\]/, '')
if releases_mb.is_a? Hash if tracks_hashed.include? track_title.downcase.scan(/[a-z0-9]*/).join
releases_mb = [releases_mb] next
end end
tracks_hashed << track_title.downcase.scan(/[a-z0-9]*/).join
releases_mb.delete_if { |a| a['date'].nil? } track = Track.new
track.name = track_title
unless releases_mb.empty? track.album_id = album.id
track.position = brainz_track.position
releases_mb.each do |release| track.length = brainz_track.length
if release['date'].length == 4 track.country = brainz_release.country
release['date'] += '-12-28' track.bonus = (i == 0 ? 0 : 1)
elsif release['date'].length == 7 track.live = track_title.downcase.include? 'live'
release['date'] += '-28' track.acoustic = track_title.downcase.include? 'acoustic'
end track.mbid = brainz_track.recording_id
end dry_run ? ap(track) : track.save
end
# Sorting releases by date end
releases_mb.sort! { |a, b| a['date'].to_date <=> b['date'].to_date } end
main_release = releases_mb.shift
# Get tracks from the first release and then exclude release
tracks_mb = []
tracks_mb_r = brainz.release(main_release['id'], {'inc' => 'recordings'})
unless tracks_mb_r['release']['medium_list']['medium'].nil?
tracks_mb_r = tracks_mb_r['release']['medium_list']['medium']
if tracks_mb_r.is_a? Hash
if tracks_mb_r['track_list']['track'].is_a? Hash
tracks_mb_r['track_list']['track'] = [tracks_mb_r['track_list']['track']]
end
tracks_mb = tracks_mb_r['track_list']['track']
elsif tracks_mb_r.is_a? Array
tracks_mb = tracks_mb_r.shift['track_list']['track']
tracks_mb_r.each do |cd|
if cd['track_list']['track'].is_a? Hash
cd['track_list']['track'] = [cd['track_list']['track']]
end
cd['track_list']['track'].each { |mb_track| mb_track['bonus'] = 1 }
tracks_mb |= cd['track_list']['track']
end
end
unless tracks_mb.empty?
# Saving album
begin
album_lastfm = lastfm.album.get_info(artist_lastfm['name'], album_name)
album_image = album_lastfm['image'][3]['content']
rescue
album_image = ''
end
album = Album.new
album.name = album_name
album.year = main_release['date'].to_date.year
album.artist_id = artist.id
album.mbid = mb_album['id']
album.album_type = mb_album['type']
album.pic_url = album_image
album.has_pic = (album_image != '' and not album_image.nil?)
album.save unless dry_run
ap album if dry_run
# Creating hashed array for matching
mb_main_tracks_hashed = []
tracks_mb.each do |mb_track|
unless mb_main_tracks_hashed.include? mb_track['recording']['title'].downcase.scan(/[a-z0-9]*/).join
mb_main_tracks_hashed << mb_track['recording']['title'].downcase.scan(/[a-z0-9]*/).join
mb_track['country'] = main_release['country'] unless main_release['country'].nil?
mb_track['bonus'] = 0 if mb_track['bonus'].nil?
end
end
# Searching for tracks in other releases
releases_mb.each do |mb_release|
other_tracks_mb = []
other_tracks_mb_r = brainz.release(mb_release['id'], {'inc' => 'recordings'})['release']['medium_list']['medium']
if other_tracks_mb_r.is_a? Hash
if other_tracks_mb_r['track_list']['track'].is_a? Hash
other_tracks_mb_r['track_list']['track'] = [other_tracks_mb_r['track_list']['track']]
end
other_tracks_mb = other_tracks_mb_r['track_list']['track']
elsif other_tracks_mb_r.is_a? Array
other_tracks_mb_r.each do |cd|
if cd['track_list']['track'].is_a? Hash
cd['track_list']['track'] = [cd['track_list']['track']]
end
other_tracks_mb |= cd['track_list']['track']
end
end
other_tracks_mb.each do |mb_track|
unless mb_main_tracks_hashed.include? mb_track['recording']['title'].downcase.scan(/[a-z0-9]*/).join
mb_track['country'] = mb_release['country'] unless mb_release['country'].nil?
mb_track['bonus'] = 1
tracks_mb << mb_track
end
end
end
# Saving tracks
tracks_mb.each do |mb_track|
unless ['[silence]', '[untitled]'].include? mb_track['recording']['title']
track = Track.new
track.name = mb_track['recording']['title'].gsub(/\s\/\s\[.*?\]/, '')
track.album_id = album.id
track.position = mb_track['position']
track.length = mb_track['length'] unless mb_track['length'].nil?
track.length = mb_track['recording']['length'] unless mb_track['recording']['length'].nil?
track.country = mb_track['country']
track.bonus = mb_track['bonus']
track.live = mb_track['recording']['title'].downcase.include? 'live'
track.acoustic = mb_track['recording']['title'].downcase.include? 'acoustic'
track.mbid = mb_track['recording']['id']
track.save unless dry_run
end
end
end # unless tracks_mb_r['release']['medium_list']['medium'].nil?
end # releases_mb.empty?
end # unless mb_main_tracks.empty?
end # unless releases_mb.nil?
end # release_groups_mb.each do |mb_album|
end # unless release_groups_mb['release_group_list']['release_group'].nil?
artist.status = 1 artist.status = 1
rescue => e rescue => e
ap e.message ap e.message
ap e.backtrace ap e.backtrace
@ -216,8 +82,5 @@ class ImportController < ApplicationController
end end
artist.save unless dry_run artist.save unless dry_run
ap artist if dry_run end
end # def self.importArtist name
end end

15
lib/tasks/db_import.rake
View File

@ -1,12 +1,13 @@
require 'uri'
require 'open-uri'
namespace :db do namespace :db do
desc 'Imports test data from MusicBrainz database' desc 'Imports test data from MusicBrainz database'
task :import => :environment do task :import, [:name, :dry_run] => :environment do |t, argv|
if argv[:dry_run].nil? or argv[:dry_run] == "true"
ati = ['Vampire Weekend'] argv[:dry_run] = true
ati.each do |name| else
ImportController.importArtist(name, true) argv[:dry_run] = false
end
unless argv[:name].nil?
ImportController.importArtist(argv[:name], argv[:dry_run])
end end
end end
end end