localhots/oldhaven
1
0
Fork 0
Code Releases Activity

MB + Last.fm + Discogs = Import!

Browse Source
This commit is contained in:
magnolia-fan 2011-09-26 09:10:23 +04:00
parent 0e04dba130
commit 7582bd18a5
8 changed files with 246 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
app/controllers/artist_controller.rb
View File

@ -88,8 +88,7 @@ class ArtistController < ApplicationController
end
@artist[:albums] = []
artist.albums.with_format(:album).each do |album|
ap album
artist.albums.each do |album|
if true # media_types.include? album.album_type.downcase.to_sym and album.status == 1
tmp_album = {id: album.id, name: album.name, year: album.year, pic: album.pic_url}
album_tracks = []
@ -103,7 +102,7 @@ class ArtistController < ApplicationController
(track.bonus == 0 ? album_tracks : bonus_tracks) << tmp_track
end
tmp_album[:tracks] = {album: album_tracks, bonus: bonus_tracks}
@artist[:albums] << tmp_album
@artist[:albums] << tmp_album if (album_tracks+bonus_tracks).length > 0
end
end

164
app/controllers/import_controller.rb
View File

@ -1,5 +1,7 @@
require 'musicbrainz'
require 'discogs'
require 'nokogiri'
require 'zlib'
class ImportController < ApplicationController
def self.importArtist name, dry_run = false
@ -46,70 +48,106 @@ class ImportController < ApplicationController
:url => url
).save
end
brainz_artist.release_groups.each do |brainz_release_group|
# Saving album
begin
album_lastfm = LastFM::Album.get_info( :artist => lastfm_artist['artist']['name'], :album => brainz_release_group.title )
album_image = album_lastfm['album']['image'][3]['#text']
rescue
album_image = ''
end
album = Album.new
album.name = brainz_release_group.title
album.year = brainz_release_group.first_release_date.year
album.artist_id = artist.id
album.mbid = brainz_release_group.id
album.album_type = brainz_release_group.type
album.pic_url = album_image
album.has_pic = (album_image != '' and not album_image.nil?)
album.status = 0
album.save
# Tracks from the first release
tracks_hashed = []
brainz_release_group.releases.each_with_index do |brainz_release, i|
local_brainz_release = LocalBrainz::Release.new(
:mbid => brainz_release.id,
:title => brainz_release.title,
:status => brainz_release.status,
:date => brainz_release.date,
:country => brainz_release.country,
:format => brainz_release.format,
:album_id => album.id
)
local_brainz_release.save
# Processing tracks
brainz_release.tracks.each do |brainz_track|
local_brainz_track = LocalBrainz::Track.new(
:position => brainz_track.position,
:recording_id => brainz_track.recording_id,
:title => brainz_track.title,
:length => brainz_track.length,
:release_id => local_brainz_release.id
)
local_brainz_track.save
track_title = brainz_track.title.gsub(/\s\/\s\[.*?\]/, '')
if tracks_hashed.include? track_title.downcase.scan(/[a-z0-9]*/).join
next
end
tracks_hashed << track_title.downcase.scan(/[a-z0-9]*/).join
track = Track.new
track.name = track_title
track.album_id = album.id
track.position = brainz_track.position
track.length = brainz_track.length
track.country = brainz_release.country
track.bonus = (i == 0 ? 0 : 1)
track.live = track_title.downcase.include? 'live'
track.acoustic = track_title.downcase.include? 'acoustic'
track.mbid = brainz_track.recording_id
track.save
end
end
album.status = 1
album.save unless dry_run
end
unless brainz_artist.urls[:discogs].nil?
Discogs.get_master_albums(brainz_artist.urls[:discogs]).each do |info|
begin
puts " * "+info[:title]+" ("+info[:year]+") -- http://api.discogs.com/"+info[:uri]+")"
stream = open("http://api.discogs.com/"+info[:uri]+"?f=xml",
'User-Agent' => 'Haven Import Bot',
'Accept-Encoding' => 'gzip, deflate'
)
if (stream.content_encoding.empty?)
body = stream.read
else
body = Zlib::GzipReader.new(stream).read
end
# Creating album
album = Album.new(
:name => info[:title],
:artist_id => artist.id,
:year => info[:year],
:status => 1,
:master => true
)
begin
album_lastfm = LastFM::Album.get_info( :artist => lastfm_artist['artist']['name'], :album => info[:title] )
album_image = album_lastfm['album']['image'][3]['#text']
rescue
album_image = ''
end
album.pic_url = album_image
album.has_pic = (album_image != '' and not album_image.nil?)
album.save
Discogs.save_album(artist, album, Nokogiri::HTML(body))
end
sleep 1
end
end
# brainz_artist.release_groups.each do |brainz_release_group|
# # Saving album
# begin
# album_lastfm = LastFM::Album.get_info( :artist => lastfm_artist['artist']['name'], :album => '' )
# album_image = album_lastfm['album']['image'][3]['#text']
# rescue
# album_image = ''
# end
# album = Album.new
# album.name = brainz_release_group.title
# album.year = brainz_release_group.first_release_date.year
# album.artist_id = artist.id
# album.mbid = brainz_release_group.id
# album.album_type = brainz_release_group.type
# album.pic_url = album_image
# album.has_pic = (album_image != '' and not album_image.nil?)
# album.status = 0
# album.save
# # Tracks from the first release
# tracks_hashed = []
# brainz_release_group.releases.each_with_index do |brainz_release, i|
# local_brainz_release = LocalBrainz::Release.new(
# :mbid => brainz_release.id,
# :title => brainz_release.title,
# :status => brainz_release.status,
# :date => brainz_release.date,
# :country => brainz_release.country,
# :format => brainz_release.format,
# :album_id => album.id
# )
# local_brainz_release.save
# # Processing tracks
# brainz_release.tracks.each do |brainz_track|
# local_brainz_track = LocalBrainz::Track.new(
# :position => brainz_track.position,
# :recording_id => brainz_track.recording_id,
# :title => brainz_track.title,
# :length => brainz_track.length,
# :release_id => local_brainz_release.id
# )
# local_brainz_track.save
# track_title = brainz_track.title.gsub(/\s\/\s\[.*?\]/, '')
# if tracks_hashed.include? track_title.downcase.scan(/[a-z0-9]*/).join
# next
# end
# tracks_hashed << track_title.downcase.scan(/[a-z0-9]*/).join
# track = Track.new
# track.name = track_title
# track.album_id = album.id
# track.position = brainz_track.position
# track.length = brainz_track.length
# track.country = brainz_release.country
# track.bonus = (i == 0 ? 0 : 1)
# track.live = track_title.downcase.include? 'live'
# track.acoustic = track_title.downcase.include? 'acoustic'
# track.mbid = brainz_track.recording_id
# track.save
# end
# end
# album.status = 1
# album.save unless dry_run
# end
artist.status = 1
rescue => e
ap e.message

2
app/models/music/album.rb
View File

@ -1,6 +1,6 @@
class Album < ActiveRecord::Base
belongs_to :artist
has_many :tracks, :order => 'bonus ASC, position ASC', :dependent => :destroy
has_many :tracks, :conditions => ['length > 0'], :order => 'bonus ASC, position ASC', :dependent => :destroy
has_many :album_formats
has_many :release_formats, :through => :album_formats

11
db/migrate/20110926031740_prepare_to_mixed_import.rb Normal file
View File

@ -0,0 +1,11 @@
class PrepareToMixedImport < ActiveRecord::Migration
def up
add_column :track_artists, :name, :string
add_column :albums, :master_id, :integer
end
def down
remove_column :track_artists, :name
remove_column :albums, :master_id
end
end

4
db/schema.rb
View File

@ -11,7 +11,7 @@
#
# It's strongly recommended to check this file into your version control system.
ActiveRecord::Schema.define(:version => 20110926002631) do
ActiveRecord::Schema.define(:version => 20110926031740) do
create_table "album_formats", :force => true do |t|
t.integer "album_id"
@ -49,6 +49,7 @@ ActiveRecord::Schema.define(:version => 20110926002631) do
t.integer "has_pic"
t.integer "status"
t.boolean "master"
t.integer "master_id"
end
add_index "albums", ["artist_id"], :name => "index_albums_on_artist_id"
@ -182,6 +183,7 @@ ActiveRecord::Schema.define(:version => 20110926002631) do
t.integer "artist_id"
t.boolean "main"
t.string "join"
t.string "name"
end
add_index "track_artists", ["artist_id"], :name => "index_track_artists_on_artist_id"

204
lib/discogs.rb
View File

@ -4,6 +4,7 @@ class Discogs
self.get_nodes('tmp/data/discogs_artists.xml', 'artist') do |node|
artist = Artist.new(
:name => (node.css('name').first.text),
:original_name => (node.css('name').first.text),
:pic_url => (node.css('images > image[type="primary"]').first.attr('uri') unless node.css('images > image[type="primary"]').empty?),
:status => 1
)
@ -26,96 +27,127 @@ class Discogs
main_artist = Artist.find_or_create_by_name(node.css('artists > artist > name').first.text)
end
# Creating album
album = Album.new(
:name => (node.css('title').first.text unless node.css('title').empty?),
:artist_id => (main_artist.id unless main_artist.nil?),
:year => (node.css('released').first.text.split('-').first.to_i unless node.css('released').empty?),
:status => 1,
:master => (not node.css('master_id').empty?)
)
# Parsing image
unless node.css('images > image[type="primary"]').empty?
album.pic_url = node.css('images > image[type="primary"]').first.attr('uri')
album.has_pic = 1
else
album.has_pic = 0
end
album.save ## ## ##
# Defining formats
formats = node.css('formats > format > descriptions > description').each do |f|
format = ReleaseFormat.find_or_create_by_name(f.text)
if format.hash.nil?
format.hash = f.text.scan(/\w/).join().downcase
format.save
end
AlbumFormat.new(
:album_id => album.id,
:release_format_id => format.id
).save
end
# Defining genres
unless node.css('genres > genre').empty?
node.css('genres > genre').each do |g|
genre = Genre.find_or_create_by_name(g.text)
AlbumGenre.new(
:album_id => album.id,
:genre_id => genre.id
).save
end
end
# Defining styles
unless node.css('styles > style').empty?
node.css('styles > style').each do |s|
style = Style.find_or_create_by_name(s.text)
AlbumStyle.new(
:album_id => album.id,
:style_id => style.id
).save
end
end
# Writing tracklist
unless node.css('tracklist > track').empty?
node.css('tracklist > track').each do |t|
track = Track.new(
:album_id => album.id,
:name => (t.css('title').first.text unless t.css('title').empty?),
:position => (t.css('position').first.text.to_i(36) unless t.css('position').empty?),
:country => (node.css('country').first.text unless node.css('country').empty?),
:length => (self.duration_to_length(t.css('duration').first.text) unless t.css('duration').empty?)
)
track.save
if t.css('artists > artist').empty?
TrackArtist.new(
:track_id => track.id,
:artist_id => (main_artist.id unless main_artist.nil?),
:main => true
).save
else
t.css('artists > artist').each_with_index do |a, i|
t_artist = Artist.find_or_create_by_name(a.css('name').first.text)
TrackArtist.new(
:track_id => track.id,
:artist_id => t_artist.id,
:main => (i == 0),
:join => a.css('join').first.text
).save
end
end
end
end
self.save_album main_artist, node
end
end
def self.save_album main_artist, album, node
unless node.css('master').empty?
album.master_id = node.css('master').first.attr('id')
end
# Parsing image
# unless node.css('images > image[type="primary"]').empty?
# album.pic_url = node.css('images > image[type="primary"]').first.attr('uri')
# album.has_pic = 1
# else
# if node.css('images > image[type="secondary"]').empty?
# album.has_pic = 0
# else
# album.pic_url = node.css('images > image[type="secondary"]').first.attr('uri')
# album.has_pic = 1
# end
# end
album.save ## ## ##
# Defining formats
formats = node.css('formats > format > descriptions > description').each do |f|
format = ReleaseFormat.find_or_create_by_name(f.text)
if format.hash.nil?
format.hash = f.text.scan(/\w/).join().downcase
format.save
end
AlbumFormat.new(
:album_id => album.id,
:release_format_id => format.id
).save
end
# Defining genres
unless node.css('genres > genre').empty?
node.css('genres > genre').each do |g|
genre = Genre.find_or_create_by_name(g.text)
AlbumGenre.new(
:album_id => album.id,
:genre_id => genre.id
).save
end
end
# Defining styles
unless node.css('styles > style').empty?
node.css('styles > style').each do |s|
style = Style.find_or_create_by_name(s.text)
AlbumStyle.new(
:album_id => album.id,
:style_id => style.id
).save
end
end
# Writing tracklist
unless node.css('tracklist > track').empty?
node.css('tracklist > track').each do |t|
track = Track.new(
:album_id => album.id,
:name => (t.css('title').first.text unless t.css('title').empty?),
:position => (t.css('position').first.text.scan(/[a-zA-Z0-9]/).join('').to_i(36) unless t.css('position').empty?),
:country => (node.css('country').first.text unless node.css('country').empty?),
:length => (self.duration_to_length(t.css('duration').first.text) unless t.css('duration').empty?)
)
track.save
if t.css('artists > artist').empty?
TrackArtist.new(
:track_id => track.id,
:artist_id => (main_artist.id unless main_artist.nil?),
:main => true,
:name => main_artist.name
).save
else
t.css('artists > artist').each_with_index do |a, i|
t_artist = Artist.find_or_create_by_name(a.css('name').first.text)
TrackArtist.new(
:track_id => track.id,
:artist_id => t_artist.id,
:main => (i == 0),
:join => a.css('join').first.text,
:name => a.css('name').first.text
).save
end
end
end
end
end
def self.get_master_albums url
x = Nokogiri::HTML(open(url,
'User-Agent' => 'Haven Search Bot',
'Cookie' => 'artist_layout=med; artist_limit=500'
).read)
albums = []
split = 0
x.css('table.discography > tbody > tr').each do |_|
if _.attr('class') == 'credit_header'
split+=1
end
if split == 2
break
end
if _.attr('class').match(/^main.*/) and _.css('td')[3].text.scan(/\d/).join('') != '' and _.css('a.mr_toggler').length == 1
albums << {
:uri => (_.attr('class').match(/master/) ? 'master/' : 'release/')+_.attr('id')[1,100],
:title => _.css('h4 > a').first.text,
:year => _.css('td')[3].text.scan(/\d/).join('')
}
end
end
albums
end
private
def self.get_nodes filename, nodename, &block

2
lib/tasks/lastfm_import.rake
View File

@ -2,6 +2,7 @@ namespace :lastfm do
namespace :import do
desc 'Import top 1000 artists from last.fm'
task :top => :environment do
ActiveRecord::Base.logger = Logger.new('/dev/null')
ImportController.parseLastfmXML('tmp/data/top1000.xml').each do |artist|
puts artist[:name] + (artist[:mbid].empty? ? '' : ' (' + artist[:mbid] + ')')
record = Artist.find_or_create_by_name(artist[:name])
@ -19,6 +20,7 @@ namespace :lastfm do
end
end
task :hyped => :environment do
ActiveRecord::Base.logger = Logger.new('/dev/null')
ImportController.parseLastfmXML('tmp/data/hyped500.xml').each do |artist|
puts artist[:name] + (artist[:mbid].empty? ? '' : ' (' + artist[:mbid] + ')')
record = Artist.find_or_create_by_name(artist[:name])

8
lib/tasks/music_artist_import.rake Normal file
View File

@ -0,0 +1,8 @@
namespace :music do
namespace :artist do
desc 'Imports artist'
task :import => :environment do
ImportController.importArtist('Jet')
end
end
end