Refactored artist import using mbrainz n lastfm gems. Improved stability

2011-06-24 13:36:44 +04:00
parent ecb47b27e7
commit 89776db403
10 changed files with 177 additions and 128 deletions
@@ -1,6 +1,7 @@
 require 'open-uri'

 class ArtistController < ApplicationController
+  @@default_album_types = ['Album', 'Soundtrack']
  def data
    data = {}
    name = params[:name].gsub('%20', ' ').gsub('+', ' ')
@@ -15,6 +16,9 @@ class ArtistController < ApplicationController
      end
      render :json => {status: 'loading', pics: pics}
      return
+    elsif artist and artist.status == 2
+      render :json => {status: 'loading_failed', pics: []}
+      return
    end
    unless artist
      results = ArtistController.musicBrainzExactSearch(name)
@@ -42,7 +46,7 @@ class ArtistController < ApplicationController
    data['albums'] = []
    albums = artist.albums
    albums.each do |album|
-      if album.album_type == 'Album'
+      if @@default_album_types.include? album.album_type
        tmp_album = {name: album.name, year: album.year, pic: album.pic_url}
        album_tracks = []
        bonus_tracks = []
@@ -1,143 +1,159 @@
-class ImportController < ApplicationController
-  @@user_agent = 'BeatHaven.org'
-  @@lastfm_api_key = '04fda005dbf61a50af5abc3e90f111f2'
-  def self.importArtist name
+require 'lastfm'
+require 'musicbrainz'

-    # Get artist info
-    artist_mb_data = ArtistController.musicBrainzExactSearch(name).first
-    begin
-      artist_lastfm_xml = open(
-        'http://ws.audioscrobbler.com/2.0/'+
-        '?method=artist.getinfo&api_key='+ @@lastfm_api_key +
-        '&artist='+ URI.escape(name).gsub(/amp;/, '').gsub(/\&/, '%26').gsub(/\?/, '%3F')
-      ).read
-      artist_desc = artist_lastfm_xml.scan(/<summary><\!\[CDATA\[?(.*)\]\]><\/summary>/m)[0][0]
-      artist_pic = artist_lastfm_xml.scan(/<image\ssize=\"extralarge\">?(.*)<\/image>/)[0][0]
-    rescue
-      artist_desc = ''
-      artist_pic = ''
-    end
+class ImportController < ApplicationController
+  
+  @@user_agent = 'BeatHaven.org'
+  
+  @@lastfm_api_key = '04fda005dbf61a50af5abc3e90f111f2'
+  @@lastfm_secret = '19e70e98b291e9f15d0516925945eb1b'
+  
+  @@brainz_login = 'magnolia_fan'
+  @@brainz_password = '111111'
+  
+  def self.importArtist name
+    
+    # Initializing gems
+    brainz = MusicBrainz::Client.new(@@brainz_login, @@brainz_password, @@user_agent)
+    lastfm = Lastfm.new(@@lastfm_api_key, @@lastfm_secret)
    
-    # Save artist
    artist = Artist.find_by_name(name)
-    #artist.name = artist_mb_data[:name]
-    artist.desc = artist_desc
-    artist.pic_url = artist_pic
-    artist.artist_type = artist_mb_data[:type]
-    artist.mbid = artist_mb_data[:mbid]
-    artist.save
    
-    # Get albums from MB
-    release_groups_mb_xml = open(
-      'http://musicbrainz.org/ws/2/release-group/?artist='+ artist_mb_data[:mbid],
-      'User-Agent' => @@user_agent
-    ).read
-    release_groups_mb_data = release_groups_mb_xml.scan(/<release-group\stype=\"([a-zA-Z]+?)\"\sid=\"([a-f0-9-]+?)\"><title>(.+?)<\/title>/m)
+    begin
    
-    release_groups_mb_data.each do |mb_album|
-      
-      # Get album releases from MB
-      releases_mb_xml = open(
-        'http://musicbrainz.org/ws/2/release/?release-group='+ mb_album[1],
-        'User-Agent' => @@user_agent
-      ).read
-      releases_mb_data = releases_mb_xml.scan(/<release\sid=\"([a-f0-9-]+?)\">.*?<status>([a-zA-Z]+?)<\/status>.*?<date>([0-9-]+?)<\/date>.*?<country>([A-Z]+?)<\/country>/m)
-      releases_mb_data.each do |item|
-        if item[2].length == 4
-          item[2] += '-12-28'
-        elsif item[2].length == 7
-          item[2] += '-28'
-        end
+      # Get artist info
+      artist_mb_data = ArtistController.musicBrainzExactSearch(name).first
+      begin
+        artist_lastfm = lastfm.artist.get_info(name)
+        artist_mb = brainz.artist(artist_lastfm['mbid'])
+      rescue
+        return
      end
+    
+      # Save artist
+      artist.desc = artist_lastfm['bio']['summary']
+      artist.pic_url = artist_lastfm['image'][3]['content']
+      artist.artist_type = artist_mb['artist']['type']
+      artist.mbid = artist_lastfm['mbid']
+    
+      # Get albums from MB
+      release_groups_mb = brainz.release_group(nil, :artist => artist_lastfm['mbid'])['release_group_list']['release_group']
+      release_groups_mb.each do |mb_album|
      
-      unless releases_mb_data.empty?
-        
-        # Sorting releases by date
-        releases_mb_data.sort! { |a, b| a[2].to_date <=> b[2].to_date }
-        main_release = releases_mb_data.shift
-        
-        # Get tracks from the first release and then exclude release
-        main_tracks_mb_xml = open(
-          'http://musicbrainz.org/ws/2/release/'+ main_release[0] +'/?inc=recordings',
-          'User-Agent' => @@user_agent
-        ).read
-        mb_tracks = main_tracks_mb_xml.scan(/<track>.*?<position>(\d+?)<\/position>.*?<recording\sid=\"([a-f0-9-]+?)\"><title>(.*?)<\/title>(<length>(.*?)<\/length>)?/m)
-        mb_tracks.map! do |item|
-          item << main_release[3]
+        # Get album releases from MB
+        releases_mb = brainz.release(nil, {'release-group' => mb_album['id']})['release_list']['release']
+        if releases_mb.is_a? Hash
+          releases_mb = [releases_mb]
        end
-        
-        unless mb_tracks.empty?
-          
-          # Saving album
-          begin
-            album_lastm_xml = open(
-              'http://ws.audioscrobbler.com/2.0/'+
-              '?method=album.getinfo&api_key='+ @@lastfm_api_key +
-              '&artist='+ URI.escape(name).gsub(/amp;/, '').gsub(/\&/, '%26').gsub(/\?/, '%3F')+
-              '&album='+ URI.escape(mb_album[2]).gsub(/amp;/, '').gsub(/\&/, '%26').gsub(/\?/, '%3F')
-            ).read
-            album_image = album_lastm_xml.scan(/<image\ssize=\"extralarge\">?(.*)<\/image>/)[0][0]
-          rescue
-            album_image = ''
+        releases_mb.delete_if { |a| a['date'].nil? }
+        releases_mb.each do |release|
+          if release['date'].length == 4
+            release['date'] += '-12-28'
+          elsif release['date'].length == 7
+            release['date'] += '-28'
          end
-          album = Album.new
-          album.name = mb_album[2]
-          album.year = main_release[2].to_date.year
-          album.artist_id = artist.id
-          album.mbid = mb_album[1]
-          album.album_type = mb_album[0]
-          album.pic_url = album_image
-          album.save
-          
-          # Creating hashed array for matching
-          mb_main_tracks_hashed = []
-          mb_tracks.each do |mb_track|
-            unless mb_main_tracks_hashed.include? mb_track[2].downcase.scan(/[a-z0-9]*/).join
-              mb_main_tracks_hashed << mb_track[2].downcase.scan(/[a-z0-9]*/).join
-              mb_track << main_release[3] # Release country
-              mb_track << 0 # Bonus flag
+        end
+      
+        unless releases_mb.empty? or true
+        
+          # Sorting releases by date
+          releases_mb.sort! { |a, b| a['date'].to_date <=> b['date'].to_date }
+          main_release = releases_mb.shift
+        
+          # Get tracks from the first release and then exclude release
+          tracks_mb = []
+          tracks_mb_r = brainz.release(main_release['id'], {'inc' => 'recordings'})['release']['medium_list']['medium']
+          if tracks_mb_r.is_a? Hash
+            tracks_mb = tracks_mb_r['track_list']['track']
+          elsif tracks_mb_r.is_a? Array
+            tracks_mb = tracks_mb_r.shift['track_list']['track']
+            tracks_mb_r.each do |cd|
+              cd['track_list']['track'].map! { |mb_track| mb_track['bonus'] = 1 }
+              tracks_mb |= cd['track_list']['track']
            end
          end
+        
+          unless tracks_mb.empty?
+
+            # Saving album
+            begin
+              album_lastfm = lastfm.album.get_info(artist_lastfm['name'], mb_album['title'])
+              album_image = album_lastfm['image'][3]['content']
+            rescue
+              album_image = ''
+            end
          
-          # Searching for tracks in other releases
-          releases_mb_data.each do |mb_release|
-            other_tracks_mb_xml = open(
-              'http://musicbrainz.org/ws/2/release/'+ mb_release[0] +'/?inc=recordings',
-              'User-Agent' => @@user_agent
-            ).read
-            mb_other_tracks = other_tracks_mb_xml.scan(/<track>.*?<position>(\d+?)<\/position>.*?<recording\sid=\"([a-f0-9-]+?)\"><title>(.*?)<\/title><length>(.*?)<\/length>/m)
-            mb_other_tracks.each do |item|
-              unless mb_main_tracks_hashed.include? item[2].downcase.scan(/[a-z0-9]*/).join
-                item << mb_release[3] # Release country
-                item << 1 # Bonus flag
-                mb_tracks << item
+            album = Album.new
+            album.name = mb_album['title']
+            album.year = main_release['date'].to_date.year
+            album.artist_id = artist.id
+            album.mbid = mb_album['id']
+            album.album_type = mb_album['type']
+            album.pic_url = album_image
+            album.save
+          
+            # Creating hashed array for matching
+            mb_main_tracks_hashed = []
+            tracks_mb.each do |mb_track|
+              unless mb_main_tracks_hashed.include? mb_track['recording']['title'].downcase.scan(/[a-z0-9]*/).join
+                mb_main_tracks_hashed << mb_track['recording']['title'].downcase.scan(/[a-z0-9]*/).join
+                mb_track['country'] = main_release['country'] unless main_release['country'].nil?
+                mb_track['bonus'] = 0 if mb_track['bonus'].nil?
              end
            end
-          end
          
-          # Saving tracks
-          mb_tracks.each do |mb_track|
-            unless mb_track[2] == '[silence]' or mb_track[2] == '[untitled]'
-              track = Track.new
-              track.name = mb_track[2]
-              track.album_id = album.id
-              track.position = mb_track[0]
-              track.length = mb_track[4]
-              track.country = mb_track[5]
-              track.bonus = mb_track[6]
-              track.live = mb_track[2].downcase.include? 'live'
-              track.acoustic = mb_track[2].downcase.include? 'acoustic'
-              track.save
+            # Searching for tracks in other releases
+            releases_mb.each do |mb_release|
+            
+              other_tracks_mb = []
+              other_tracks_mb_r = brainz.release(mb_release['id'], {'inc' => 'recordings'})['release']['medium_list']['medium']
+              if other_tracks_mb_r.is_a? Hash
+                other_tracks_mb = other_tracks_mb_r['track_list']['track']
+              elsif other_tracks_mb_r.is_a? Array
+                other_tracks_mb_r.each do |cd|
+                  other_tracks_mb |= cd['track_list']['track']
+                end
+              end
+            
+              other_tracks_mb.each do |mb_track|
+                unless mb_main_tracks_hashed.include? mb_track['recording']['title'].downcase.scan(/[a-z0-9]*/).join
+                  mb_track['country'] = mb_release['country'] unless mb_release['country'].nil?
+                  mb_track['bonus'] = 1
+                  tracks_mb << mb_track
+                end
+              end
            end
-          end
          
-        end # unless mb_main_tracks.empty?
+            # Saving tracks
+            tracks_mb.each do |mb_track|
+              unless ['[silence]', '[untitled]'].include? mb_track['recording']['title']
+                track = Track.new
+                track.name = mb_track['recording']['title']
+                track.album_id = album.id
+                track.position = mb_track['position']
+                track.length = mb_track['length'] unless mb_track['length'].nil?
+                track.length = mb_track['recording']['length'] unless mb_track['recording']['length'].nil?
+                track.country = mb_track['country']
+                track.bonus = mb_track['bonus']
+                track.live = mb_track['recording']['title'].downcase.include? 'live'
+                track.acoustic = mb_track['recording']['title'].downcase.include? 'acoustic'
+                track.mbid = mb_track['recording']['id']
+                track.save
+              end
+            end
+          
+          end # unless mb_main_tracks.empty?
        
-      end # unless mb_releases.empty?
+        end # unless mb_releases.empty?
      
-    end # mb_albums.each do |mb_album|
+      end # mb_albums.each do |mb_album|
+      
+      artist.status = 1
+      
+    rescue
+      artist.status = 2
+    end
    
-    artist.status = 1
    artist.save
    
  end # def self.importArtist name