Last.fm mega import tasks

Gemfile

@@ -3,6 +3,7 @@ source 'http://rubygems.org'
 gem 'rails', '3.1.0'
 
 gem 'json'
+gem 'nokogiri'
 
 gem 'haml'
 gem 'sass'

Gemfile.lock

@@ -136,6 +136,7 @@ DEPENDENCIES
   json
   lastfm!
   musicbrainz (~> 0.4.3)
+  nokogiri
   pg
   rails (= 3.1.0)
   sass

app/controllers/import_controller.rb

@@ -1,5 +1,6 @@
 require 'lastfm'
 require 'musicbrainz'
+require 'nokogiri'
 
 class ImportController < ApplicationController
   @@lastfm_api_key = '04fda005dbf61a50af5abc3e90f111f2'
@@ -10,15 +11,21 @@ class ImportController < ApplicationController
     # Initializing gems
     lastfm = Lastfm.new(@@lastfm_api_key, @@lastfm_secret)
     
-    artist = Artist.find_by_name(name)
+    artist = Artist.find_or_create_by_name(name)
+    return 3 if artist.status == 1
     
     begin
       lastfm_artist = lastfm.artist.get_info(name)
-      brainz_artist = MusicBrainz::Artist.find_by_name(name)
+      unless artist.mbid.nil?
+        brainz_artist = MusicBrainz::Artist.find(artist.mbid)
+      else
+        brainz_artist = MusicBrainz::Artist.find_by_name(name)
+      end
     rescue => e
       lastfm_artist = {
         'bio' => { 'summary' => '' },
-        'image' => [ nil, nil, nil, { 'content' => '' } ]
+        'image' => [ nil, nil, nil, { 'content' => '' } ],
+        'stats' => { 'listeners' => 0 }
       }
       ap e.message
       ap e.backtrace
@@ -28,6 +35,7 @@ class ImportController < ApplicationController
       # Save artist
       artist.desc = lastfm_artist['bio']['summary']
       artist.pic_url = lastfm_artist['image'][3]['content']
+      artist.listeners = lastfm_artist['stats']['listeners']
       artist.artist_type = brainz_artist.type
       artist.mbid = brainz_artist.id
       dry_run ? ap(artist) : artist.save
@@ -85,5 +93,17 @@ class ImportController < ApplicationController
     end
     
     artist.save unless dry_run
+    artist.status
   end
+  
+  def self.parseLastfmXML path
+    Nokogiri::XML(open(path)).css('lfm > artists > artist').map do |node|
+      artist = {}
+      artist[:name] = node.css('name').text
+      artist[:mbid] = nil
+      artist[:mbid] = node.css('mbid').text unless node.css('mbid').empty?
+      artist
+    end
+  end
+  
 end

db/migrate/20110915220228_add_listeners_to_artist.rb

@@ -0,0 +1,5 @@
+class AddListenersToArtist < ActiveRecord::Migration
+  def change
+    add_column :artists, :listeners, :integer
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 20110915043009) do
+ActiveRecord::Schema.define(:version => 20110915220228) do
 
   create_table "albums", :force => true do |t|
     t.string   "name"
@@ -35,6 +35,7 @@ ActiveRecord::Schema.define(:version => 20110915043009) do
     t.string   "artist_type"
     t.string   "mbid"
     t.integer  "status"
+    t.integer  "listeners"
   end
 
   create_table "delayed_jobs", :force => true do |t|

lib/tasks/lastfm_import.rake

@@ -0,0 +1,22 @@
+namespace :lastfm do
+  namespace :import do
+    desc 'Import top 1000 artists from last.fm'
+    task :top => :environment do
+      ImportController.parseLastfmXML('tmp/data/top1000.xml').each do |artist|
+        puts artist[:name] + (artist[:mbid].nil? ? '' : ' (' + artist[:mbid] + ')')
+        record = Artist.find_or_create_by_name(artist[:name])
+        record.mbid = artist[:mbid]
+        record.save
+        res = ImportController.importArtist(artist[:name])
+        if res == 1
+          puts 'OK'
+        elsif res == 2
+          puts 'FAIL'
+        elsif res == 3
+          puts 'SKIP'
+        end
+        puts ''
+      end
+    end
+  end
+end