From 1494e81fa7cd524c26daded9d15fb3d386f80f46 Mon Sep 17 00:00:00 2001 From: magnolia-fan Date: Sun, 10 Apr 2011 04:15:29 +0400 Subject: [PATCH 1/4] Vkontakte html & parse --- Gemfile | 1 + Gemfile.lock | 2 + README | 257 +-------------------------- app/controllers/artist_controller.rb | 1 + app/controllers/track_controller.rb | 6 + app/helpers/vkontakte.rb | 67 ++++++- 6 files changed, 70 insertions(+), 264 deletions(-) diff --git a/Gemfile b/Gemfile index b0ffd8b..34eab8b 100644 --- a/Gemfile +++ b/Gemfile @@ -9,6 +9,7 @@ gem 'pg' gem 'mongo', '1.3.0' gem 'bson_ext', '1.3.0' gem 'mongoid', '~> 2.0' +gem 'nokogiri' # Use unicorn as the web server # gem 'unicorn' diff --git a/Gemfile.lock b/Gemfile.lock index 7324307..4172ad4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -48,6 +48,7 @@ GEM mongo (~> 1.3) tzinfo (~> 0.3.22) will_paginate (~> 3.0.pre) + nokogiri (1.4.4) pg (0.10.1) polyglot (0.3.1) rack (1.2.2) @@ -82,5 +83,6 @@ DEPENDENCIES bson_ext (= 1.3.0) mongo (= 1.3.0) mongoid (~> 2.0) + nokogiri pg rails (= 3.0.5) diff --git a/README b/README index fe7013d..903681d 100644 --- a/README +++ b/README @@ -1,256 +1 @@ -== Welcome to Rails - -Rails is a web-application framework that includes everything needed to create -database-backed web applications according to the Model-View-Control pattern. - -This pattern splits the view (also called the presentation) into "dumb" -templates that are primarily responsible for inserting pre-built data in between -HTML tags. The model contains the "smart" domain objects (such as Account, -Product, Person, Post) that holds all the business logic and knows how to -persist themselves to a database. The controller handles the incoming requests -(such as Save New Account, Update Product, Show Post) by manipulating the model -and directing data to the view. - -In Rails, the model is handled by what's called an object-relational mapping -layer entitled Active Record. This layer allows you to present the data from -database rows as objects and embellish these data objects with business logic -methods. You can read more about Active Record in -link:files/vendor/rails/activerecord/README.html. - -The controller and view are handled by the Action Pack, which handles both -layers by its two parts: Action View and Action Controller. These two layers -are bundled in a single package due to their heavy interdependence. This is -unlike the relationship between the Active Record and Action Pack that is much -more separate. Each of these packages can be used independently outside of -Rails. You can read more about Action Pack in -link:files/vendor/rails/actionpack/README.html. - - -== Getting Started - -1. At the command prompt, create a new Rails application: - rails new myapp (where myapp is the application name) - -2. Change directory to myapp and start the web server: - cd myapp; rails server (run with --help for options) - -3. Go to http://localhost:3000/ and you'll see: - "Welcome aboard: You're riding Ruby on Rails!" - -4. Follow the guidelines to start developing your application. You can find -the following resources handy: - -* The Getting Started Guide: http://guides.rubyonrails.org/getting_started.html -* Ruby on Rails Tutorial Book: http://www.railstutorial.org/ - - -== Debugging Rails - -Sometimes your application goes wrong. Fortunately there are a lot of tools that -will help you debug it and get it back on the rails. - -First area to check is the application log files. Have "tail -f" commands -running on the server.log and development.log. Rails will automatically display -debugging and runtime information to these files. Debugging info will also be -shown in the browser on requests from 127.0.0.1. - -You can also log your own messages directly into the log file from your code -using the Ruby logger class from inside your controllers. Example: - - class WeblogController < ActionController::Base - def destroy - @weblog = Weblog.find(params[:id]) - @weblog.destroy - logger.info("#{Time.now} Destroyed Weblog ID ##{@weblog.id}!") - end - end - -The result will be a message in your log file along the lines of: - - Mon Oct 08 14:22:29 +1000 2007 Destroyed Weblog ID #1! - -More information on how to use the logger is at http://www.ruby-doc.org/core/ - -Also, Ruby documentation can be found at http://www.ruby-lang.org/. There are -several books available online as well: - -* Programming Ruby: http://www.ruby-doc.org/docs/ProgrammingRuby/ (Pickaxe) -* Learn to Program: http://pine.fm/LearnToProgram/ (a beginners guide) - -These two books will bring you up to speed on the Ruby language and also on -programming in general. - - -== Debugger - -Debugger support is available through the debugger command when you start your -Mongrel or WEBrick server with --debugger. This means that you can break out of -execution at any point in the code, investigate and change the model, and then, -resume execution! You need to install ruby-debug to run the server in debugging -mode. With gems, use sudo gem install ruby-debug. Example: - - class WeblogController < ActionController::Base - def index - @posts = Post.find(:all) - debugger - end - end - -So the controller will accept the action, run the first line, then present you -with a IRB prompt in the server window. Here you can do things like: - - >> @posts.inspect - => "[#nil, "body"=>nil, "id"=>"1"}>, - #"Rails", "body"=>"Only ten..", "id"=>"2"}>]" - >> @posts.first.title = "hello from a debugger" - => "hello from a debugger" - -...and even better, you can examine how your runtime objects actually work: - - >> f = @posts.first - => #nil, "body"=>nil, "id"=>"1"}> - >> f. - Display all 152 possibilities? (y or n) - -Finally, when you're ready to resume execution, you can enter "cont". - - -== Console - -The console is a Ruby shell, which allows you to interact with your -application's domain model. Here you'll have all parts of the application -configured, just like it is when the application is running. You can inspect -domain models, change values, and save to the database. Starting the script -without arguments will launch it in the development environment. - -To start the console, run rails console from the application -directory. - -Options: - -* Passing the -s, --sandbox argument will rollback any modifications - made to the database. -* Passing an environment name as an argument will load the corresponding - environment. Example: rails console production. - -To reload your controllers and models after launching the console run -reload! - -More information about irb can be found at: -link:http://www.rubycentral.com/pickaxe/irb.html - - -== dbconsole - -You can go to the command line of your database directly through rails -dbconsole. You would be connected to the database with the credentials -defined in database.yml. Starting the script without arguments will connect you -to the development database. Passing an argument will connect you to a different -database, like rails dbconsole production. Currently works for MySQL, -PostgreSQL and SQLite 3. - -== Description of Contents - -The default directory structure of a generated Ruby on Rails application: - - |-- app - | |-- controllers - | |-- helpers - | |-- mailers - | |-- models - | `-- views - | `-- layouts - |-- config - | |-- environments - | |-- initializers - | `-- locales - |-- db - |-- doc - |-- lib - | `-- tasks - |-- log - |-- public - | |-- images - | |-- javascripts - | `-- stylesheets - |-- script - |-- test - | |-- fixtures - | |-- functional - | |-- integration - | |-- performance - | `-- unit - |-- tmp - | |-- cache - | |-- pids - | |-- sessions - | `-- sockets - `-- vendor - `-- plugins - -app - Holds all the code that's specific to this particular application. - -app/controllers - Holds controllers that should be named like weblogs_controller.rb for - automated URL mapping. All controllers should descend from - ApplicationController which itself descends from ActionController::Base. - -app/models - Holds models that should be named like post.rb. Models descend from - ActiveRecord::Base by default. - -app/views - Holds the template files for the view that should be named like - weblogs/index.html.erb for the WeblogsController#index action. All views use - eRuby syntax by default. - -app/views/layouts - Holds the template files for layouts to be used with views. This models the - common header/footer method of wrapping views. In your views, define a layout - using the layout :default and create a file named default.html.erb. - Inside default.html.erb, call <% yield %> to render the view using this - layout. - -app/helpers - Holds view helpers that should be named like weblogs_helper.rb. These are - generated for you automatically when using generators for controllers. - Helpers can be used to wrap functionality for your views into methods. - -config - Configuration files for the Rails environment, the routing map, the database, - and other dependencies. - -db - Contains the database schema in schema.rb. db/migrate contains all the - sequence of Migrations for your schema. - -doc - This directory is where your application documentation will be stored when - generated using rake doc:app - -lib - Application specific libraries. Basically, any kind of custom code that - doesn't belong under controllers, models, or helpers. This directory is in - the load path. - -public - The directory available for the web server. Contains subdirectories for - images, stylesheets, and javascripts. Also contains the dispatchers and the - default HTML files. This should be set as the DOCUMENT_ROOT of your web - server. - -script - Helper scripts for automation and generation. - -test - Unit and functional tests along with fixtures. When using the rails generate - command, template test files will be generated for you and placed in this - directory. - -vendor - External libraries that the application depends on. Also includes the plugins - subdirectory. If the app has frozen rails, those gems also go here, under - vendor/rails/. This directory is in the load path. +Gouranga gives a fuck \ No newline at end of file diff --git a/app/controllers/artist_controller.rb b/app/controllers/artist_controller.rb index c2813eb..66b1044 100644 --- a/app/controllers/artist_controller.rb +++ b/app/controllers/artist_controller.rb @@ -8,6 +8,7 @@ class ArtistController < ApplicationController name = params[:name].gsub('+', ' ').gsub('%20', ' ') end @artist = Artist.getByName(name) + pp @artist end end diff --git a/app/controllers/track_controller.rb b/app/controllers/track_controller.rb index dd48272..6027c15 100644 --- a/app/controllers/track_controller.rb +++ b/app/controllers/track_controller.rb @@ -2,6 +2,12 @@ class TrackController < ApplicationController require 'net/http' require 'uri' def listen + artist = 'The White Stripes' + track = 'Blue Orchid' + length = 157 + Vkontakte.get(artist, track, length) + + render :json => 0 end end diff --git a/app/helpers/vkontakte.rb b/app/helpers/vkontakte.rb index d614e9c..de00929 100644 --- a/app/helpers/vkontakte.rb +++ b/app/helpers/vkontakte.rb @@ -1,19 +1,70 @@ module Vkontakte - @@accounts = YAML.load_file("#{RAILS_ROOT}/config/vk_accounts.yml") + require 'net/http' + require 'erb' + #require 'nokogiri' - def get artist, track, length - + @@accounts = YAML.load_file("#{Rails.root.to_s}/config/vk_accounts.yml") + + def self.get artist, track, length + #html = self.getHtml artist +' - '+ track + files = self.parseHtml 1 #html + files = self.calcWeight files end - def getHtml q - + + private + + def self.randomBot + @@accounts[@@accounts.keys[rand(@@accounts.keys.length - 1)]] end - def parseHtml html + def self.getHtml q + bot = self.randomBot + headers = { + 'Cookie' => 'remixsid='+ bot['remixsid'] + ';remixchk='+ bot['remixchk'].to_s, + 'Referer' => 'http://vkontakte.ru/audio?album_id=0', + 'X-Requested-With' => 'XMLHttpRequest', + 'Origin' => 'http://vkontakte.ru', + 'Content-Type' => 'application/x-www-form-urlencoded', + 'User-Agent' => bot['user_agent'], + 'Connection' => 'close' + } + data = { + 'act' => 'search', + 'al' => 1, + 'gid' => 0, + 'id' => bot['user_id'], + 'offset' => 0, + 'q' => q, + 'sort' => 2 + } + data_str = [] + data.each {|k, v| data_str << k +'='+ ERB::Util.url_encode(v.to_s)} + http = Net::HTTP.new('vkontakte.ru', 80) + resp, data = http.post('/audio', data_str.join('&'), headers) + pp data end - def calcWeight files - + def self.parseHtml html + html = open('/Users/chez/Sites/beathaven/tmp/vkres.txt').read + files = [] + html + .gsub(/\\/, '') + .scan(/.*?(.*?)<\/div>.*?.*?selectPerformer\(event\,\s\'(.*?)\'\).*?(.*?)<\/span><\/div>.*?<\/table>/i) + .each do |item| + t = item[1].split(':') + files << { + 'url' => item[0], + 'length' => (t[0].to_i * 60) + t[1].to_i, + 'artist' => item[2], + 'title' => item[3].gsub(/<\/?span>/, '') + } + end + files + end + + def self.calcWeight files + pp files end end \ No newline at end of file From 135f10d73acf12cf70aba5155bf1383aa290c486 Mon Sep 17 00:00:00 2001 From: magnolia-fan Date: Sun, 10 Apr 2011 04:55:22 +0400 Subject: [PATCH 2/4] File weight calculator --- app/helpers/vkontakte.rb | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/app/helpers/vkontakte.rb b/app/helpers/vkontakte.rb index de00929..45268f3 100644 --- a/app/helpers/vkontakte.rb +++ b/app/helpers/vkontakte.rb @@ -8,7 +8,8 @@ module Vkontakte def self.get artist, track, length #html = self.getHtml artist +' - '+ track files = self.parseHtml 1 #html - files = self.calcWeight files + files = self.calcWeight files, artist, track, length + self.stream files.first['url'] end @@ -58,13 +59,37 @@ module Vkontakte 'url' => item[0], 'length' => (t[0].to_i * 60) + t[1].to_i, 'artist' => item[2], - 'title' => item[3].gsub(/<\/?span>/, '') + 'track' => item[3].gsub(/<\/?.*?>/, '') } end files end - def self.calcWeight files - pp files + def self.calcWeight files, artist, track, length + files.each do |file| + weight = 0 + + if file['artist'] == artist then weight += 10 else + if file['artist'].include? artist then weight += 5 else + if file['track'].include? artist then weight += 4 + end end end + + if file['track'] == track then weight += 10 else + if file['track'].include? track then weight += 5 + end end + + if file['length'] == length then weight +=10 else + delta = (length - file['length']).abs + if delta < 5 then weight += (5 - delta) end + end + + file['weight'] = weight + end + + files.sort_by{|file| file['weight']}.reverse + end + + def self.stream url + end end \ No newline at end of file From 259cbd6a84ca1db9f352cea5746731a64859c523 Mon Sep 17 00:00:00 2001 From: magnolia-fan Date: Sun, 10 Apr 2011 07:17:34 +0400 Subject: [PATCH 3/4] It fuckin' works\! --- app/controllers/track_controller.rb | 14 ++++--- app/helpers/vkontakte.rb | 58 ++++++++++++++++---------- app/views/layouts/application.html.erb | 2 +- public/javascripts/player.js | 12 ++---- 4 files changed, 50 insertions(+), 36 deletions(-) diff --git a/app/controllers/track_controller.rb b/app/controllers/track_controller.rb index 6027c15..4e284a5 100644 --- a/app/controllers/track_controller.rb +++ b/app/controllers/track_controller.rb @@ -2,12 +2,16 @@ class TrackController < ApplicationController require 'net/http' require 'uri' def listen - artist = 'The White Stripes' - track = 'Blue Orchid' - length = 157 + pp track = Track.where(:id => params[:id].to_i).first + pp release = Release.where(:id => track.release_id).first + pp album = Album.where(:id => release.album_id).first + pp artist = Artist.where(:id => album.artist_id).first - Vkontakte.get(artist, track, length) + data = Vkontakte.get(artist.name, track.name, (track.length / 1000).round) + #data = open('/Users/chez/Sites/beathaven/tmp/media.mp3').read + #send_data data, :type=>"audio/mp3" - render :json => 0 + #render :text => data + redirect_to data end end diff --git a/app/helpers/vkontakte.rb b/app/helpers/vkontakte.rb index 45268f3..a380b22 100644 --- a/app/helpers/vkontakte.rb +++ b/app/helpers/vkontakte.rb @@ -1,40 +1,46 @@ module Vkontakte + require 'uri' require 'net/http' require 'erb' #require 'nokogiri' @@accounts = YAML.load_file("#{Rails.root.to_s}/config/vk_accounts.yml") + @@bot = nil def self.get artist, track, length - #html = self.getHtml artist +' - '+ track - files = self.parseHtml 1 #html - files = self.calcWeight files, artist, track, length - self.stream files.first['url'] + self.randomBot + html = self.getHtml(artist << ' - ' << track).force_encoding("WINDOWS-1251").encode("UTF-8") + #pp html.encoding + #File.open('/Users/chez/Sites/beathaven/tmp/vkres1.html', 'w') {|f| f.write(html) } + #html = open('/Users/chez/Sites/beathaven/tmp/vkres1.html').read + files = self.parseHtml(html) + files = self.calcWeight(files, artist, track, length) + #self.stream files.first['url'] + files.first['url'] end private def self.randomBot - @@accounts[@@accounts.keys[rand(@@accounts.keys.length - 1)]] + @@bot = @@accounts[@@accounts.keys[rand(@@accounts.keys.length - 1)]] end def self.getHtml q - bot = self.randomBot headers = { - 'Cookie' => 'remixsid='+ bot['remixsid'] + ';remixchk='+ bot['remixchk'].to_s, + 'Cookie' => 'remixsid='+ @@bot['remixsid'] + ';remixchk='+ @@bot['remixchk'].to_s, 'Referer' => 'http://vkontakte.ru/audio?album_id=0', 'X-Requested-With' => 'XMLHttpRequest', 'Origin' => 'http://vkontakte.ru', 'Content-Type' => 'application/x-www-form-urlencoded', - 'User-Agent' => bot['user_agent'], + 'User-Agent' => @@bot['user_agent'], 'Connection' => 'close' } data = { 'act' => 'search', 'al' => 1, 'gid' => 0, - 'id' => bot['user_id'], + 'id' => @@bot['user_id'], 'offset' => 0, 'q' => q, 'sort' => 2 @@ -44,24 +50,22 @@ module Vkontakte http = Net::HTTP.new('vkontakte.ru', 80) resp, data = http.post('/audio', data_str.join('&'), headers) - pp data + data end def self.parseHtml html - html = open('/Users/chez/Sites/beathaven/tmp/vkres.txt').read files = [] html - .gsub(/\\/, '') - .scan(/.*?(.*?)<\/div>.*?.*?selectPerformer\(event\,\s\'(.*?)\'\).*?(.*?)<\/span><\/div>.*?<\/table>/i) - .each do |item| - t = item[1].split(':') - files << { - 'url' => item[0], - 'length' => (t[0].to_i * 60) + t[1].to_i, - 'artist' => item[2], - 'track' => item[3].gsub(/<\/?.*?>/, '') - } - end + .scan(/.*?(.*?)<\/div>.*?.*?selectPerformer\(event\,\s\'(.*?)\'\).*?(.*?)<\/span><\/div>.*?<\/table>/mi) + .each do |item| + t = item[1].split(':') + files << { + 'url' => item[0], + 'length' => (t[0].to_i * 60) + t[1].to_i, + 'artist' => item[2], + 'track' => item[3].gsub(/<.*?>/, '') + } + end files end @@ -90,6 +94,16 @@ module Vkontakte end def self.stream url + headers = { + 'Cookie' => 'remixsid='+ @@bot['remixsid'] + ';remixchk='+ @@bot['remixchk'].to_s, + 'Referer' => 'http://vkontakte.ru/audio?album_id=0', + 'User-Agent' => @@bot['user_agent'], + } + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, 80) + resp, data = http.get2(uri.path, headers) + + data end end \ No newline at end of file diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index 16cc2f2..580cb90 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -13,7 +13,7 @@ - - + <%= yield %> diff --git a/public/javascripts/player.js b/public/javascripts/player.js index 98244ae..8f2c7da 100644 --- a/public/javascripts/player.js +++ b/public/javascripts/player.js @@ -1,12 +1,8 @@ $(function(){ $('.play').click(function(){ - $.get('/listen/'+ $(this).parent().attr('id') +'/', function(res){ - if (res.status == 'ok') { - $('#player audio').attr('src', res.data.url); - $('#player #artist').html(res.data.artist); - $('#player #title').html(res.data.track); - $('#player audio').play(); - } - }) + $('#player audio').attr('src', '/listen/'+ $(this).parent().attr('id')) +'/'; + $('#player #artist').html($('h1.artist').html()); + $('#player #title').html($(this).parent().find('.track-name').html()); + $('#player audio').play(); }) }) \ No newline at end of file From 1b6bb9c1b32208e50cea5acdcd0f8152a6419143 Mon Sep 17 00:00:00 2001 From: magnolia-fan Date: Sun, 10 Apr 2011 09:01:11 +0400 Subject: [PATCH 4/4] Bot debug --- app/helpers/vkontakte.rb | 4 +++- config/vk_accounts.yml | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/app/helpers/vkontakte.rb b/app/helpers/vkontakte.rb index a380b22..7c151d6 100644 --- a/app/helpers/vkontakte.rb +++ b/app/helpers/vkontakte.rb @@ -23,7 +23,9 @@ module Vkontakte private def self.randomBot - @@bot = @@accounts[@@accounts.keys[rand(@@accounts.keys.length - 1)]] + botname = @@accounts.keys[rand(@@accounts.keys.length - 1)] + @@bot = @@accounts[botname] + pp 'Using bot ' << botname end def self.getHtml q diff --git a/config/vk_accounts.yml b/config/vk_accounts.yml index 4ed249d..c735dd7 100644 --- a/config/vk_accounts.yml +++ b/config/vk_accounts.yml @@ -30,11 +30,11 @@ Vivaldi: # Flint remixchk: 5 user_agent: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.204 Safari/534.16" -Prokofiev: # Daddy - user_id:24538630 - remixsid: 832fae47c8dfc8df66551b8c72927ca26f6ee49f5754b24e9a9a317805c5 - remixchk: 5 - user_agent: "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; WebMoney Advisor; GTB6.3; InfoPath.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; OfficeLiveConnector.1.3; OfficeLivePatch.0.0; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)" +# Prokofiev: # Daddy +# user_id:24538630 +# remixsid: 832fae47c8dfc8df66551b8c72927ca26f6ee49f5754b24e9a9a317805c5 +# remixchk: 5 +# user_agent: "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; WebMoney Advisor; GTB6.3; InfoPath.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; OfficeLiveConnector.1.3; OfficeLivePatch.0.0; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)" Paganini: # Tom_Orrow user_id: 1554168