diff --git a/Gemfile b/Gemfile index 34eab8b..23b291f 100644 --- a/Gemfile +++ b/Gemfile @@ -10,6 +10,7 @@ gem 'mongo', '1.3.0' gem 'bson_ext', '1.3.0' gem 'mongoid', '~> 2.0' gem 'nokogiri' +gem 'daemons' # Use unicorn as the web server # gem 'unicorn' @@ -33,3 +34,4 @@ gem 'nokogiri' # group :development, :test do # gem 'webrat' # end + diff --git a/Gemfile.lock b/Gemfile.lock index 4172ad4..0f7cf2f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -32,6 +32,7 @@ GEM bson (1.3.0) bson_ext (1.3.0) builder (2.1.2) + daemons (1.1.2) erubis (2.6.6) abstract (>= 1.0.0) i18n (0.5.0) @@ -81,6 +82,7 @@ PLATFORMS DEPENDENCIES bson_ext (= 1.3.0) + daemons mongo (= 1.3.0) mongoid (~> 2.0) nokogiri diff --git a/app/controllers/bot_controller.rb b/app/controllers/bot_controller.rb new file mode 100644 index 0000000..fcc244c --- /dev/null +++ b/app/controllers/bot_controller.rb @@ -0,0 +1,131 @@ +class BotController < ApplicationController + require 'daemons' + + PIDS_PATH = "#{Rails.root.to_s}/tmp/pids" + + @@accounts = YAML.load_file("#{Rails.root.to_s}/config/vk_accounts.yml") + @error = "" + + def list + @bots = [] + @@accounts.each do |bot_name, data| + bot = Bot.new(bot_name) + @bots << { + :name => bot_name, + :started => bot.running?, + :current_track => bot.getCurrentTrack, + :started_at => bot.running? ? bot.startedAt : nil + } + end + pp @bots + end + + def stats + end + + + def queue + bot_name = params['bot_name'] + group_by_artist = params['group_by_artist'] + limit = params['limit'] + + if not bot_name.nil? + if not @@accounts.include?(bot_name) + throw "Wrong bot name." + end + end + if not group_by_artist.nil? + group_by_artist = true + end + if params['limit'].to_i != 0 + limit = params['limit'].to_i + end + + statuses = [0, 1, 4] + data = ParseQueue.any_in(status: statuses) + if not bot_name.nil? + data = data.where(:bot_name => bot_name) + end +# if group_by_artist +# data = data.only(:artist_name).group +# end + if not limit.nil? + data = data.limit(limit) + end + @queue = [] + unless data.nil? + data.each do |track| + @queue << track + end + end + pp @queue + end + + def start + name = params['name'] + begin + if name.nil? or name.strip.empty? + throw "Empty bot name" + end + + bot = Bot.new(name) + bot.start + rescue + return render :file => "#{Rails.root}/public/404.html", :status => 404, :layout => false + end + end + + def stop + name = params['name'] + begin + if name.nil? or name.strip.empty? + throw "Empty bot name" + end + bot = Bot.new(name) + bot.stop + rescue + return render :file => "#{Rails.root}/public/404.html", :status => 404, :layout => false + end + end + + # Add artist to parse_queue + def add + name = params['name'] + if name.nil? or name.strip.empty? + redirect_to '/' + end + + artist = Artist.getByName(name) + if artist.nil? + @error = "There is no such artist." + else + data = ParseQueue.where(:artist_id => artist.id).first + if not data.nil? + throw "This artist already exist in queue." + else + tracks = [] + artist.albums.each do |album| + unless album.releases.empty? + album.releases.first.tracks.each do |track| + tracks << track + end + end + end + tracks.each do |track| + ParseQueue.collection.insert({ + :track_id => track.id, + :artist_id => artist.id, + :artist_name => artist.name, + :track_name => track.name, + :bot_name => nil, + :status => 0, + :date => nil, + :times_failed => 0 + }) + end + end + end + end + +end + diff --git a/app/models/bot.rb b/app/models/bot.rb new file mode 100644 index 0000000..e2eb8da --- /dev/null +++ b/app/models/bot.rb @@ -0,0 +1,54 @@ +# encoding: UTF-8 +class Bot < Daemons::PidFile + require "daemons" + + PIDS_PATH = "#{Rails.root.to_s}/tmp/pids" + DAEMON_PATH = "#{Rails.root.to_s}/lib/daemons" + @@accounts = YAML.load_file("#{Rails.root.to_s}/config/vk_accounts.yml") + + def initialize(name) + unless @@accounts.include?(name) + throw "Empty bot name." + end + + @dir = File.expand_path(PIDS_PATH) + @progname = name + @multiple = false + @number = nil + end + + def getCurrentTrack + current_track = ParseQueue.where(:status => 4, :bot_name => @progname).first + end + + def startedAt + unless self.running? + return false + end + ts = eval('`date -d "\`ps -p ' + self.pid.to_s + ' -o lstart=\`" +"%Y-%m-%d %H:%M:%S"`') + ts = ts.strip + end + + def start + if bot.running? + throw "This bot already started." + end + eval('`ruby ' + DAEMON_PATH + '/parse_controller.rb ' + Rails.root.to_s + ' ' + name + '`') + end + + def stop + if not self.running? + throw "This bot already stopped." + end + + signal = (RUBY_PLATFORM =~ /win32/ ? 'KILL' : 'TERM') + pid = self.pid + begin + Process.kill(signal, self.pid) + self.cleanup + rescue Errno::ESRCH => e + throw "#{e} #{pid}" + end + end +end + diff --git a/app/models/parse_queue.rb b/app/models/parse_queue.rb new file mode 100644 index 0000000..b09a556 --- /dev/null +++ b/app/models/parse_queue.rb @@ -0,0 +1,14 @@ +class ParseQueue + include Mongoid::Document + store_in :bot_parse_queue + + key :track_id, Integer + key :artist_id, Integer + key :artist_name, String + key :track_name, String + key :bot_name, String + key :status, Integer + key :date, Array + key :times_failed, Integer +end + diff --git a/app/views/bot/list.html.erb b/app/views/bot/list.html.erb new file mode 100644 index 0000000..ac979f8 --- /dev/null +++ b/app/views/bot/list.html.erb @@ -0,0 +1,2 @@ +

Bot#list

+

Find me in app/views/bot/list.html.erb

diff --git a/app/views/bot/stats.html.erb b/app/views/bot/stats.html.erb new file mode 100644 index 0000000..903ed07 --- /dev/null +++ b/app/views/bot/stats.html.erb @@ -0,0 +1,2 @@ +

Bot#stats

+

Find me in app/views/bot/stats.html.erb

diff --git a/config/routes.rb b/config/routes.rb index 598c993..5ad0a82 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,4 +1,13 @@ Beathaven::Application.routes.draw do + get "bot/list" + + get "bot/stats" + + match "bot/queue/" => "bot#queue" + match "bot/start/:name" => "bot#start" + match "bot/stop/:name" => "bot#stop" + match "bot/add/:name" => "bot#add" + get "main/index" root :to => "main#index", :as => "main" @@ -6,12 +15,12 @@ Beathaven::Application.routes.draw do match 'artist(/:name)' => 'artist#view' match 'listen/:id' => 'track#listen' match 'search/autocomplete' => 'artist#autocomplete' - + # Registration & login match 'reg/:email/:code' => 'user#register', :constraints => { :email => /[-a-z0-9\._@]+/i, :code => /[a-f0-9]{64}/ } match 'reg/complete' => 'user#complete' match 'login' => 'user#login' - + match '*a', :to => 'errors#routing' end diff --git a/lib/daemons/parse.rb b/lib/daemons/parse.rb new file mode 100644 index 0000000..1de394c --- /dev/null +++ b/lib/daemons/parse.rb @@ -0,0 +1,73 @@ +ENV["RAILS_ENV"] ||= "development" + +require ARGV[0] + "/config/environment.rb" +require "daemons" +require "pp" + +ROOT_PATH = ARGV[0] +LOGS_PATH = ROOT_PATH + "/log" +BOT_NAME = ARGV[1] + +@@logger = Logger.new("/var/www/bh/log/Parser.log") +@@logger.info(ARGV) + +loop { + sleep(10) +} +#loop { + +# Получаем список песен для загрузки, помечаем в очереди, что мы их взяли в работу +# bot_name = +# ParseQueue.collection.update({status: {$in: [0, 1, 4]}, $atomic: true}, {$set: {bot_name: bot_name}}) +# $queue = $db->getRows($db->q("SELECT * FROM beathaven.queue WHERE status=0 ORDER BY priority DESC, times_failed ASC LIMIT ". QUEUE_PACK)); +# $ids = array(); +# foreach ($queue as $t) { +# $ids[] = $t['track_id']; +# } +# $db->q('UPDATE beathaven.queue SET status=1 WHERE track_id IN('. implode(',', $ids) .')'); +# $db->q('COMMIT'); + +# if (!$queue || count($queue) == 0) { +# sleep(EMPTY_QUEUE_TIMEOUT); +# } else { +# $stats['queue_size'] = count($queue); +# foreach ($queue as $t) { +# $t1 = microtime(true); +# echo "{$t['track_id']} {$t['track_title']} -- "; +# $ok = $vk->getTracks($t['track_title']); + +# if (strpos($vk->getHtml(), 'searchOffset') === false) { +# echo "Session kaput!\n"; +# die; +# } + +# if ($ok) { +# echo "OK\n"; +# $db->q("UPDATE beathaven.queue SET status=3 WHERE track_id=". $t['track_id']); +# $file_name = Config::get('app:Parser:good_html_dir'). $t['track_id'] .'.html'; +# $stats['good_results']++; +# } else { +# echo "FAILED\n"; +# $db->q("UPDATE beathaven.queue SET status = 2, times_failed = times_failed + 1 WHERE track_id=". $t['track_id']); +# $file_name = Config::get('app:Parser:bad_html_dir'). $t['track_id'] .'.html'; +# $stats['bad_results']++; +# } +# file_put_contents($file_name, $vk->getHtml()); +# chmod($file_name, 0777); + +# $stats['last_request'] = $t['track_title']; +# $stats['queue_size']--; +# $stats['eneded_job'] = time(); + +# $bot_stats_file_name = Config::get('app:Parser:bot_stats_dir'). $bot_name .'.json'; +# file_put_contents($bot_stats_file_name, json_encode($stats)); +# chmod($bot_stats_file_name, 0777); + +# $t2 = microtime(true); +# if ($t2 - $t1 < VKTIMEOUT) { +# sleep(ceil(VKTIMEOUT - ($t2 - $t1))); +# } +# } +# } +#} + diff --git a/lib/daemons/parse_controller.rb b/lib/daemons/parse_controller.rb new file mode 100644 index 0000000..e793d25 --- /dev/null +++ b/lib/daemons/parse_controller.rb @@ -0,0 +1,21 @@ +ENV["RAILS_ENV"] ||= "development" + +return false if ARGV.empty? + +require ARGV[0] + "/config/environment.rb" +require "daemons" +require "pp" + +ROOT_PATH = ARGV[0] +PIDS_PATH = ROOT_PATH + "/tmp/pids" + +options = { + :app_name => ARGV[1], + :dir_mode => :normal, + :ARGV => ["start", "--", ROOT_PATH, ARGV[1]], + :dir => PIDS_PATH, + :ontop => false, + :backtrace => false +} +Daemons.run(File.dirname(__FILE__) << '/parse.rb', options) + diff --git a/log/test.log b/log/test.log deleted file mode 100644 index e69de29..0000000 diff --git a/test/functional/bot_controller_test.rb b/test/functional/bot_controller_test.rb new file mode 100644 index 0000000..0f869c8 --- /dev/null +++ b/test/functional/bot_controller_test.rb @@ -0,0 +1,34 @@ +require 'test_helper' + +class BotControllerTest < ActionController::TestCase + test "should get list" do + get :list + assert_response :success + end + + test "should get stats" do + get :stats + assert_response :success + end + + test "should get queue" do + get :queue + assert_response :success + end + + test "should get start" do + get :start + assert_response :success + end + + test "should get stop" do + get :stop + assert_response :success + end + + test "should get add" do + get :add + assert_response :success + end + +end diff --git a/test/unit/helpers/bot_helper_test.rb b/test/unit/helpers/bot_helper_test.rb new file mode 100644 index 0000000..342f51c --- /dev/null +++ b/test/unit/helpers/bot_helper_test.rb @@ -0,0 +1,4 @@ +require 'test_helper' + +class BotHelperTest < ActionView::TestCase +end