localhots/oldhaven
1
0
Fork 0
Code Releases Activity

Preparing to multi-account search

Browse Source
This commit is contained in:
magnolia-fan
2011-04-02 18:57:17 +04:00
parent d411680b1b
commit 41efeface3
13 changed files with 160 additions and 460 deletions
Download Patch File Download Diff File Expand all files Collapse all files
php/bin/parse_queue.php
+9 -4
View File
@@ -3,14 +3,15 @@
chdir('..');
require_once 'common.php';
Config::loadFile('app', 'config/app.ini');
define('VKTIMEOUT', 1);
$db = Db::getInstance();
$vk = new Vkontakte();
$continue = true;
while ($continue) {
$queue = $db->getRows($db->q("SELECT * FROM beathaven.queue WHERE status=0 OR status=2 ORDER BY priority DESC, times_failed ASC LIMIT 10"));
$queue = $db->getRows($db->q("SELECT * FROM beathaven.queue WHERE status=0 OR status=2 ORDER BY priority DESC, status ASC, times_failed ASC LIMIT 100"));
if (!$queue || count($queue) == 0) {
$continue = false;
} else {
@@ -19,6 +20,7 @@ while ($continue) {
echo "Loading file list: ";
$t1 = microtime(true);
$vk->parse($t['track_title']);
$vk->parseHtml();
$files = $vk->getFiles();
if ($files && count($files) > 0) {
echo "OK (".count($files).")\n";
@@ -27,14 +29,17 @@ while ($continue) {
BeatDB::set($t['track_id'], $files);
echo "Good files: ". count($files) ."\n\n";
$db->q("UPDATE beathaven.queue SET status=1 WHERE track_id=". $t['track_id']);
@unlink('/www/parser_data/html/'. $t['track_id'] .'.html');
} else {
file_put_contents('/www/parser_data/html/'. $t['track_id'] .'.html', $vk->getHtml());
echo "FAILED\n\n";
$db->q("UPDATE beathaven.queue SET status = 2, times_failed = times_failed + 1 WHERE track_id=". $t['track_id']);
}
$t2 = microtime(true);
if ($t2 - $t1 < 5) {
sleep(ceil(5 - ($t2 - $t1)));
if ($t2 - $t1 < VKTIMEOUT) {
sleep(ceil(VKTIMEOUT - ($t2 - $t1)));
}
}
sleep(5);
}
}
php/bin/parser/accounts.txt
+10
View File
@@ -0,0 +1,10 @@
[
{
"email" : "chezzzy@yandex.ru",
"password" : "somepass","user_id":,"remixsid":""}
]
alexgreen1978@gmail.com
fbcn136
bc5386a4f49f8bf7df20e11bdd311a7120818d83c23d93cd08177d5d3674
69139853
php/bin/parser/handler.php
Executable
+30
View File
@@ -0,0 +1,30 @@
#!/opt/local/bin/php
<?php
chdir('../..');
require_once 'common.php';
$workers = json_decode(file_get_contents('/www/parser_data/pid'));
if (!$workers) {
$workers = array('html_grabber' => array(), 'curl_mtgrabber' => array());
}
switch ($argv[1]) {
case 'add':
shell_exec("./worker_". $argv[2] .".php > /www/parse.log &");
break;
case 'remove':
if (isset($workers[$argv[2]]) && count($workers[$argv[2]]) > 0) {
$pid = $workers[$argv[2]][0];
shell_exec("kill $pid");
}
case 'status':
case 'list':
foreach($workers as $type => $pids) {
echo "$type: ". count($pids) ."\n";
}
break;
default:
break;
}
php/bin/parser/worker_html_grabber.php
+33
View File
@@ -0,0 +1,33 @@
#!/opt/local/bin/php
<?php
chdir('../..');
require_once 'common.php';
$db = Db::getInstance();
$vk = new Vkontakte();
$continue = true;
while ($continue) {
$queue = $db->getRows($db->q("SELECT * FROM beathaven.queue WHERE status=0 OR status=2 ORDER BY priority DESC, times_failed ASC LIMIT 10"));
if (!$queue || count($queue) == 0) {
$continue = false;
} else {
foreach ($queue as $t) {
echo "#{$t['track_id']} {$t['track_title']}\n";
$vk->parse($t['track_title']);
echo $vk->getHtml();
$db->q("UPDATE beathaven.queue SET status=1 WHERE track_id=". $t['track_id']);
} else {
echo "FAILED\n\n";
$db->q("UPDATE beathaven.queue SET status = 2, times_failed = times_failed + 1 WHERE track_id=". $t['track_id']);
}
$t2 = microtime(true);
if ($t2 - $t1 < 5) {
sleep(ceil(5 - ($t2 - $t1)));
}
}
}
}