localhots
/
oldhaven
1
0
Fork 0
Code Releases Activity

Workers handler & api parser draft

This commit is contained in:
magnolia-fan 2011-04-03 00:15:22 +04:00
parent 150109fd15
commit 10486b5150
2 changed files with 121 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
php/api_test.php Executable file
View File

@ -0,0 +1,46 @@
<?php
echo '<pre>';
require_once 'common.php';
$q = 'Fighters';
$api_config = array(
'api_id' => '2259817',
'api_key' => 'h0MiRNiffT',
'api_key1' => 'FUrv4Dutqz4Lnzdh6UTe',
'user_id' => '5728795',
);
$post_v = array(
'api_id' => $api_config['api_id'],
'method' => 'audio.search&q='.$q,
'sig' => md5($api_config['user_id'].'api_id='.$api_config['api_id'].'method=audio.searchq='.$q.'v=3.0format=JSONtest_mode=1'.$api_config['api_key']),
'v' => '3.0',
'format' => 'JSON',
'test_mode' => 1,
'sid' => '47c2f5501b22a3e3aa6947e5e74d1a72381267df2502570eb75c94481ade'
);
$post = '';
foreach($post_v as $key => $val) {
$post .= '&'. $key .'='. $val;
}
$post = substr($post, 1);
$api_url = 'http://api.vk.com/api.php';
$referer = 'http://vk.com/app'. $api_config['api_id'] .'_'. $api_config['user_id'];
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $api_url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_REFERER, $referer);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $post);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 15);
curl_setopt($ch, CURLOPT_USERAGENT, 'test');
$data = curl_exec($ch);
curl_close($ch);
echo $post."\n\n";
echo(htmlentities($data));

88
php/bin/parser/handler.php
View File

@ -4,27 +4,89 @@
chdir('../..');
require_once 'common.php';
$workers = json_decode(file_get_contents('/www/parser_data/pid'));
if (!$workers) {
$workers = array('html_grabber' => array(), 'curl_mtgrabber' => array());
$tmp = array_slice(scandir(Config::get('app:Parser:bot_stats_dir')), 2);
$bots = array();
foreach ($tmp as $bfile) {
$bots[str_replace('.json', '', $bfile)] = json_decode(file_get_contents(Config::get('app:Parser:bot_stats_dir').$bfile));
}
foreach ($bots as $name => $bot) {
$tmp = exec("ps ".$bot->pid);
$bot->active = (int) (strpos($tmp, strval($bot->pid)) !== false);
}
if (!isset($argv[1])) {
echo "Bad action\n";
die;
}
switch ($argv[1]) {
case 'add':
shell_exec("./worker_". $argv[2] .".php > /www/parse.log &");
break;
case 'remove':
if (isset($workers[$argv[2]]) && count($workers[$argv[2]]) > 0) {
$pid = $workers[$argv[2]][0];
shell_exec("kill $pid");
$bot_name = (isset($argv[2]) ? ucfirst($argv[2]) : false);
if ($bot_name && isset($bots[$bot_name])) {
$bot = $bots[$bot_name];
if($bot->active) {
echo $bot_name ." is working already\n";
} else {
echo "Launching ". $bot_name ."\n";
shell_exec("./worker_html_grabber.php ". $bot_name ." > /www/parser_data/log/". $bot_name .".log &");
}
} else {
$bots = custom_shuffle($bots);
foreach ($bots as $name => $bot) {
if (!$bot->active) {
echo "Launching ". $name ."\n";
shell_exec("./worker_html_grabber.php ". $name ." > /www/parser_data/log/". $name .".log &");
die;
}
}
echo "All bots are working already\n";
}
case 'status':
break;
case 'retire':
$bot_name = (isset($argv[2]) ? ucfirst($argv[2]) : false);
if ($bot_name == 'all') {
foreach ($bots as $name => $bot) {
if ($bot->active) {
echo "Stopping ". $name ."\n";
shell_exec("kill ". $bot->pid);
}
}
} elseif ($bot_name && isset($bots[$bot_name])) {
$bot = $bots[$bot_name];
if(!$bot->active) {
echo $bot_name ." is not working now\n";
} else {
echo "Stopping ". $bot_name ."\n";
shell_exec("kill ". $bot->pid);
}
} else {
$bots = custom_shuffle($bots);
foreach ($bots as $name => $bot) {
if ($bot->active) {
echo "Stopping ". $name ."\n";
shell_exec("kill ". $bot->pid);
die;
}
}
echo "All bots are working already\n";
}
break;
case 'list':
foreach($workers as $type => $pids) {
echo "$type: ". count($pids) ."\n";
foreach($bots as $name => $bot) {
echo $name .': '. ($bot->active ? 'working' : 'down') ."\n";
}
break;
default:
echo "Bad action\n";
break;
}
function custom_shuffle($my_array = array()) {
$copy = array();
while (count($my_array)) {
$element = array_rand($my_array);
$copy[$element] = $my_array[$element];
unset($my_array[$element]);
}
return $copy;
}