论坛徽章:: 0

电梯直达

1楼 [收藏(0)] [报告]

发表于 2008-03-03 18:24 |只看该作者 |倒序浏览

10可用积分

这是一份开源的代码,但是似乎有点问题,只能取出英文的关键词,似乎还是有问题的,YUI
例子里用的YUI,这个我没有上传

我想分析和统计整个网站的关键词和出现的频率和分数
请问有没有好的实现方法

keyworddensity.rar (5.44 KB, 下载次数: 75)

[ 本帖最后由 workingbeijing 于 2008-3-3 18:39 编辑 ]

文库|博客

workingbeijing

白手起家

论坛徽章:: 0

2楼 [报告]

发表于 2008-03-04 09:29 |只看该作者

config.ini
<?php
set_time_limit(360);
//define('URL_SEED','http://games.hawkenterprises.org/keyworddensity/test.html');//trailing slash
//define('DOMAIN','hawkenterprises.com');
define('SINGLE_SITE',true);
define('SINGLE_PAGE',true);
define('DROP_COUNT',0); // drop keywords that appear less than 4 times

$exclude_terms_url = array(0=>'javascript:','mailto:','file:','.pdf','.jpg','.gif','.png','.doc','#','.xls','.tar','.gz','feed:');
$dropwords = array(0=>'and','the','is','it','a',' ','an','or','of','on','for','to');
?>

run.php
<?php
function get_document($urltofetch){
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $urltofetch);

curl_setopt($ch, CURLOPT_RETURNTRANSFER,1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
$str = curl_exec($ch);
curl_close($ch);
return $str;
}
function parsed_document_links($document_string){
$hrefs = array();
$doc = new DOMDocument();
@$doc->loadHTML($document_string);
$anchor_tags = $doc->getElementsByTagName('a');
foreach($anchor_tags as $anchor){
      $hrefs[] = $anchor->getAttribute('href');
}
return $hrefs;
}
function normalize_links($links_array){
global $exclude_terms_url;
$url_normalized = array();
foreach($links_array as $k=>$v){
      $skip_url = false;
      foreach($exclude_terms_url as $key=>$value){
         if(stristr($v,$value) !== false)
            $skip_url = true;
      }
      if(!$skip_url){
         $parsed_link = @parse_url($v);
         if(SINGLE_SITE){
            if(isset($parsed_link['scheme']) && $parsed_link['scheme'] == 'http'){
                  if(stristr($v,DOMAIN) !== false){
                     $url_normalized[] = $v;
                  }
            }else{
                  $url_normalized[] = URL_SEED . $v;
            }
         }
      }
}
return $url_normalized;
}
?>

crawl.php
<?php
function get_document($urltofetch){
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $urltofetch);

curl_setopt($ch, CURLOPT_RETURNTRANSFER,1);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
$str = curl_exec($ch);
curl_close($ch);
return $str;
}
function parsed_document_links($document_string){
$hrefs = array();
$doc = new DOMDocument();
@$doc->loadHTML($document_string);
$anchor_tags = $doc->getElementsByTagName('a');
foreach($anchor_tags as $anchor){
      $hrefs[] = $anchor->getAttribute('href');
}
return $hrefs;
}
function normalize_links($links_array){
global $exclude_terms_url;
$url_normalized = array();
foreach($links_array as $k=>$v){
      $skip_url = false;
      foreach($exclude_terms_url as $key=>$value){
         if(stristr($v,$value) !== false)
            $skip_url = true;
      }
      if(!$skip_url){
         $parsed_link = @parse_url($v);
         if(SINGLE_SITE){
            if(isset($parsed_link['scheme']) && $parsed_link['scheme'] == 'http'){
                  if(stristr($v,DOMAIN) !== false){
                     $url_normalized[] = $v;
                  }
            }else{
                  $url_normalized[] = URL_SEED . $v;
            }
         }
      }
}
return $url_normalized;
}
?>

[ 本帖最后由 workingbeijing 于 2008-3-4 09:58 编辑 ]