论坛徽章:: 0

电梯直达

1楼 [收藏(0)] [报告]

发表于 2004-10-20 01:35 |只看该作者 |倒序浏览

public class Spider implements Runnable {
private ArrayList urls; //URL列表
private HashMap indexedURLs; //已经检索过的URL列表
private int threads ; //初始化线程数
public static void main(String argv[]) throws Exception {
if(argv[0] == null){
System.out.println("Missing required argument: [Sit URL]"

;
         return ;
      }
Spider Spider = new Spider(argv[0]);
Spider.go();
}
public Spider(String strURL) {
urls = new ArrayList();
      threads = 10;
      urls.add(strURL);
      threadList = new ArrayList();
      indexedURLs = new HashMap();

      if (urls.size() == 0)
         throw new IllegalArgumentException("Missing required argument: -u [start url]"

;
      if (threads < 1)
         throw new IllegalArgumentException("Invalid number of threads: " +
            threads);
}
public void go(String strURL) throws Exception {
      // index each entry point URL
      long start = System.currentTimeMillis();
      for (int i = 0; i < threads; i++) {
         Thread t = new Thread(this, "Spide " + (i+1));
         t.start();
         threadList.add(t);
      }
      while (threadList.size() >; 0) {
         Thread child = (Thread)threadList.remove(0);
         child.join();
      }
      long elapsed = System.currentTimeMillis() - start;
}
public void run() {
      String url;
      try {
         while ((url = dequeueURL()) != null) {
            indexURL(url);
         }
      }catch(Exception e) {
            logger.info(e.getMessage());
      }
}
//检测URL列表容器中有没有URL没有被解析,如果有则返回URL由线程继续执行

public synchronized String dequeueURL() throws Exception {
      while (true) {
         if (urls.size() >; 0) {
            return (String)urls.remove(0);
         }else {
            threads--;
            if (threads >; 0) {
                  wait();
                  threads++;
            }else {
                  notifyAll();
                  return null;
            }
         }
      }
}
/*
   * 添加URL和当前URL的级数，并唤醒睡眠线程
   */
public synchronized void enqueueURL(String url,int level) {
      if (indexedURLs.get(url) == null) {
         urls.add(url);
         indexedURLs.put(url, new Integer(level));
         notifyAll();
      }
}
/**
   * 通过URL解析出网页内容并解析出页面上的URL
   * @param url 页面链接
   * @throws java.lang.Exception
   */
private void indexURL(String url) throws Exception {
      boolean flag = true ;
   //判断网页链接的级别，系统默认为三级
      int level = 1 ;
      if (indexedURLs.get(url) == null) {
         indexedURLs.put(url, new Integer(level));
      }else{
         level = ((Integer)indexedURLs.get(url)).intValue();
         //只检测到页面的第二级
         if(level >; 2 )
         return ;
         level++ ;
      }

      String strBody = null ;
      try{
            //解析页面内容
            strBody = loadURL(url);
      }catch(Exception e){
            return ;
      }
      if (strBody != null) {
      String urlGroups[] = null ;
      try{
               //解析出页面所以URL
               urlGroups = parseURLs(summary);
      }catch(Exception e){
               logger.info(e.getMessage());
      }
      if(urlGroups == null)
               urlGroups = new String[0] ;

      strBody = null ;
      for (int i = 0; i < urlGroups.length; i++) {
            enqueueURL(urlGroups,level);
      }
      }
}

}

请各位高手评评，这里面有几个很大的问题，线程出现死锁、并且该程序占用系统cpu很高，请高手指出问题，并提出解决方案。沟通让人进步!!!!!!