/* 1 生成 HttpClinet 對(duì)象并設(shè)置參數(shù)*/
 HttpClient httpClient=new HttpClient();
 //設(shè)置 Http 連接超時(shí)為5秒
httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);
  
 /*2 生成 GetMethod 對(duì)象并設(shè)置參數(shù)*/
 GetMethod getMethod=new GetMethod(url);   
 //設(shè)置 get 請(qǐng)求超時(shí)為 5 秒
getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT,5000);
 //設(shè)置請(qǐng)求重試處理，用的是默認(rèn)的重試處理：請(qǐng)求三次
getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
     new DefaultHttpMethodRetryHandler());
  
 /*3 執(zhí)行 HTTP GET 請(qǐng)求*/
 try{ 
   int statusCode = httpClient.executeMethod(getMethod);
   /*4 判斷訪問(wèn)的狀態(tài)碼*/
   if (statusCode != HttpStatus.SC_OK) 
   {
System.err.println("Method failed: "+ getMethod.getStatusLine());
   }
  
   /*5 處理 HTTP 響應(yīng)內(nèi)容*/
   //HTTP響應(yīng)頭部信息，這里簡(jiǎn)單打印
 Header[] headers=getMethod.getResponseHeaders();
   for(Header h: headers)
      System.out.println(h.getName()+" "+h.getValue());*/
   //讀取 HTTP 響應(yīng)內(nèi)容，這里簡(jiǎn)單打印網(wǎng)頁(yè)內(nèi)容
   byte[] responseBody = getMethod.getResponseBody();//讀取為字節(jié)數(shù)組
System.out.println(new String(responseBody));
   //讀取為 InputStream，在網(wǎng)頁(yè)內(nèi)容數(shù)據(jù)量大時(shí)候推薦使用
   InputStream response = getMethod.getResponseBodyAsStream();//
   …
}
catch (HttpException e) 
{
   // 發(fā)生致命的異常，可能是協(xié)議不對(duì)或者返回的內(nèi)容有問(wèn)題
     System.out.println("Please check your provided http address!");
e.printStackTrace();
   } 
catch (IOException e)
 {
      // 發(fā)生網(wǎng)絡(luò)異常
    e.printStackTrace();
   } finally {
         /*6 .釋放連接*/
      getMethod.releaseConnection();      
      } 

PostMethod postMethod = new PostMethod("http://dict.cn/");
postMethod.setRequestBody(new NameValuePair[]{new NameValuePair("q","java")}); 

// 循環(huán)訪問(wèn)所有節(jié)點(diǎn)，輸出包含關(guān)鍵字的值節(jié)點(diǎn)
public static void extractKeyWordText(String url, String keyword) {
    try {
      //生成一個(gè)解析器對(duì)象，用網(wǎng)頁(yè)的 url 作為參數(shù)
      Parser parser = new Parser(url);
      //設(shè)置網(wǎng)頁(yè)的編碼,這里只是請(qǐng)求了一個(gè) gb2312 編碼網(wǎng)頁(yè)
      parser.setEncoding("gb2312");
      //迭代所有節(jié)點(diǎn), null 表示不使用 NodeFilter
      NodeList list = parser.parse(null);
      //從初始的節(jié)點(diǎn)列表跌倒所有的節(jié)點(diǎn)
      processNodeList(list, keyword);
    } catch (ParserException e) {
      e.printStackTrace();
    }
  }
 
  private static void processNodeList(NodeList list, String keyword) {
    //迭代開(kāi)始
    SimpleNodeIterator iterator = list.elements();
    while (iterator.hasMoreNodes()) {
      Node node = iterator.nextNode();
      //得到該節(jié)點(diǎn)的子節(jié)點(diǎn)列表
      NodeList childList = node.getChildren();
      //孩子節(jié)點(diǎn)為空，說(shuō)明是值節(jié)點(diǎn)
      if (null == childList)
      {
        //得到值節(jié)點(diǎn)的值
        String result = node.toPlainTextString();
        //若包含關(guān)鍵字，則簡(jiǎn)單打印出來(lái)文本
        if (result.indexOf(keyword) != -1)
          System.out.println(result);
      } //end if
      //孩子節(jié)點(diǎn)不為空，繼續(xù)迭代該孩子節(jié)點(diǎn)
      else
      {
        processNodeList(childList, keyword);
      }//end else
    }//end wile
  } 

private static void processNodeList(NodeList list, String keyword) 

public static void extractKeyWordText(String url, String keyword) 

// 獲取一個(gè)網(wǎng)頁(yè)上所有的鏈接和圖片鏈接
public static void extracLinks(String url) {
    try {
      Parser parser = new Parser(url);
      parser.setEncoding("gb2312");
//過(guò)濾 <frame> 標(biāo)簽的 filter，用來(lái)提取 frame 標(biāo)簽里的 src 屬性所、表示的鏈接
      NodeFilter frameFilter = new NodeFilter() {
        public boolean accept(Node node) {
          if (node.getText().startsWith("frame src=")) {
            return true;
          } else {
            return false;
          }
        }
      };
    //OrFilter 來(lái)設(shè)置過(guò)濾 <a> 標(biāo)簽，<img> 標(biāo)簽和 <frame> 標(biāo)簽，三個(gè)標(biāo)簽是 or 的關(guān)系
   OrFilte rorFilter = new OrFilter(new NodeClassFilter(LinkTag.class), new
NodeClassFilter(ImageTag.class));
   OrFilter linkFilter = new OrFilter(orFilter, frameFilter);
  //得到所有經(jīng)過(guò)過(guò)濾的標(biāo)簽
  NodeList list = parser.extractAllNodesThatMatch(linkFilter);
  for (int i = 0; i < list.size(); i++) {
    Node tag = list.elementAt(i);
    if (tag instanceof LinkTag)//<a> 標(biāo)簽 
    {
      LinkTag link = (LinkTag) tag;
      String linkUrl = link.getLink();//url
      String text = link.getLinkText();//鏈接文字
      System.out.println(linkUrl + "**********" + text);
    }
    else if (tag instanceof ImageTag)//<img> 標(biāo)簽
    {
      ImageTag image = (ImageTag) list.elementAt(i);
      System.out.print(image.getImageURL() + "********");//圖片地址
      System.out.println(image.getText());//圖片文字
    }
    else//<frame> 標(biāo)簽
    {
//提取 frame 里 src 屬性的鏈接如 <frame src="test.html"/>
      String frame = tag.getText();
      int start = frame.indexOf("src=");
      frame = frame.substring(start);
      int end = frame.indexOf(" ");
      if (end == -1)
        end = frame.indexOf(">");
      frame = frame.substring(5, end - 1);
      System.out.println(frame);
    }
  }
} catch (ParserException e) {
      e.printStackTrace();
}
} 

StringBean sb = new StringBean();
sb.setLinks(false);//設(shè)置結(jié)果中去點(diǎn)鏈接
sb.setURL(url);//設(shè)置你所需要濾掉網(wǎng)頁(yè)標(biāo)簽的頁(yè)面 url 
System.out.println(sb.getStrings());//打印結(jié)果 

package com.ie;
 import java.util.Set;
public class Crawler {
  /* 使用種子 url 初始化 URL 隊(duì)列*/
  private void initCrawlerWithSeeds(String[] seeds)
  {
    for(int i=0;i<seeds.length;i++)
      LinkDB.addUnvisitedUrl(seeds[i]);
  }
   
  /* 爬取方法*/
  public void crawling(String[] seeds)
  {
    LinkFilter filter = new LinkFilter(){
      //提取以 http://www.twt.edu.cn 開(kāi)頭的鏈接
      public boolean accept(String url) {
        if(url.startsWith("http://www.twt.edu.cn"))
          return true;
        else
          return false;
      }
    };
    //初始化 URL 隊(duì)列
    initCrawlerWithSeeds(seeds);
    //循環(huán)條件：待抓取的鏈接不空且抓取的網(wǎng)頁(yè)不多于 1000
    while(!LinkDB.unVisitedUrlsEmpty()&&LinkDB.getVisitedUrlNum()<=1000)
    {
      //隊(duì)頭 URL 出對(duì)
      String visitUrl=LinkDB.unVisitedUrlDeQueue();
      if(visitUrl==null)
        continue;
      FileDownLoader downLoader=new FileDownLoader();
      //下載網(wǎng)頁(yè)
      downLoader.downloadFile(visitUrl);
      //該 url 放入到已訪問(wèn)的 URL 中
      LinkDB.addVisitedUrl(visitUrl);
      //提取出下載網(wǎng)頁(yè)中的 URL
       
      Set<String> links=HtmlParserTool.extracLinks(visitUrl,filter);
      //新的未訪問(wèn)的 URL 入隊(duì)
      for(String link:links)
      {
          LinkDB.addUnvisitedUrl(link);
      }
    }
  }
  //main 方法入口
  public static void main(String[]args)
  {
    Crawler crawler = new Crawler();
    crawler.crawling(new String[]{"http://www.twt.edu.cn"});
  }
}

package com.ie;
import java.util.HashSet;
import java.util.Set;
/**
 * 用來(lái)保存已經(jīng)訪問(wèn)過(guò) Url 和待訪問(wèn)的 Url 的類
 */
public class LinkDB {
  //已訪問(wèn)的 url 集合
  private static Set<String> visitedUrl = new HashSet<String>();
  //待訪問(wèn)的 url 集合
  private static Queue<String> unVisitedUrl = new Queue<String>();
  public static Queue<String> getUnVisitedUrl() {
    return unVisitedUrl;
  }
  public static void addVisitedUrl(String url) {
    visitedUrl.add(url);
  }
  public static void removeVisitedUrl(String url) {
    visitedUrl.remove(url);
  }
  public static String unVisitedUrlDeQueue() {
    return unVisitedUrl.deQueue();
  }
  // 保證每個(gè) url 只被訪問(wèn)一次
  public static void addUnvisitedUrl(String url) {
    if (url != null && !url.trim().equals("")
 && !visitedUrl.contains(url)
        && !unVisitedUrl.contians(url))
      unVisitedUrl.enQueue(url);
  }
  public static int getVisitedUrlNum() {
    return visitedUrl.size();
  }
  public static boolean unVisitedUrlsEmpty() {
    return unVisitedUrl.empty();
  }
}

package com.ie;
import java.util.LinkedList;
/**
 * 數(shù)據(jù)結(jié)構(gòu)隊(duì)列
 */
public class Queue<T> {
  private LinkedList<T> queue=new LinkedList<T>();
  public void enQueue(T t)
  {
    queue.addLast(t);
  }
  public T deQueue()
  {
    return queue.removeFirst();
  }
  public boolean isQueueEmpty()
  {
    return queue.isEmpty();
  }
  public boolean contians(T t)
  {
    return queue.contains(t);
  }
  public boolean empty()
  {
    return queue.isEmpty();
  }
} 

package com.ie;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
public class FileDownLoader {
  /**根據(jù) url 和網(wǎng)頁(yè)類型生成需要保存的網(wǎng)頁(yè)的文件名
   *去除掉 url 中非文件名字符 
   */
  public String getFileNameByUrl(String url,String contentType)
  {
    url=url.substring(7);//remove http://
    if(contentType.indexOf("html")!=-1)//text/html
    {
      url= url.replaceAll("[\\?/:*|<>\"]", "_")+".html";
      return url;
    }
    else//如application/pdf
    {
return url.replaceAll("[\\?/:*|<>\"]", "_")+"."+ \
     contentType.substring(contentType.lastIndexOf("/")+1);
    }  
  }
  /**保存網(wǎng)頁(yè)字節(jié)數(shù)組到本地文件
   * filePath 為要保存的文件的相對(duì)地址
   */
  private void saveToLocal(byte[] data,String filePath)
  {
    try {
      DataOutputStream out=new DataOutputStream(
new FileOutputStream(new File(filePath)));
      for(int i=0;i<data.length;i++)
      out.write(data[i]);
      out.flush();
      out.close();
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
  /*下載 url 指向的網(wǎng)頁(yè)*/
  public String downloadFile(String url)
  {
     String filePath=null;
     /* 1.生成 HttpClinet 對(duì)象并設(shè)置參數(shù)*/
     HttpClient httpClient=new HttpClient();
     //設(shè)置 Http 連接超時(shí) 5s
        httpClient.getHttpConnectionManager().getParams().
setConnectionTimeout(5000);
     /*2.生成 GetMethod 對(duì)象并設(shè)置參數(shù)*/
     GetMethod getMethod=new GetMethod(url);   
     //設(shè)置 get 請(qǐng)求超時(shí) 5s
     getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT,5000);
     //設(shè)置請(qǐng)求重試處理
     getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
      new DefaultHttpMethodRetryHandler());
     /*3.執(zhí)行 HTTP GET 請(qǐng)求*/
     try{ 
       int statusCode = httpClient.executeMethod(getMethod);
       //判斷訪問(wèn)的狀態(tài)碼
       if (statusCode != HttpStatus.SC_OK) 
       {
System.err.println("Method failed: "+ getMethod.getStatusLine());
         filePath=null;
       }
       /*4.處理 HTTP 響應(yīng)內(nèi)容*/
 byte[] responseBody = getMethod.getResponseBody();//讀取為字節(jié)數(shù)組
       //根據(jù)網(wǎng)頁(yè) url 生成保存時(shí)的文件名
filePath="temp\\"+getFileNameByUrl(url,
      getMethod.getResponseHeader("Content-Type").getValue());
      saveToLocal(responseBody,filePath);
     } catch (HttpException e) {
          // 發(fā)生致命的異常，可能是協(xié)議不對(duì)或者返回的內(nèi)容有問(wèn)題
          System.out.println("Please check your provided http 
address!");
          e.printStackTrace();
         } catch (IOException e) {
          // 發(fā)生網(wǎng)絡(luò)異常
          e.printStackTrace();
         } finally {
          // 釋放連接
          getMethod.releaseConnection();      
         }
         return filePath;
  }
  //測(cè)試的 main 方法
  public static void main(String[]args)
  {
    FileDownLoader downLoader = new FileDownLoader();
    downLoader.downloadFile("http://www.twt.edu.cn");
  }
} 

package com.ie;
import java.util.HashSet;
import java.util.Set;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.filters.OrFilter;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
public class HtmlParserTool {
  // 獲取一個(gè)網(wǎng)站上的鏈接,filter 用來(lái)過(guò)濾鏈接
  public static Set<String> extracLinks(String url,LinkFilter filter) {
    Set<String> links = new HashSet<String>();
    try {
      Parser parser = new Parser(url);
      parser.setEncoding("gb2312");
      // 過(guò)濾 <frame >標(biāo)簽的 filter，用來(lái)提取 frame 標(biāo)簽里的 src 屬性所表示的鏈接
      NodeFilter frameFilter = new NodeFilter() {
        public boolean accept(Node node) {
          if (node.getText().startsWith("frame src=")) {
            return true;
          } else {
            return false;
          }
        }
      };
      // OrFilter 來(lái)設(shè)置過(guò)濾 <a> 標(biāo)簽，和 <frame> 標(biāo)簽
      OrFilter linkFilter = new OrFilter(new NodeClassFilter(
          LinkTag.class), frameFilter);
      // 得到所有經(jīng)過(guò)過(guò)濾的標(biāo)簽
      NodeList list = parser.extractAllNodesThatMatch(linkFilter);
      for (int i = 0; i < list.size(); i++) {
        Node tag = list.elementAt(i);
        if (tag instanceof LinkTag)// <a> 標(biāo)簽
        {
          LinkTag link = (LinkTag) tag;
          String linkUrl = link.getLink();// url
          if(filter.accept(linkUrl))
            links.add(linkUrl);
        } else// <frame> 標(biāo)簽
        {
        // 提取 frame 里 src 屬性的鏈接如 <frame src="test.html"/>
          String frame = tag.getText();
          int start = frame.indexOf("src=");
          frame = frame.substring(start);
          int end = frame.indexOf(" ");
          if (end == -1)
            end = frame.indexOf(">");
          String frameUrl = frame.substring(5, end - 1);
          if(filter.accept(frameUrl))
            links.add(frameUrl);
        }
      }
    } catch (ParserException e) {
      e.printStackTrace();
    }
    return links;
  }
  //測(cè)試的 main 方法
  public static void main(String[]args)
  {
Set<String> links = HtmlParserTool.extracLinks(
"http://www.twt.edu.cn",new LinkFilter()
    {
      //提取以 http://www.twt.edu.cn 開(kāi)頭的鏈接
      public boolean accept(String url) {
        if(url.startsWith("http://www.twt.edu.cn"))
          return true;
        else
          return false;
      }
    });
    for(String link : links)
      System.out.println(link);
  }
} 

package com.ie;
public interface LinkFilter {
  public boolean accept(String url);
}