Heritrix 3.1.0 源码解析（三十三）
本文要分析的是FetchHTTP处理器，该处理器实现CrawlURI curi对象URL的请求（OutputStream写入数据）并且返回InputStream
FetchHTTP处理器有很多成员变量，均为设置远程请求的相关参数，本人只解释其中重要成员变量
/** * Whether or not to perform an on-the-fly digest hash of retrieved * content-bodies. */ { setDigestContent(true); } public boolean getDigestContent() { return (Boolean) kp.get("digestContent"); } /** * 是否设置摘要内容 * @param digest */ public void setDigestContent(boolean digest) { kp.put("digestContent",digest); } /** * 摘要算法 * Which algorithm (for example MD5 or SHA-1) to use to perform an * on-the-fly digest hash of retrieved content-bodies. */ String digestAlgorithm = "sha1"; public String getDigestAlgorithm() { return digestAlgorithm; } public void setDigestAlgorithm(String digestAlgorithm) { this.digestAlgorithm = digestAlgorithm; } //请求执行对象 private transient HttpClient httpclient = null; CookieStorage cookieStorage = new BdbCookieStorage(); /** * cookies存储对象 * @param storage */ @Autowired(required=false) public void setCookieStorage(CookieStorage storage) { this.cookieStorage = storage; } public CookieStorage getCookieStorage() { return this.cookieStorage; } /** * Used to store credentials. */ { // initialize with empty store so declaration not required setCredentialStore(new CredentialStore()); } public CredentialStore getCredentialStore() { return (CredentialStore) kp.get("credentialStore"); } /** * 证书存储对象 * @param credentials */ @Autowired(required=false) public void setCredentialStore(CredentialStore credentials) { kp.put("credentialStore",credentials); } /** * Used to do DNS lookups. */ protected ServerCache serverCache; public ServerCache getServerCache() { return this.serverCache; } /** * 服务器缓存 * @param serverCache */ @Autowired public void setServerCache(ServerCache serverCache) { this.serverCache = serverCache; }
静态代码块实现注册http和https协议（创建SOCKET工厂，重用IP缓存）
/** * 注册http和https协议 */ static { Protocol.registerProtocol("http", new Protocol("http", new HeritrixProtocolSocketFactory(), 80)); try { ProtocolSocketFactory psf = new HeritrixSSLProtocolSocketFactory(); Protocol p = new Protocol("https", psf, 443); Protocol.registerProtocol("https", p); } catch (KeyManagementException e) { e.printStackTrace(); } catch (KeyStoreException e) { e.printStackTrace(); } catch (NoSuchAlgorithmException e) { e.printStackTrace(); } }
void start()初始化方法实例化HttpClient httpclient对象，设置Cookies，设置SSL工厂
public void start() { if(isRunning()) { return; } super.start(); configureHttp(); if (cookieStorage != null) { cookieStorage.start(); httpclient.getState().setCookiesMap(cookieStorage.getCookiesMap()); } setSSLFactory(); }
void configureHttp(int soTimeout, String addressStr,String proxy, int port, String user, String password)方法初始化httpclient对象
/** * 初始化httpclient对象 */ protected void configureHttp(int soTimeout, String addressStr, String proxy, int port, String user, String password) { // Get timeout. Use it for socket and for connection timeout. int timeout = (soTimeout > 0) ? soTimeout : 0; // HttpConnectionManager cm = new ThreadLocalHttpConnectionManager(); HttpConnectionManager cm = new SingleHttpConnectionManager(); // TODO: The following settings should be made in the corresponding // HttpConnectionManager, not here. HttpConnectionManagerParams hcmp = cm.getParams(); hcmp.setConnectionTimeout(timeout); hcmp.setStaleCheckingEnabled(true); // Minimizes bandwidth usage. Setting to true disables Nagle's // algorithm. IBM JVMs < 142 give an NPE setting this boolean // on ssl sockets. hcmp.setTcpNoDelay(false); this.httpclient = new HttpClient(cm); HttpClientParams hcp = this.httpclient.getParams(); // Set default socket timeout. hcp.setSoTimeout(timeout); // Set client to be version 1.0. hcp.setVersion(HttpVersion.HTTP_1_0); // configureHttpCookies(defaults); // Configure how we want the method to act. this.httpclient.getParams().setParameter( HttpMethodParams.SINGLE_COOKIE_HEADER, new Boolean(true)); this.httpclient.getParams().setParameter( HttpMethodParams.UNAMBIGUOUS_STATUS_LINE, new Boolean(false)); this.httpclient.getParams().setParameter( HttpMethodParams.STRICT_TRANSFER_ENCODING, new Boolean(false)); this.httpclient.getParams().setIntParameter( HttpMethodParams.STATUS_LINE_GARBAGE_LIMIT, 10); if ((proxy != null) && (proxy.length() == 0)) { proxy = null; } HostConfiguration config = httpclient.getHostConfiguration(); //配置代理 configureProxy(proxy, port, user, password, config); //绑定地址 configureBindAddress(addressStr,config); hcmp.setParameter(SSL_FACTORY_KEY, this.sslfactory); }
void configureProxy(String proxy, int port, String user, String password,HostConfiguration config)方法配置代理
/** * 配置代理 * @param proxy * @param port * @param user * @param password * @param config */ private void configureProxy(String proxy, int port, String user, String password, HostConfiguration config) { if(StringUtils.isNotEmpty(proxy)) { config.setProxy(proxy, port); if (StringUtils.isNotEmpty(user)) { Credentials credentials = new NTCredentials(user, password, "", ""); AuthScope authScope = new AuthScope(proxy, port); this.httpclient.getState().setProxyCredentials(authScope, credentials); } } }
void configureBindAddress(String address, HostConfiguration config)绑定地址
/** * 绑定地址 * @param address * @param config */ private void configureBindAddress(String address, HostConfiguration config) { if (StringUtils.isNotEmpty(address)) { try { InetAddress localAddress = InetAddress.getByName(address); config.setLocalAddress(localAddress); } catch (UnknownHostException e) { // Convert all to RuntimeException so get an exception out // if initialization fails. throw new RuntimeException("Unknown host " + address + " in local-address"); } } }
void setSSLFactory() 方法初始化SSLSocketFactory sslfactory工厂
private void setSSLFactory() { // I tried to get the default KeyManagers but doesn't work unless you // point at a physical keystore. Passing null seems to do the right // thing so we'll go w/ that. try { SSLContext context = SSLContext.getInstance("SSL"); context.init(null, new TrustManager[] { new ConfigurableX509TrustManager( getSslTrustLevel()) }, null); this.sslfactory = context.getSocketFactory(); } catch (Exception e) { logger.log(Level.WARNING, "Failed configure of ssl context " + e.getMessage(), e); } }
下面方法为FetchHTTP处理器生命周期相关方法，持久化Cookies并且销毁HttpClient httpclient对象
public void stop() { if(!isRunning()) { return; } super.stop(); // At the end save cookies to the file specified in the order file. if (cookieStorage != null) { @SuppressWarnings("unchecked") Map<String, Cookie> map = httpclient.getState().getCookiesMap(); cookieStorage.saveCookiesMap(map); cookieStorage.stop(); } cleanupHttp(); // XXX happens at finish; move to teardown? } /** * Perform any final cleanup related to the HttpClient instance. */ protected void cleanupHttp() { this.httpclient = null; }
下面我们来看一下FetchHttp处理器的void innerProcess(final CrawlURI curi)方法
protected void innerProcess(final CrawlURI curi) throws InterruptedException { // Note begin time curi.setFetchBeginTime(System.currentTimeMillis());// Get a reference to the HttpRecorder that is set into this ToeThread. Recorder rec = curi.getRecorder(); // Shall we get a digest on the content downloaded? boolean digestContent = getDigestContent(); String algorithm = null; if (digestContent) { algorithm = getDigestAlgorithm(); //设置摘要算法 rec.getRecordedInput().setDigest(algorithm); } else { // clear rec.getRecordedInput().setDigest((MessageDigest)null); } // Below we do two inner classes that add check of midfetch // filters just as we're about to receive the response body. String curiString = curi.getUURI().toString(); HttpMethodBase method = null; if (curi.getFetchType() == HTTP_POST) { method = new HttpRecorderPostMethod(curiString, rec) { protected void readResponseBody(HttpState state, HttpConnection conn) throws IOException, HttpException { addResponseContent(this, curi); if (checkMidfetchAbort(curi, this.httpRecorderMethod, conn)) { doAbort(curi, this, MIDFETCH_ABORT_LOG); } else { super.readResponseBody(state, conn); } } }; } else { method = new HttpRecorderGetMethod(curiString, rec) { protected void readResponseBody(HttpState state, HttpConnection conn) throws IOException, HttpException { //StatusCode和ContentType设置到CrawlURI curi对象 //this为HttpRecorderGetMethod对象 addResponseContent(this, curi); if (checkMidfetchAbort(curi, this.httpRecorderMethod, conn)) { doAbort(curi, this, MIDFETCH_ABORT_LOG); } else { super.readResponseBody(state, conn); } } }; } // Save method into curi too. Midfetch filters may want to leverage // info in here. curi.setHttpMethod(method); //配置HttpMethod设置选项和头部 HostConfiguration customConfigOrNull = configureMethod(curi, method); //添加证书 如Form提交参数等 // Populate credentials. Set config so auth. is not automatic. //添加证书对象 boolean addedCredentials = populateCredentials(curi, method); if (httpclient.getState().getProxyCredentials(new AuthScope(getProxyHost(), getProxyPort())) != null) { addedCredentials = true; } method.setDoAuthentication(addedCredentials); // set hardMax on bytes (if set by operator) long hardMax = getMaxLengthBytes(); // set overall timeout (if set by operator) long timeoutMs = 1000 * getTimeoutSeconds(); // Get max fetch rate (bytes/ms). It comes in in KB/sec long maxRateKBps = getMaxFetchKBSec(); rec.getRecordedInput().setLimits(hardMax, timeoutMs, maxRateKBps); try { //提交http请求 this.httpclient.executeMethod(customConfigOrNull, method); } catch (RecorderTooMuchHeaderException ex) { // when too much header material, abort like other truncations doAbort(curi, method, HEADER_TRUNC); } catch (IOException e) { failedExecuteCleanup(method, curi, e); return; } catch (ArrayIndexOutOfBoundsException e) { // For weird windows-only ArrayIndex exceptions in native // code... see // http://forum.java.sun.com/thread.jsp?forum=11&thread=378356 // treating as if it were an IOException failedExecuteCleanup(method, curi, e); return; } // set softMax on bytes to get (if implied by content-length) long softMax = method.getResponseContentLength(); try { if (!method.isAborted()) { // Force read-to-end, so that any socket hangs occur here, // not in later modules. rec.getRecordedInput().readFullyOrUntil(softMax); } } catch (RecorderTimeoutException ex) { doAbort(curi, method, TIMER_TRUNC); } catch (RecorderLengthExceededException ex) { doAbort(curi, method, LENGTH_TRUNC); } catch (IOException e) { cleanup(curi, e, "readFully", S_CONNECT_LOST); return; } catch (ArrayIndexOutOfBoundsException e) { // For weird windows-only ArrayIndex exceptions from native code // see http://forum.java.sun.com/thread.jsp?forum=11&thread=378356 // treating as if it were an IOException cleanup(curi, e, "readFully", S_CONNECT_LOST); return; } finally { // ensure recording has stopped rec.closeRecorders(); if (!method.isAborted()) { method.releaseConnection(); } // Note completion time curi.setFetchCompletedTime(System.currentTimeMillis()); // Set the response charset into the HttpRecord if available. setCharacterEncoding(curi, rec, method); setSizes(curi, rec); setOtherCodings(curi, rec, method); } if (digestContent) { //设置摘要 curi.setContentDigest(algorithm, rec.getRecordedInput().getDigestValue()); } if (logger.isLoggable(Level.FINE)) { logger.fine(((curi.getFetchType() == HTTP_POST) ? "POST" : "GET") + " " + curi.getUURI().toString() + " " + method.getStatusCode() + " " + rec.getRecordedInput().getSize() + " " + curi.getContentType()); } if (isSuccess(curi) && addedCredentials) { // Promote the credentials from the CrawlURI to the CrawlServer // so they are available for all subsequent CrawlURIs on this // server. //更新服务器证书 promoteCredentials(curi); if (logger.isLoggable(Level.FINE)) { // Print out the cookie. Might help with the debugging. Header setCookie = method.getResponseHeader("set-cookie"); if (setCookie != null) { logger.fine(setCookie.toString().trim()); } } } else if (method.getStatusCode() == HttpStatus.SC_UNAUTHORIZED) { // 401 is not 'success'. //basic/digest认证 handle401(method, curi); } if (rec.getRecordedInput().isOpen()) { logger.severe(curi.toString() + " RIS still open. Should have" + " been closed by method release: " + Thread.currentThread().getName()); try { rec.getRecordedInput().close(); } catch (IOException e) { logger.log(Level.SEVERE, "second-chance RIS close failed", e); } } }
其中boolean populateCredentials(CrawlURI curi, HttpMethod method)方法为添加证书对象
/** * 添加证书对象 * @param curi * @param method * @return */ private boolean populateCredentials(CrawlURI curi, HttpMethod method) { // First look at the server avatars. Add any that are to be volunteered // on every request (e.g. RFC2617 credentials). Every time creds will // return true when we call 'isEveryTime(). String serverKey; try { serverKey = CrawlServer.getServerKey(curi.getUURI()); } catch (URIException e) { return false; } CrawlServer server = serverCache.getServerFor(serverKey); if (server.hasCredentials()) { for (Credential cred : server.getCredentials()) { if (cred.isEveryTime()) { //每次都要添加证书对象 cred.populate(curi, this.httpclient, method); } } } boolean result = false; // Now look in the curi. The Curi will have credentials loaded either // by the handle401 method if its a rfc2617 or it'll have been set into // the curi by the preconditionenforcer as this login uri came through. //在PreconditionEnforcer处理器设置过CrawlURI curi对象的Set<Credential>集合 for (Credential c: curi.getCredentials()) { if (c.populate(curi, this.httpclient, method)) { result = true; } } return result; }
void promoteCredentials(final CrawlURI curi)方法为更新服务器证书
/** * 更新服务器证书 * @param curi */ private void promoteCredentials(final CrawlURI curi) { Set<Credential> credentials = curi.getCredentials(); for (Iterator<Credential> i = credentials.iterator(); i.hasNext();) { Credential c = i.next(); i.remove(); // The server to attach too may not be the server that hosts // this passed curi. It might be of another subdomain. // The avatar needs to be added to the server that is dependent // on this precondition. Find it by name. Get the name from // the credential this avatar represents. String cd = c.getDomain(); if (cd != null) { CrawlServer cs = serverCache.getServerFor(cd); if (cs != null) { cs.addCredential(c); } } } }
void handle401(final HttpMethod method, final CrawlURI curi)方法为处理basic/digest认证
protected void handle401(final HttpMethod method, final CrawlURI curi) { AuthScheme authscheme = getAuthScheme(method, curi); if (authscheme == null) { return; } String realm = authscheme.getRealm(); /* * ======================================================= // Look to * see if this curi had rfc2617 avatars loaded. If so, are // any of * them for this realm? If so, then the credential failed // if we got a * 401 and it should be let die a natural 401 death. if * (curi.detachRfc2617Credential(realm)) { // Then, already tried this * credential. Remove ANY rfc2617 // credential since presence of a * rfc2617 credential serves // as flag to frontier to requeue this curi * and let the curi // die a natural death. logger.warning("Auth failed * (401) though supplied realm " + realm + " to " + curi.toString()); * return; } curi.attachRfc2617Credential(realm); * ============================================================= */ // Look to see if this curi had rfc2617 avatars loaded. If so, are // any of them for this realm? If so, then the credential failed // if we got a 401 and it should be let die a natural 401 death. Set<Credential> curiRfc2617Credentials = getCredentials(curi, HttpAuthenticationCredential.class); HttpAuthenticationCredential extant = HttpAuthenticationCredential.getByRealm( curiRfc2617Credentials, realm, curi); if (extant != null) { // Then, already tried this credential. Remove ANY rfc2617 // credential since presence of a rfc2617 credential serves // as flag to frontier to requeue this curi and let the curi // die a natural death. extant.detachAll(curi); logger.warning("Auth failed (401) though supplied realm " + realm + " to " + curi.toString()); } else { // Look see if we have a credential that corresponds to this // realm in credential store. Filter by type and credential // domain. If not, let this curi die. Else, add it to the // curi and let it come around again. Add in the AuthScheme // we got too. Its needed when we go to run the Auth on // second time around. String serverKey = getServerKey(curi); CrawlServer server = serverCache.getServerFor(serverKey); Set<Credential> storeRfc2617Credentials = getCredentialStore().subset(curi, HttpAuthenticationCredential.class, server.getName()); if (storeRfc2617Credentials == null || storeRfc2617Credentials.size() <= 0) { logger.fine("No rfc2617 credentials for " + curi); } else { HttpAuthenticationCredential found = HttpAuthenticationCredential.getByRealm( storeRfc2617Credentials, realm, curi); if (found == null) { logger.fine("No rfc2617 credentials for realm " + realm + " in " + curi); } else { found.attach(curi); logger.fine("Found credential for realm " + realm + " in store for " + curi.toString()); } } } }
设置CrawlURI curi对象的Recorder httpRecorder成员编码方法
private void setCharacterEncoding(CrawlURI curi, final Recorder rec, final HttpMethod method) { String encoding = ((HttpMethodBase) method).getResponseCharSet(); try { rec.setCharset(Charset.forName(encoding)); } catch (IllegalArgumentException e) { curi.getAnnotations().add("unsatisfiableCharsetInHeader:"+StringUtils.stripToEmpty(encoding)); rec.setCharset(getDefaultCharset()); } }
这种方式获取到的网页编码不一定完全准确，在必要的时候需要我们改写
---------------------------------------------------------------------------
本系列Heritrix 3.1.0 源码解析系本人原创
转载请注明出处 博客园 刺猬的温驯
本文链接 http://www.cnblogs.com/chenying99/archive/2013/05/01/3052930.html