hmtl 乱码总结

  改写源码:


    package org.htmlparser.lexer;  
    import java.io.*;  
    import java.lang.reflect.InvocationTargetException;  
    import java.lang.reflect.Method;  
    import java.net.*;  
    import java.util.zip.*;  
    import org.htmlparser.http.ConnectionManager;  
    import org.htmlparser.util.ParserException;  
      
    // Referenced classes of package org.htmlparser.lexer:  
    //            InputStreamSource, PageIndex, StringSource, Cursor,   
    //            Stream, Source  
      
    public class Page  
        implements Serializable  
    {  
      
        public Page()  
        {  
            this("");  
        }  
      
        public Page(URLConnection connection)  
            throws ParserException  
        {  
            if(null == connection)  
            {  
                throw new IllegalArgumentException("connection cannot be null");  
            } else  
            {  
                setConnection(connection);  
                mBaseUrl = null;  
                return;  
            }  
        }  
      
        public Page(InputStream stream, String charset)  
            throws UnsupportedEncodingException  
        {  
            if(null == stream)  
                throw new IllegalArgumentException("stream cannot be null");  
            if(null == charset)  
                charset = "ISO-8859-1";  
            mSource = new InputStreamSource(stream, charset);  
            mIndex = new PageIndex(this);  
            mConnection = null;  
            mUrl = null;  
            mBaseUrl = null;  
        }  
      
        public Page(String text, String charset)  
        {  
            if(null == text)  
                throw new IllegalArgumentException("text cannot be null");  
            if(null == charset)  
                charset = "ISO-8859-1";  
            mSource = new StringSource(text, charset);  
            mIndex = new PageIndex(this);  
            mConnection = null;  
            mUrl = null;  
            mBaseUrl = null;  
        }  
      
        public Page(String text)  
        {  
            this(text, null);  
        }  
      
        public Page(Source source)  
        {  
            if(null == source)  
            {  
                throw new IllegalArgumentException("source cannot be null");  
            } else  
            {  
                mSource = source;  
                mIndex = new PageIndex(this);  
                mConnection = null;  
                mUrl = null;  
                mBaseUrl = null;  
                return;  
            }  
        }  
      
        public static ConnectionManager getConnectionManager()  
        {  
            return mConnectionManager;  
        }  
      
        public static void setConnectionManager(ConnectionManager manager)  
        {  
            mConnectionManager = manager;  
        }  
      
        public String getCharset(String content)  
        {  
            String CHARSET_STRING = "charset";  
            String ret;  
            if(null == mSource)  
                ret = "ISO-8859-1";  
            else  
                ret = mSource.getEncoding();  
            if(null != content)  
            {  
                int index = content.indexOf("charset");  
                if(index != -1)  
                {  
                    content = content.substring(index + "charset".length()).trim();  
                    if(content.startsWith("="))  
                    {  
                        content = content.substring(1).trim();  
                        index = content.indexOf(";");  
                        if(index != -1)  
                            content = content.substring(0, index);  
                        if(content.startsWith("\"") && content.endsWith("\"") && 1 < content.length())  
                            content = content.substring(1, content.length() - 1);  
                        if(content.startsWith("'") && content.endsWith("'") && 1 < content.length())  
                            content = content.substring(1, content.length() - 1);  
                        ret = findCharset(content, ret);  
                    }  
                }  
            }  
            return ret;  
        }  
      
        public static String findCharset(String name, String fallback)  
        {  
            String ret;  
            try  
            {  
                Class cls = Class.forName("java.nio.charset.Charset");  
                Method method = cls.getMethod("forName", new Class[] {  
                    java.lang.String.class  
                });  
                Object object = method.invoke(null, new Object[] {  
                    name  
                });  
                method = cls.getMethod("name", new Class[0]);  
                object = method.invoke(object, new Object[0]);  
                ret = (String)object;  
            }  
            catch(ClassNotFoundException cnfe)  
            {  
                ret = name;  

            }  
            catch(NoSuchMethodException nsme)  
            {  
                ret = name;  
            }  
            catch(IllegalAccessException ia)  
            {  
                ret = name;  
            }  
            catch(InvocationTargetException ita)  
            {  
                ret = fallback;  
                System.out.println("unable to determine cannonical charset name for " + name + " - using " + fallback);  
            }  
            return ret;  
        }  
      
        private void writeObject(ObjectOutputStream out)  
            throws IOException  
        {  
            if(null != getConnection())  
            {  
                out.writeBoolean(true);  
                out.writeInt(mSource.offset());  
                String href = getUrl();  
                out.writeObject(href);  
                setUrl(getConnection().getURL().toExternalForm());  
                Source source = getSource();  
                mSource = null;  
                PageIndex index = mIndex;  
                mIndex = null;  
                out.defaultWriteObject();  
                mSource = source;  
                mIndex = index;  
            } else  
            {  
                out.writeBoolean(false);  
                String href = getUrl();  
                out.writeObject(href);  
                setUrl(null);  
                out.defaultWriteObject();  
                setUrl(href);  
            }  
        }  
      
        private void readObject(ObjectInputStream in)  
            throws IOException, ClassNotFoundException  
        {  
            boolean fromurl = in.readBoolean();  
            if(fromurl)  
            {  
                int offset = in.readInt();  
                String href = (String)in.readObject();  
                in.defaultReadObject();  
                if(null != getUrl())  
                {  
                    URL url = new URL(getUrl());  
                    try  
                    {  
                        setConnection(url.openConnection());  
                    }  
                    catch(ParserException pe)  
                    {  
                        throw new IOException(pe.getMessage());  
                    }  
                }  
                Cursor cursor = new Cursor(this, 0);  
                for(int i = 0; i < offset; i++)  
                    try  
                    {  
                        getCharacter(cursor);  
                    }  
                    catch(ParserException pe)  
                    {  
                        throw new IOException(pe.getMessage());  
                    }  
      
                setUrl(href);  
            } else  
            {  
                String href = (String)in.readObject();  
                in.defaultReadObject();  
                setUrl(href);  
            }  
        }  
      
        public void reset()  
        {  
            getSource().reset();  
            mIndex = new PageIndex(this);  
        }  
      
        public void close()  
            throws IOException  
        {  
            if(null != getSource())  
                getSource().destroy();  
        }  
      
        protected void finalize()  
            throws Throwable  
        {  
            close();  
        }  
      
        public URLConnection getConnection()  
        {  
            return mConnection;  
        }  
      
        public void setConnection(URLConnection connection)  
            throws ParserException  
        {  
            mConnection = connection;  
            mConnection.setConnectTimeout(6000);  
            mConnection.setReadTimeout(6000);  
            try  
            {  
                getConnection().connect();  
            }  
            catch(UnknownHostException uhe)  
            {  
                throw new ParserException("Connect to " + mConnection.getURL().toExternalForm() + " failed.", uhe);  
            }  
            catch(IOException ioe)  
            {  
                throw new ParserException("Exception connecting to " + mConnection.getURL().toExternalForm() + " (" + ioe.getMessage() + ").", ioe);  
            }  
            String type = getContentType();  
            String charset = getCharset(type);  
            try  
            {  
                String contentEncoding = connection.getContentEncoding();  
                System.out.println("contentEncoding="+contentEncoding);  
                Stream stream;  
                if(null != contentEncoding && -1 != contentEncoding.indexOf("gzip"))  
                    stream = new Stream(new GZIPInputStream(getConnection().getInputStream()));  
                else  
                if(null != contentEncoding && -1 != contentEncoding.indexOf("deflate"))  
                    stream = new Stream(new InflaterInputStream(getConnection().getInputStream(), new Inflater(true)));  
                else{  
                    stream = new Stream(getConnection().getInputStream());  
                }  
      
                try  
                {  
                    /*
                     * 时间:2010年8月6日
                     * 原因:当String charset = getCharset(type);返回来的是ISO-8859-1的时候,需要处理一下
                     */  
                    if(charset.indexOf("ISO-8859-1")!=-1){  
                          
                        charset =getGaoBinDEFAULT_CHARSET() ;  
                          
                    }  
            mSource = new InputStreamSource(stream, charset);  
                }  
                catch(UnsupportedEncodingException uee)  
                {  
                    charset = "ISO-8859-1";  
                    mSource = new InputStreamSource(stream, charset);  
                }  
            }  
            catch(IOException ioe)  
            {  
                throw new ParserException("Exception getting input stream from " + mConnection.getURL().toExternalForm() + " (" + ioe.getMessage() + ").", ioe);  
            }  
            mUrl = connection.getURL().toExternalForm();  
            mIndex = new PageIndex(this);  
        }  
      
        public String getUrl()  
        {  
            return mUrl;  
        }  
      
        public void setUrl(String url)  
        {  
            mUrl = url;  
        }  
      
        public String getBaseUrl()  
        {  
            return mBaseUrl;  
        }  
      
        public void setBaseUrl(String url)  
        {  
            mBaseUrl = url;  
        }  
      
        public Source getSource()  
        {  
            return mSource;  
        }  
      
        public String getContentType()  
        {  
            String ret = "text/html";  
            URLConnection connection = getConnection();  
            if(null != connection)  
            {  
                String content = connection.getHeaderField("Content-Type");  
                if(null != content)  
                    ret = content;  
            }  
            return ret;  
        }  
      
        public char getCharacter(Cursor cursor)  
            throws ParserException  
        {  
            int i = cursor.getPosition();  
            int offset = mSource.offset();  
            char ret;  
            if(offset == i)  
                try  
                {  
                    i = mSource.read();  
                    if(-1 == i)  
                    {  
                        ret = '\uFFFF';  
                    } else  
                    {  
                        ret = (char)i;  
                        cursor.advance();  
                    }  
                }  
                catch(IOException ioe)  
                {  
                    throw new ParserException("problem reading a character at position " + cursor.getPosition(), ioe);  
                }  
            else  
            if(offset > i)  
            {  
                try  
                {  
                    ret = mSource.getCharacter(i);  
                }  
                catch(IOException ioe)  
                {  
                    throw new ParserException("can't read a character at position " + i, ioe);  
                }  
                cursor.advance();  
            } else  
            {  
                throw new ParserException("attempt to read future characters from source " + i + " > " + mSource.offset());  
            }  
            if('\r' == ret)  
            {  
                ret = '\n';  
                if(mSource.offset() == cursor.getPosition())  
                    try  
                    {  
                        i = mSource.read();  
                        if(-1 != i)  
                            if('\n' == (char)i)  
                                cursor.advance();  
                            else  
                                try  
                                {  
                                    mSource.unread();  
                                }  
                                catch(IOException ioe)  
                                {  
                                    throw new ParserException("can't unread a character at position " + cursor.getPosition(), ioe);  
                                }  
                    }  
                    catch(IOException ioe)  
                    {  
                        throw new ParserException("problem reading a character at position " + cursor.getPosition(), ioe);  
                    }  
                else  
                    try  
                    {  
                        if('\n' == mSource.getCharacter(cursor.getPosition()))  
                            cursor.advance();  
                    }  
                    catch(IOException ioe)  
                    {  
                        throw new ParserException("can't read a character at position " + cursor.getPosition(), ioe);  
                    }  
            }  
            if('\n' == ret)  
                mIndex.add(cursor);  
            return ret;  
        }  
      
        public void ungetCharacter(Cursor cursor)  
            throws ParserException  
        {  
            cursor.retreat();  
            int i = cursor.getPosition();  
            try  
            {  
                char ch = mSource.getCharacter(i);  
                if('\n' == ch && 0 != i)  
                {  
                    ch = mSource.getCharacter(i - 1);  
                    if('\r' == ch)  
                        cursor.retreat();  
                }  
            }  
            catch(IOException ioe)  
            {  
                throw new ParserException("can't read a character at position " + cursor.getPosition(), ioe);  
            }  
        }  
      
        public String getEncoding()  
        {  
            return getSource().getEncoding();  
        }  
      
        public void setEncoding(String character_set)  
            throws ParserException  
        {  
            Page.GaoBinDEFAULT_CHARSET = character_set;  
            getSource().setEncoding(character_set);  
        }  
      
        public URL constructUrl(String link, String base)  
            throws MalformedURLException  
        {  
            return constructUrl(link, base, false);  
        }  
      
        public URL constructUrl(String link, String base, boolean strict)  
            throws MalformedURLException  
        {  
            int index;  
            URL url;  
            if(!strict && '?' == link.charAt(0))  
            {  
                if(-1 != (index = base.lastIndexOf('?')))  
                    base = base.substring(0, index);  
                url = new URL(base + link);  
            } else  
            {  
                url = new URL(new URL(base), link);  
            }  
            String path = url.getFile();  
            boolean modified = false;  
            boolean absolute = link.startsWith("/");  
            if(!absolute)  
                do  
                {  
                    if(!path.startsWith("/."))  
                        break;  
                    if(path.startsWith("/../"))  
                    {  
                        path = path.substring(3);  
                        modified = true;  
                        continue;  
                    }  
                    if(!path.startsWith("/./") && !path.startsWith("/."))  
                        break;  
                    path = path.substring(2);  
                    modified = true;  
                } while(true);  
            while(-1 != (index = path.indexOf("/\\")))   
            {  
                path = path.substring(0, index + 1) + path.substring(index + 2);  
                modified = true;  
            }  
            if(modified)  
                url = new URL(url, path);  
            return url;  
        }  
      
        public String getAbsoluteURL(String link)  
        {  
            return getAbsoluteURL(link, false);  
        }  
      
        public String getAbsoluteURL(String link, boolean strict)  
        {  
            String ret;  
            if(null == link || "".equals(link))  
                ret = "";  
            else  
                try  
                {  
                    String base = getBaseUrl();  
                    if(null == base)  
                        base = getUrl();  
                    if(null == base)  
                    {  
                        ret = link;  
                    } else  
                    {  
                        URL url = constructUrl(link, base, strict);  
                        ret = url.toExternalForm();  
                    }  
                }  
                catch(MalformedURLException murle)  
                {  
                    ret = link;  
                }  
            return ret;  
        }  
      
        public int row(Cursor cursor)  
        {  
            return mIndex.row(cursor);  
        }  
      
        public int row(int position)  
        {  
            return mIndex.row(position);  
        }  
      
        public int column(Cursor cursor)  
        {  
            return mIndex.column(cursor);  
        }  
      
        public int column(int position)  
        {  
            return mIndex.column(position);  
        }  
      
        public String getText(int start, int end)  
            throws IllegalArgumentException  
        {  
            String ret;  
            try  
            {  
                ret = mSource.getString(start, end - start);  
            }  
            catch(IOException ioe)  
            {  
                throw new IllegalArgumentException("can't get the " + (end - start) + "characters at position " + start + " - " + ioe.getMessage());  
            }  
            return ret;  
        }  
      
        public void getText(StringBuffer buffer, int start, int end)  
            throws IllegalArgumentException  
        {  
            if(mSource.offset() < start || mSource.offset() < end)  
                throw new IllegalArgumentException("attempt to extract future characters from source" + start + "|" + end + " > " + mSource.offset());  
            int length;  
            if(end < start)  
            {  
                length = end;  
                end = start;  
                start = length;  
            }  
            length = end - start;  
            try  
            {  
                mSource.getCharacters(buffer, start, length);  
            }  
            catch(IOException ioe)  
            {  
                throw new IllegalArgumentException("can't get the " + (end - start) + "characters at position " + start + " - " + ioe.getMessage());  
            }  
        }  
      
        public String getText()  
        {  
            return getText(0, mSource.offset());  
        }  
      
        public void getText(StringBuffer buffer)  
        {  
            getText(buffer, 0, mSource.offset());  
        }  
      
        public void getText(char array[], int offset, int start, int end)  
            throws IllegalArgumentException  
        {  
            if(mSource.offset() < start || mSource.offset() < end)  
                throw new IllegalArgumentException("attempt to extract future characters from source");  
            int length;  
            if(end < start)  
            {  
                length = end;  
                end = start;  
                start = length;  
            }  
            length = end - start;  
            try  
            {  
                mSource.getCharacters(array, offset, start, end);  
            }  
            catch(IOException ioe)  
            {  
                throw new IllegalArgumentException("can't get the " + (end - start) + "characters at position " + start + " - " + ioe.getMessage());  
            }  
        }  
      
        public String getLine(Cursor cursor)  
        {  
            int line = row(cursor);  
            int size = mIndex.size();  
            int start;  
            int end;  
            if(line < size)  
            {  
                start = mIndex.elementAt(line);  
                if(++line <= size)  
                    end = mIndex.elementAt(line);  
                else  
                    end = mSource.offset();  
            } else  
            {  
                start = mIndex.elementAt(line - 1);  
                end = mSource.offset();  
            }  
            return getText(start, end);  
        }  
      
        public String getLine(int position)  
        {  
            return getLine(new Cursor(this, position));  
        }  
      
        public String toString()  
        {  
            String ret;  
            if(mSource.offset() > 0)  
            {  
                StringBuffer buffer = new StringBuffer(43);  
                int start = mSource.offset() - 40;  
                if(0 > start)  
                    start = 0;  
                else  
                    buffer.append("...");  
                getText(buffer, start, mSource.offset());  
                ret = buffer.toString();  
            } else  
            {  
                ret = super.toString();  
            }  
            return ret;  
        }  
      
        public static final String DEFAULT_CHARSET = "ISO-8859-1";  
        public static String GaoBinDEFAULT_CHARSET;  
        public static final String DEFAULT_CONTENT_TYPE = "text/html";  
        public static final char EOF = 65535;  
        protected String mUrl;  
        protected String mBaseUrl;  
        protected Source mSource;  
        protected PageIndex mIndex;  
        protected transient URLConnection mConnection;  
        protected static ConnectionManager mConnectionManager = new ConnectionManager();  
          
        public static String getGaoBinDEFAULT_CHARSET() {  
            return GaoBinDEFAULT_CHARSET;  
        }  
      
        public static void setGaoBinDEFAULT_CHARSET(String gaoBinDEFAULT_CHARSET) {  
            GaoBinDEFAULT_CHARSET = gaoBinDEFAULT_CHARSET;  
        }  
      
    } 

你可能感兴趣的:(乱码)