nutch-1.7-二次开发-Content中增加编码

1 识别nutch-1.7的编码，完成



以前1.2是在 org.apache.nutch.parse.html.HtmlParser

               EncodingDetector detector = new EncodingDetector(conf);

               detector.autoDetectClues(content, true);

               detector.addClue(sniffCharacterEncoding(contentInOctets), "sniffed");

               String encoding = detector.guessEncoding(content,

                         defaultCharEncoding);



               metadata.set(Metadata.ORIGINAL_CHAR_ENCODING, encoding);

               metadata.set(Metadata.CHAR_ENCODING_FOR_CONVERSION, encoding);



              ------在这里讲识别出来的编码写进content的metadata中

               System.out.println("set charset in Content " + encoding);

               content.getMetadata().add("charset", encoding);



但是在1.7中却不行了。原因不详



目前的策略是，org.apache.nutch.protocol.http.api.HttpBase 
位置：src/plugin/lib-http/src/java下的org.apache.nutch.protocol.http.api包中

中进行修改。



做法是将HtmlParser中的代码搬过来。



               EncodingDetector detector = new EncodingDetector(conf);

               detector.autoDetectClues(c, true);

               detector

               .addClue(sniffCharacterEncoding(c.getContent()), "sniffed");

               String encoding = detector.guessEncoding(c, "windows-1252");



               System.out.println("set charset in Content " + encoding);

               c.getMetadata().add("charset", encoding);

同时将相关函数字段拷贝过来：

// NUTCH-1006 Meta equiv with single quotes not accepted

    private static Pattern metaPattern = Pattern.compile(

            "<meta\\s+([^>]*http-equiv=(\"|')?content-type(\"|')?[^>]*)>",

            Pattern.CASE_INSENSITIVE);

    private static Pattern charsetPattern = Pattern.compile(

            "charset=\\s*([a-z][_\\-0-9a-z]*)", Pattern.CASE_INSENSITIVE);



    // I used 1000 bytes at first, but found that some documents have

    // meta tag well past the first 1000 bytes.

    // (e.g. http://cn.promo.yahoo.com/customcare/music.html)

    private static final int CHUNK_SIZE = 2000;



    /**

     * Given a <code>byte[]</code> representing an html file of an

     * <em>unknown</em> encoding, read out 'charset' parameter in the meta tag

     * from the first <code>CHUNK_SIZE</code> bytes. If there's no meta tag for

     * Content-Type or no charset is specified, <code>null</code> is returned. <br />

     * FIXME: non-byte oriented character encodings (UTF-16, UTF-32) can't be

     * handled with this. We need to do something similar to what's done by

     * mozilla

     * (http://lxr.mozilla.org/seamonkey/source/parser/htmlparser/src/nsParser

     * .cpp#1993). See also http://www.w3.org/TR/REC-xml/#sec-guessing <br />

     * 

     * @param content

     *            <code>byte[]</code> representation of an html file

     */



    private static String sniffCharacterEncoding(byte[] content) {

        int length = content.length < CHUNK_SIZE ? content.length : CHUNK_SIZE;



        // We don't care about non-ASCII parts so that it's sufficient

        // to just inflate each byte to a 16-bit value by padding.

        // For instance, the sequence {0x41, 0x82, 0xb7} will be turned into

        // {U+0041, U+0082, U+00B7}.

        String str = "";

        try {

            str = new String(content, 0, length, Charset.forName("ASCII")

                    .toString());

        } catch (UnsupportedEncodingException e) {

            // code should never come here, but just in case...

            return null;

        }



        Matcher metaMatcher = metaPattern.matcher(str);

        String encoding = null;

        if (metaMatcher.find()) {

            Matcher charsetMatcher = charsetPattern.matcher(metaMatcher

                    .group(1));

            if (charsetMatcher.find())

                encoding = new String(charsetMatcher.group(1));

        }



        return encoding;

    }

 

同时修改：SegmentReader的中的方法
public void reduce(Text key, Iterator<NutchWritable> values,

      OutputCollector<Text, Text> output, Reporter reporter)

          throws IOException {

    StringBuffer dump = new StringBuffer();



    dump.append("\nRecno:: ").append(recNo++).append("\n");

    dump.append("URL:: " + key.toString() + "\n");

    while (values.hasNext()) {

      Writable value = values.next().get(); // unwrap

      if (value instanceof CrawlDatum) {

        dump.append("\nCrawlDatum::\n").append(((CrawlDatum) value).toString());

      } else if (value instanceof Content) {

        Content ct = (Content)value; String charset = ct.getMetadata().get("charset"); dump.append("\nContent::\n").append(ct.toString(charset));

      } else if (value instanceof ParseData) {

        dump.append("\nParseData::\n").append(((ParseData) value).toString());

      } else if (value instanceof ParseText) {

        dump.append("\nParseText::\n").append(((ParseText) value).toString());

      } else if (LOG.isWarnEnabled()) {

        LOG.warn("Unrecognized type: " + value.getClass());

      }

    }

    output.collect(key, new Text(dump.toString()));

  }