杀智勇双全杀

MapReduce入门

MapReduce模板
- Driver模板
- Map模板
- Reduce模板
WordCount小项目
- Driver类
- Mapper类
- Reducer类
- 集群运行
- - 打开集群
  - 启动ZooKeeper：
  - 启动HDFS：
  - 启动YARN
  - 上传文件
  - 运行脚本
  - 关闭集群
- 本地运行
- - 本地部署
二手房数量统计小项目

MapReduce概述

YARN运行Jar脚本

MapReduce模板

编写代码时，要注意导包的问题。全部选择含Hadoop字眼的包，导错包将会导致程序无法正常运行。

Driver模板

package com.aa.mode;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class MRDriverMode extends Configured implements Tool {
     

    //构建、配置、连接、提交Job
    public int run(String[] args) throws Exception{
     

        //构建Job
        //实例化一个MapReduce的Job对象
        Job job = Job.getInstance(this.getConf(), "mode");
        //指定允许jar包运行的类
        job.setJarByClass(MRDriverMode.class);

        //配置Job
        //Input：配置输入
        //指定输入类的类型
        job.setInputFormatClass(TextInputFormat.class);//可以不指定，不指定就是用默认值TextInputFormat
        //指定输入源
        Path inputPath = new Path(args[0]);//使用第一个参数作为程序的输入
        TextInputFormat.setInputPaths(job,inputPath);

        //Map：配置Map...
        job.setMapperClass(MRMapperMode.class);//设置调用Mapper类
        job.setMapOutputKeyClass(Text.class);//设置K2的类型
        job.setMapOutputValueClass(Text.class);//这只V2的类型

        //Shuffle:配置Shuffle...先写null占位，再慢慢改
        //job.setPartitionerClass(null);//设置分区器
        //job.setSortComparatorClass(null);//设置排序器
        //job.setGroupingComparatorClass(null);//设置分组器
        //job.setCombinerClass(null);//设置Map端聚合



        //Reduce：配置Reduce...
        job.setReducerClass(MRReducerMode.class);//设置调用reduce的类
        job.setOutputKeyClass(Text.class);//设置K3的类型
        job.setOutputValueClass(Text.class);//设置V3的类型
        //job.setNumReduceTasks(1);//设置ReduceTask的个数，默认为1个，写不写都可以

        //Output：配置输出
        //指定输出类的类型
        job.setOutputFormatClass(TextOutputFormat.class);//默认就是TextOutputFormat
        //设置输出的路径
        Path outputPath = new Path(args[1]);
        //判断输出路径是否存在，存在就删除
        FileSystem fs = FileSystem.get(this.getConf());
        if(fs.exists(outputPath)){
     
            fs.delete(outputPath,true);//使用true为递归删除，可以删除目录
        }
        TextOutputFormat.setOutputPath(job,outputPath);

        //提交Job
        return job.waitForCompletion(true)?0:-1;//true会显示执行过程。该方法内部会submit
    }


    //程序入口方法
    public static void main(String[] args) throws Exception{
     
        //构建配置连接对象
        Configuration conf = new Configuration();
        //调用run方法
        int status = ToolRunner.run(conf, new MRDriverMode(), args);
        System.out.println("status = " + status);
        //退出程序
        System.exit(status);
    }
}

配置Map与Reduce的参数时，可以先写null占位（防止报错），再慢慢改。

Map模板

package com.aa.mode;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class MRMapperMode extends Mapper<LongWritable, Text,Text,Text> {
     //这4个参数为：KEYIN, VALUEIN, KEYOUT, VALUEOUT
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
     
        //Map处理的逻辑由实际的需求决定
    }
}

Reduce模板

package com.aa.mode;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

/**
 *
 *
 */
public class MRReducerMode extends Reducer<Text,Text,Text,Text> {
     //这4个参数为：KEYIN, VALUEIN, KEYOUT, VALUEOUT
    @Override
    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
     
        //Reduce的处理逻辑由实际的需求决定
    }
}

以后就可以参照这套模板来写MapReduce代码，又回到了Ctrl+C，Ctrl+V改参数的年代。。。

WordCount小项目

Driver类

package com.aa.wordcount;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class WordCountDriver extends Configured implements Tool {
     

    @Override
    public int run(String[] args) throws Exception {
     

        //构建
        Job job = Job.getInstance(this.getConf(), "userwc");
        job.setJarByClass(WordCountDriver.class);

        //配置
        job.setInputFormatClass(TextInputFormat.class);
        //使用程序的第一个参数作为输入
        TextInputFormat.setInputPaths(job, new Path(args[0]));

        job.setMapperClass(WordCountMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        job.setReducerClass(WordCountReduce.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        job.setOutputFormatClass(TextOutputFormat.class);

        //使用程序的第二个参数作为输出路径
        Path outputPath = new Path(args[1]);
        FileSystem fs = FileSystem.get(this.getConf());
        if (fs.exists(outputPath)) {
     
            fs.delete(outputPath, true);
        }
        TextOutputFormat.setOutputPath(job, outputPath);
        //提交
        return job.waitForCompletion(true) ? 0 : 1;
    }

    public static void main(String[] args) throws Exception {
     
        Configuration conf = new Configuration();
        int status = ToolRunner.run(conf, new WordCountDriver(), args);
        System.out.println("status = " + status);
        System.exit(status);
    }

}

Mapper类

package com.aa.wordcount;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class WordCountMapper extends Mapper<LongWritable, Text,Text, IntWritable> {
     
    //输出的K2
    Text outputKey = new Text();
    //输出的V2
    IntWritable outputValue = new IntWritable(1);

    //每一条KV调用一次map

    @Override//直接输入map就可以重写
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
     
        //将每行的内容分割得到每个单词
        String[] words = value.toString().split("\\s+");
        //根据给定的正则表达式的匹配来拆分此字符串,空格、回车、换行等空白符都可以

        //迭代取出每个单词作为K2，iter，反编译看到，数组的增强for循环遍历底层还是普通for循环
        for (String word : words) {
     
            //将当前的单词作为K2
            this.outputKey.set(word);
            //将K2和V2传递到下一步
            context.write(outputKey,outputValue);
        }

    }
}

Reducer类

package com.aa.wordcount;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class WordCountReduce extends Reducer<Text, IntWritable, Text, IntWritable> {
     

    //输出V3
    IntWritable outputValue = new IntWritable();
    //由于K3=K2，直接使用即可，不需要新建变量

    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
     
        int sum=0;
        for (IntWritable value : values) {
     
            //取出当前单词所有1，进行累加
            sum+=value.get();
        }
        //给V3赋值
        this.outputValue.set(sum);
        //传递到下一步
        context.write(key,outputValue);
    }
}

集群运行

打开集群

参照集群的开机顺序。

启动ZooKeeper：

cd /export/server/zookeeper-3.4.6/

bin/zkServer.sh status

bin/zkServer.sh start
bin/zkServer.sh status

启动HDFS：

start-dfs.sh

启动YARN

start-yarn.sh

使用jps查看进程。

上传文件

上传wc.txt文件：

cd /export/data/

rz上传。
创建HDFS目录：

hdfs dfs -mkdir -p /wordcount/input

复制文件到HDFS：

hdfs dfs -put /export/data/wc.txt /wordcount/input

查看是否成功：

hdfs dfs -ls /wordcount/input

上传Maven编译好的jar包：

cd /export/data/

使用rz上传。

运行脚本

使用YARN运行jar脚本：

yarn jar /export/data/day210426-1.0-SNAPSHOT.jar com.aa.wordcount.WordCountDriver /wordcount/input/wc.txt /wordcount/output1

运行过程：

[root@node1 data]# yarn jar /export/data/day210426-1.0-SNAPSHOT.jar com.aa.wordcount.WordCountDriver /wordcount/input/wc.txt /wordcount/output1       
21/04/26 20:23:27 INFO client.RMProxy: Connecting to ResourceManager at node3/192.168.88.11:8032
21/04/26 20:23:27 INFO input.FileInputFormat: Total input paths to process : 1
21/04/26 20:23:28 INFO mapreduce.JobSubmitter: number of splits:3
21/04/26 20:23:28 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1619438325526_0002
21/04/26 20:23:28 INFO impl.YarnClientImpl: Submitted application application_1619438325526_0002
21/04/26 20:23:28 INFO mapreduce.Job: The url to track the job: http://node3:8088/proxy/application_1619438325526_0002/
21/04/26 20:23:28 INFO mapreduce.Job: Running job: job_1619438325526_0002
21/04/26 20:23:39 INFO mapreduce.Job: Job job_1619438325526_0002 running in uber mode : false
21/04/26 20:23:39 INFO mapreduce.Job:  map 0% reduce 0%
21/04/26 20:24:00 INFO mapreduce.Job:  map 9% reduce 0%
21/04/26 20:24:03 INFO mapreduce.Job:  map 10% reduce 0%
21/04/26 20:24:06 INFO mapreduce.Job:  map 12% reduce 0%
21/04/26 20:24:07 INFO mapreduce.Job:  map 13% reduce 0%
21/04/26 20:24:09 INFO mapreduce.Job:  map 14% reduce 0%
21/04/26 20:24:10 INFO mapreduce.Job:  map 18% reduce 0%
21/04/26 20:24:13 INFO mapreduce.Job:  map 20% reduce 0%
21/04/26 20:24:15 INFO mapreduce.Job:  map 22% reduce 0%
21/04/26 20:24:16 INFO mapreduce.Job:  map 24% reduce 0%
21/04/26 20:24:18 INFO mapreduce.Job:  map 25% reduce 0%
21/04/26 20:24:19 INFO mapreduce.Job:  map 28% reduce 0%
21/04/26 20:24:21 INFO mapreduce.Job:  map 31% reduce 0%
21/04/26 20:24:22 INFO mapreduce.Job:  map 32% reduce 0%
21/04/26 20:24:24 INFO mapreduce.Job:  map 35% reduce 0%
21/04/26 20:24:25 INFO mapreduce.Job:  map 36% reduce 0%
21/04/26 20:24:27 INFO mapreduce.Job:  map 39% reduce 0%
21/04/26 20:24:28 INFO mapreduce.Job:  map 40% reduce 0%
21/04/26 20:24:31 INFO mapreduce.Job:  map 43% reduce 0%
21/04/26 20:24:32 INFO mapreduce.Job:  map 45% reduce 0%
21/04/26 20:24:34 INFO mapreduce.Job:  map 47% reduce 0%
21/04/26 20:24:35 INFO mapreduce.Job:  map 48% reduce 0%
21/04/26 20:24:37 INFO mapreduce.Job:  map 52% reduce 0%
21/04/26 20:24:39 INFO mapreduce.Job:  map 53% reduce 0%
21/04/26 20:24:40 INFO mapreduce.Job:  map 55% reduce 0%
21/04/26 20:24:42 INFO mapreduce.Job:  map 56% reduce 0%
21/04/26 20:24:43 INFO mapreduce.Job:  map 58% reduce 0%
21/04/26 20:24:45 INFO mapreduce.Job:  map 60% reduce 0%
21/04/26 20:24:46 INFO mapreduce.Job:  map 64% reduce 0%
21/04/26 20:24:49 INFO mapreduce.Job:  map 65% reduce 0%
21/04/26 20:24:52 INFO mapreduce.Job:  map 68% reduce 0%
21/04/26 20:24:54 INFO mapreduce.Job:  map 69% reduce 0%
21/04/26 20:24:55 INFO mapreduce.Job:  map 71% reduce 0%
21/04/26 20:24:57 INFO mapreduce.Job:  map 73% reduce 0%
21/04/26 20:24:58 INFO mapreduce.Job:  map 75% reduce 0%
21/04/26 20:25:01 INFO mapreduce.Job:  map 77% reduce 0%
21/04/26 20:25:02 INFO mapreduce.Job:  map 80% reduce 0%
21/04/26 20:25:04 INFO mapreduce.Job:  map 81% reduce 0%
21/04/26 20:25:05 INFO mapreduce.Job:  map 85% reduce 0%
21/04/26 20:25:07 INFO mapreduce.Job:  map 88% reduce 0%
21/04/26 20:25:08 INFO mapreduce.Job:  map 89% reduce 0%
21/04/26 20:25:10 INFO mapreduce.Job:  map 91% reduce 0%
21/04/26 20:25:11 INFO mapreduce.Job:  map 93% reduce 0%
21/04/26 20:25:13 INFO mapreduce.Job:  map 95% reduce 0%
21/04/26 20:25:14 INFO mapreduce.Job:  map 98% reduce 0%
21/04/26 20:25:17 INFO mapreduce.Job:  map 100% reduce 0%
21/04/26 20:25:23 INFO mapreduce.Job:  map 100% reduce 11%
21/04/26 20:25:26 INFO mapreduce.Job:  map 100% reduce 22%
21/04/26 20:25:29 INFO mapreduce.Job:  map 100% reduce 67%
21/04/26 20:25:32 INFO mapreduce.Job:  map 100% reduce 69%
21/04/26 20:25:35 INFO mapreduce.Job:  map 100% reduce 71%
21/04/26 20:25:38 INFO mapreduce.Job:  map 100% reduce 74%
21/04/26 20:25:41 INFO mapreduce.Job:  map 100% reduce 77%
21/04/26 20:25:44 INFO mapreduce.Job:  map 100% reduce 80%
21/04/26 20:25:47 INFO mapreduce.Job:  map 100% reduce 83%
21/04/26 20:25:50 INFO mapreduce.Job:  map 100% reduce 85%
21/04/26 20:25:53 INFO mapreduce.Job:  map 100% reduce 88%
21/04/26 20:25:56 INFO mapreduce.Job:  map 100% reduce 90%
21/04/26 20:25:59 INFO mapreduce.Job:  map 100% reduce 92%
21/04/26 20:26:02 INFO mapreduce.Job:  map 100% reduce 95%
21/04/26 20:26:05 INFO mapreduce.Job:  map 100% reduce 98%
21/04/26 20:26:08 INFO mapreduce.Job:  map 100% reduce 100%
21/04/26 20:26:08 INFO mapreduce.Job: Job job_1619438325526_0002 completed successfully
21/04/26 20:26:09 INFO mapreduce.Job: Counters: 49
        File System Counters
                FILE: Number of bytes read=1679368168
                FILE: Number of bytes written=2519541603
                FILE: Number of read operations=0
                FILE: Number of large read operations=0
                FILE: Number of write operations=0
                HDFS: Number of bytes read=380018507
                HDFS: Number of bytes written=82
                HDFS: Number of read operations=12
                HDFS: Number of large read operations=0
                HDFS: Number of write operations=2
        Job Counters 
                Launched map tasks=3
                Launched reduce tasks=1
                Data-local map tasks=3
                Total time spent by all maps in occupied slots (ms)=277120
                Total time spent by all reduces in occupied slots (ms)=55989
                Total time spent by all map tasks (ms)=277120
                Total time spent by all reduce tasks (ms)=55989
                Total vcore-milliseconds taken by all map tasks=277120
                Total vcore-milliseconds taken by all reduce tasks=55989
                Total megabyte-milliseconds taken by all map tasks=283770880
                Total megabyte-milliseconds taken by all reduce tasks=57332736
        Map-Reduce Framework
                Map input records=33308000
                Map output records=79388000
                Map output bytes=680908000
                Map output materialized bytes=839684018
                Input split bytes=315
                Combine input records=0
                Combine output records=0
                Reduce input groups=6
                Reduce shuffle bytes=839684018
                Reduce input records=79388000
                Reduce output records=6
                Spilled Records=238164000
                Shuffled Maps =3
                Failed Shuffles=0
                Merged Map outputs=3
                GC time elapsed (ms)=8971
                CPU time spent (ms)=212670
                Physical memory (bytes) snapshot=1000062976
                Virtual memory (bytes) snapshot=8431161344
                Total committed heap usage (bytes)=731906048
        Shuffle Errors
                BAD_ID=0
                CONNECTION=0
                IO_ERROR=0
                WRONG_LENGTH=0
                WRONG_MAP=0
                WRONG_REDUCE=0
        File Input Format Counters 
                Bytes Read=380018192
        File Output Format Counters 
                Bytes Written=82
status = 0

笔者的老爷机很吃力呢：

在node1:50070可以看到：

可以看出，自己写的wordcount程序计算出了正确的结果。和官方自带的示例程序相比，可以删除原有目录。

在node3:8088还可以看到提交过的MapReduce任务。

参照HDFS集群关机顺序关闭集群。

关闭集群

参照关闭集群的顺序。
node1关闭HDFS：

stop-dfs.sh

关闭YARN：

stop-yarn.sh

3台机关闭ZooKeeper：

cd /export/server/zookeeper-3.4.6/
bin/zkServer.sh stop

3台机断电：

poweroff

本地运行

扔集群运行还是比较麻烦的，好在Java可以跨平台，在本地运行也可以测试代码的正确性。

package com.aa.wordcount_local;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import java.io.IOException;

public class WordCountLocal extends Configured implements Tool {
     

    @Override
    public int run(String[] args) throws Exception {
     
        //构建
        Job job = Job.getInstance(this.getConf(), "userwc_local");
        job.setJarByClass(WordCountLocal.class);

        //配置
        job.setInputFormatClass(TextInputFormat.class);
        //使用程序的第一个参数作为输入
        TextInputFormat.setInputPaths(job, new Path(args[0]));

        job.setMapperClass(WCMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        job.setReducerClass(WCReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        job.setOutputFormatClass(TextOutputFormat.class);
        //使用程序的第二个参数作为输出路径
        Path outputPath = new Path(args[1]);
        FileSystem fs = FileSystem.get(this.getConf());
        if (fs.exists(outputPath)) {
     
            fs.delete(outputPath, true);//true为递归删除
        }
        TextOutputFormat.setOutputPath(job, outputPath);

        return job.waitForCompletion(true) ? 0 : -1;
    }

    public static void main(String[] args) throws Exception {
     
        Configuration conf = new Configuration();
        int status = ToolRunner.run(conf, new WordCountLocal(), args);
        System.out.println("status = " + status);
        System.exit(status);
    }

    public static class WCMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
     
        //输出的K2
        Text outputKey = new Text();
        //输出的V2
        IntWritable outputValue = new IntWritable(1);//由于是平权的，使之恒=1便可以计数

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
     
            //将每行的内容分割得到每个单词
            String[] words = value.toString().split("\\s+");
            //迭代取出每个单词作为K2
            for (String word : words) {
     
                //将当前的单词作为K2
                this.outputKey.set(word);
                //将K2和V2传递到下一步
                context.write(outputKey,outputValue);
            }
        }
    }

    public static class WCReducer extends Reducer<Text,IntWritable,Text,IntWritable>{
     
        //K3=K2，不需要重复定义变量
        //输出V3
        IntWritable outputValue = new IntWritable();

        @Override
        protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
     
            int sum=0;
            for (IntWritable value : values) {
     
                sum+=value.get();
            }
            //给V3赋值
            this.outputValue.set(sum);
            //传递到下一步
            context.write(key,this.outputValue);
        }
    }

}

当放到同一个类中时，Mapper和Reducer必须申明为static（别问我为神马）。

本地部署

Hadoop本地环境变量配置
配置成功后可以在本地运行。idea中进行如下设置：

Run→Edit →Configus

修改内容为：

com.aa.wordcount_local.WordCountLocal

传递的2个路径变量使用空格隔开：

E:\bigdata\2021.4.26\wc.txt E:\bigdata\2021.4.26\result

Apply之后即可直接运行。

运行过程：

"C:\Program Files\Java\jdk1.8.0_241\bin\java.exe" "-javaagent:C:\Program Files\JetBrains\IntelliJ IDEA 2019.3.5\lib\idea_rt.jar=59719:C:\Program Files\JetBrains\IntelliJ IDEA 2019.3.5\bin" -Dfile.encoding=UTF-8 -classpath "C:\Program Files\Java\jdk1.8.0_241\jre\lib\charsets.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\deploy.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\access-bridge-64.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\cldrdata.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\dnsns.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\jaccess.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\jfxrt.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\localedata.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\nashorn.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\sunec.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\sunjce_provider.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\sunmscapi.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\sunpkcs11.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\ext\zipfs.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\javaws.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\jce.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\jfr.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\jfxswt.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\jsse.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\management-agent.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\plugin.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\resources.jar;C:\Program Files\Java\jdk1.8.0_241\jre\lib\rt.jar;C:\Users\killer\IdeaProjects\bigdata_demo\day210426\target\classes;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\hadoop\hadoop-common\2.7.5\hadoop-common-2.7.5.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\hadoop\hadoop-annotations\2.7.5\hadoop-annotations-2.7.5.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\com\google\guava\guava\11.0.2\guava-11.0.2.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\commons-cli\commons-cli\1.2\commons-cli-1.2.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\commons\commons-math3\3.1.1\commons-math3-3.1.1.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\xmlenc\xmlenc\0.52\xmlenc-0.52.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\commons-httpclient\commons-httpclient\3.1\commons-httpclient-3.1.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\commons-codec\commons-codec\1.4\commons-codec-1.4.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\commons-io\commons-io\2.4\commons-io-2.4.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\commons-net\commons-net\3.1\commons-net-3.1.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\commons-collections\commons-collections\3.2.2\commons-collections-3.2.2.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\javax\servlet\servlet-api\2.5\servlet-api-2.5.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\mortbay\jetty\jetty\6.1.26\jetty-6.1.26.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\mortbay\jetty\jetty-util\6.1.26\jetty-util-6.1.26.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\mortbay\jetty\jetty-sslengine\6.1.26\jetty-sslengine-6.1.26.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\javax\servlet\jsp\jsp-api\2.1\jsp-api-2.1.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\com\sun\jersey\jersey-core\1.9\jersey-core-1.9.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\com\sun\jersey\jersey-json\1.9\jersey-json-1.9.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\codehaus\jettison\jettison\1.1\jettison-1.1.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\com\sun\xml\bind\jaxb-impl\2.2.3-1\jaxb-impl-2.2.3-1.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\codehaus\jackson\jackson-jaxrs\1.8.3\jackson-jaxrs-1.8.3.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\codehaus\jackson\jackson-xc\1.8.3\jackson-xc-1.8.3.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\com\sun\jersey\jersey-server\1.9\jersey-server-1.9.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\asm\asm\3.1\asm-3.1.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\commons-logging\commons-logging\1.1.3\commons-logging-1.1.3.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\log4j\log4j\1.2.17\log4j-1.2.17.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\net\java\dev\jets3t\jets3t\0.9.0\jets3t-0.9.0.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\httpcomponents\httpclient\4.1.2\httpclient-4.1.2.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\httpcomponents\httpcore\4.1.2\httpcore-4.1.2.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\com\jamesmurty\utils\java-xmlbuilder\0.4\java-xmlbuilder-0.4.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\commons-lang\commons-lang\2.6\commons-lang-2.6.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\commons-configuration\commons-configuration\1.6\commons-configuration-1.6.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\commons-digester\commons-digester\1.8\commons-digester-1.8.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\commons-beanutils\commons-beanutils\1.7.0\commons-beanutils-1.7.0.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\commons-beanutils\commons-beanutils-core\1.8.0\commons-beanutils-core-1.8.0.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\slf4j\slf4j-api\1.7.10\slf4j-api-1.7.10.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\slf4j\slf4j-log4j12\1.7.10\slf4j-log4j12-1.7.10.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\codehaus\jackson\jackson-core-asl\1.9.13\jackson-core-asl-1.9.13.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\codehaus\jackson\jackson-mapper-asl\1.9.13\jackson-mapper-asl-1.9.13.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\avro\avro\1.7.4\avro-1.7.4.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\com\thoughtworks\paranamer\paranamer\2.3\paranamer-2.3.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\xerial\snappy\snappy-java\1.0.4.1\snappy-java-1.0.4.1.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\com\google\protobuf\protobuf-java\2.5.0\protobuf-java-2.5.0.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\com\google\code\gson\gson\2.2.4\gson-2.2.4.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\hadoop\hadoop-auth\2.7.5\hadoop-auth-2.7.5.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\directory\server\apacheds-kerberos-codec\2.0.0-M15\apacheds-kerberos-codec-2.0.0-M15.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\directory\server\apacheds-i18n\2.0.0-M15\apacheds-i18n-2.0.0-M15.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\directory\api\api-asn1-api\1.0.0-M20\api-asn1-api-1.0.0-M20.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\directory\api\api-util\1.0.0-M20\api-util-1.0.0-M20.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\curator\curator-framework\2.7.1\curator-framework-2.7.1.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\com\jcraft\jsch\0.1.54\jsch-0.1.54.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\curator\curator-client\2.7.1\curator-client-2.7.1.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\curator\curator-recipes\2.7.1\curator-recipes-2.7.1.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\com\google\code\findbugs\jsr305\3.0.0\jsr305-3.0.0.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\htrace\htrace-core\3.1.0-incubating\htrace-core-3.1.0-incubating.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\zookeeper\zookeeper\3.4.6\zookeeper-3.4.6.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\commons\commons-compress\1.4.1\commons-compress-1.4.1.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\tukaani\xz\1.0\xz-1.0.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\hadoop\hadoop-client\2.7.5\hadoop-client-2.7.5.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\hadoop\hadoop-mapreduce-client-app\2.7.5\hadoop-mapreduce-client-app-2.7.5.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\hadoop\hadoop-mapreduce-client-common\2.7.5\hadoop-mapreduce-client-common-2.7.5.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\hadoop\hadoop-yarn-client\2.7.5\hadoop-yarn-client-2.7.5.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\hadoop\hadoop-yarn-server-common\2.7.5\hadoop-yarn-server-common-2.7.5.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\hadoop\hadoop-mapreduce-client-shuffle\2.7.5\hadoop-mapreduce-client-shuffle-2.7.5.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\hadoop\hadoop-yarn-api\2.7.5\hadoop-yarn-api-2.7.5.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\hadoop\hadoop-mapreduce-client-jobclient\2.7.5\hadoop-mapreduce-client-jobclient-2.7.5.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\hadoop\hadoop-hdfs\2.7.5\hadoop-hdfs-2.7.5.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\commons-daemon\commons-daemon\1.0.13\commons-daemon-1.0.13.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\io\netty\netty\3.6.2.Final\netty-3.6.2.Final.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\io\netty\netty-all\4.0.23.Final\netty-all-4.0.23.Final.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\xerces\xercesImpl\2.9.1\xercesImpl-2.9.1.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\xml-apis\xml-apis\1.3.04\xml-apis-1.3.04.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\fusesource\leveldbjni\leveldbjni-all\1.8\leveldbjni-all-1.8.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\hadoop\hadoop-mapreduce-client-core\2.7.5\hadoop-mapreduce-client-core-2.7.5.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\apache\hadoop\hadoop-yarn-common\2.7.5\hadoop-yarn-common-2.7.5.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\javax\xml\bind\jaxb-api\2.2.2\jaxb-api-2.2.2.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\javax\xml\stream\stax-api\1.0-2\stax-api-1.0-2.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\javax\activation\activation\1.1\activation-1.1.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\com\sun\jersey\jersey-client\1.9\jersey-client-1.9.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\com\google\inject\guice\3.0\guice-3.0.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\javax\inject\javax.inject\1\javax.inject-1.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\aopalliance\aopalliance\1.0\aopalliance-1.0.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\com\sun\jersey\contribs\jersey-guice\1.9\jersey-guice-1.9.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\com\google\inject\extensions\guice-servlet\3.0\guice-servlet-3.0.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\junit\junit\4.13\junit-4.13.jar;C:\Program Files\apache-maven-3.3.9\Maven_Repository\org\hamcrest\hamcrest-core\1.3\hamcrest-core-1.3.jar" com.aa.wordcount_local.WordCountLocal E:\bigdata\2021.4.26\wc.txt E:\bigdata\2021.4.26\result
21/04/26 21:26:22 INFO Configuration.deprecation: session.id is deprecated. Instead, use dfs.metrics.session-id
21/04/26 21:26:22 INFO jvm.JvmMetrics: Initializing JVM Metrics with processName=JobTracker, sessionId=
21/04/26 21:26:24 WARN mapreduce.JobResourceUploader: No job jar file set.  User classes may not be found. See Job or Job#setJar(String).
21/04/26 21:26:24 INFO input.FileInputFormat: Total input paths to process : 1
21/04/26 21:26:24 INFO mapreduce.JobSubmitter: number of splits:12
21/04/26 21:26:24 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_local588307001_0001
21/04/26 21:26:24 INFO mapreduce.Job: The url to track the job: http://localhost:8080/
21/04/26 21:26:24 INFO mapreduce.Job: Running job: job_local588307001_0001
21/04/26 21:26:24 INFO mapred.LocalJobRunner: OutputCommitter set in config null
21/04/26 21:26:24 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 21:26:24 INFO mapred.LocalJobRunner: OutputCommitter is org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
21/04/26 21:26:24 INFO mapred.LocalJobRunner: Waiting for map tasks
21/04/26 21:26:24 INFO mapred.LocalJobRunner: Starting task: attempt_local588307001_0001_m_000000_0
21/04/26 21:26:24 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 21:26:24 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
21/04/26 21:26:24 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@46db6474
21/04/26 21:26:24 INFO mapred.MapTask: Processing split: file:/E:/bigdata/2021.4.26/wc.txt:0+33554432
21/04/26 21:26:24 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
21/04/26 21:26:24 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
21/04/26 21:26:24 INFO mapred.MapTask: soft limit at 83886080
21/04/26 21:26:24 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
21/04/26 21:26:24 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
21/04/26 21:26:24 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
21/04/26 21:26:25 INFO mapreduce.Job: Job job_local588307001_0001 running in uber mode : false
21/04/26 21:26:25 INFO mapreduce.Job:  map 0% reduce 0%
21/04/26 21:26:27 INFO mapred.MapTask: Spilling map output
21/04/26 21:26:27 INFO mapred.MapTask: bufstart = 0; bufend = 29274899; bufvoid = 104857600
21/04/26 21:26:27 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 12561604(50246416); length = 13652793/6553600
21/04/26 21:26:27 INFO mapred.MapTask: (EQUATOR) 39760649 kvi 9940156(39760624)
21/04/26 21:26:28 INFO mapred.MapTask: Finished spill 0
21/04/26 21:26:28 INFO mapred.MapTask: (RESET) equator 39760649 kv 9940156(39760624) kvi 7318732(29274928)
21/04/26 21:26:30 INFO mapred.MapTask: Spilling map output
21/04/26 21:26:30 INFO mapred.MapTask: bufstart = 39760649; bufend = 69035531; bufvoid = 104857600
21/04/26 21:26:30 INFO mapred.MapTask: kvstart = 9940156(39760624); kvend = 22501764(90007056); length = 13652793/6553600
21/04/26 21:26:30 INFO mapred.MapTask: (EQUATOR) 79521285 kvi 19880316(79521264)
21/04/26 21:26:30 INFO mapred.LocalJobRunner: map > map
21/04/26 21:26:30 INFO mapred.LocalJobRunner: map > map
21/04/26 21:26:30 INFO mapred.MapTask: Starting flush of map output
21/04/26 21:26:31 INFO mapreduce.Job:  map 6% reduce 0%
21/04/26 21:26:32 INFO mapred.MapTask: Finished spill 1
21/04/26 21:26:32 INFO mapred.MapTask: (RESET) equator 79521285 kv 19880316(79521264) kvi 19146444(76585776)
21/04/26 21:26:32 INFO mapred.MapTask: Spilling map output
21/04/26 21:26:32 INFO mapred.MapTask: bufstart = 79521285; bufend = 81094879; bufvoid = 104857600
21/04/26 21:26:32 INFO mapred.MapTask: kvstart = 19880316(79521264); kvend = 19146448(76585792); length = 733869/6553600
21/04/26 21:26:32 INFO mapred.MapTask: Finished spill 2
21/04/26 21:26:32 INFO mapred.Merger: Merging 3 sorted segments
21/04/26 21:26:32 INFO mapred.Merger: Down to the last merge-pass, with 3 segments left of total size: 74143116 bytes
21/04/26 21:26:33 INFO mapred.LocalJobRunner: map > sort > 
21/04/26 21:26:36 INFO mapred.Task: Task:attempt_local588307001_0001_m_000000_0 is done. And is in the process of committing
21/04/26 21:26:36 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:26:36 INFO mapred.Task: Task 'attempt_local588307001_0001_m_000000_0' done.
21/04/26 21:26:36 INFO mapred.Task: Final Counters for attempt_local588307001_0001_m_000000_0: Counters: 17
	File System Counters
		FILE: Number of bytes read=107703100
		FILE: Number of bytes written=148580113
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Map input records=2941057
		Map output records=7009866
		Map output bytes=60123375
		Map output materialized bytes=74143113
		Input split bytes=98
		Combine input records=0
		Spilled Records=14019732
		Failed Shuffles=0
		Merged Map outputs=0
		GC time elapsed (ms)=55
		Total committed heap usage (bytes)=1497366528
	File Input Format Counters 
		Bytes Read=33558528
21/04/26 21:26:36 INFO mapred.LocalJobRunner: Finishing task: attempt_local588307001_0001_m_000000_0
21/04/26 21:26:36 INFO mapred.LocalJobRunner: Starting task: attempt_local588307001_0001_m_000001_0
21/04/26 21:26:36 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 21:26:36 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
21/04/26 21:26:36 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@30b25df7
21/04/26 21:26:36 INFO mapred.MapTask: Processing split: file:/E:/bigdata/2021.4.26/wc.txt:33554432+33554432
21/04/26 21:26:36 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
21/04/26 21:26:36 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
21/04/26 21:26:36 INFO mapred.MapTask: soft limit at 83886080
21/04/26 21:26:36 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
21/04/26 21:26:36 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
21/04/26 21:26:36 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
21/04/26 21:26:36 INFO mapreduce.Job:  map 100% reduce 0%
21/04/26 21:26:38 INFO mapred.MapTask: Spilling map output
21/04/26 21:26:38 INFO mapred.MapTask: bufstart = 0; bufend = 29274882; bufvoid = 104857600
21/04/26 21:26:38 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 12561604(50246416); length = 13652793/6553600
21/04/26 21:26:38 INFO mapred.MapTask: (EQUATOR) 39760641 kvi 9940156(39760624)
21/04/26 21:26:39 INFO mapred.MapTask: Finished spill 0
21/04/26 21:26:39 INFO mapred.MapTask: (RESET) equator 39760641 kv 9940156(39760624) kvi 7318728(29274912)
21/04/26 21:26:41 INFO mapred.MapTask: Spilling map output
21/04/26 21:26:41 INFO mapred.MapTask: bufstart = 39760641; bufend = 69035537; bufvoid = 104857600
21/04/26 21:26:41 INFO mapred.MapTask: kvstart = 9940156(39760624); kvend = 22501764(90007056); length = 13652793/6553600
21/04/26 21:26:41 INFO mapred.MapTask: (EQUATOR) 79521288 kvi 19880316(79521264)
21/04/26 21:26:41 INFO mapred.LocalJobRunner: 
21/04/26 21:26:41 INFO mapred.MapTask: Starting flush of map output
21/04/26 21:26:41 INFO mapreduce.Job:  map 8% reduce 0%
21/04/26 21:26:42 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:26:42 INFO mapreduce.Job:  map 14% reduce 0%
21/04/26 21:26:42 INFO mapred.MapTask: Finished spill 1
21/04/26 21:26:42 INFO mapred.MapTask: (RESET) equator 79521288 kv 19880316(79521264) kvi 19146444(76585776)
21/04/26 21:26:42 INFO mapred.MapTask: Spilling map output
21/04/26 21:26:42 INFO mapred.MapTask: bufstart = 79521288; bufend = 81094874; bufvoid = 104857600
21/04/26 21:26:42 INFO mapred.MapTask: kvstart = 19880316(79521264); kvend = 19146448(76585792); length = 733869/6553600
21/04/26 21:26:42 INFO mapred.MapTask: Finished spill 2
21/04/26 21:26:42 INFO mapred.Merger: Merging 3 sorted segments
21/04/26 21:26:42 INFO mapred.Merger: Down to the last merge-pass, with 3 segments left of total size: 74143105 bytes
21/04/26 21:26:45 INFO mapred.LocalJobRunner: map > sort > 
21/04/26 21:26:45 INFO mapreduce.Job:  map 16% reduce 0%
21/04/26 21:26:46 INFO mapred.Task: Task:attempt_local588307001_0001_m_000001_0 is done. And is in the process of committing
21/04/26 21:26:46 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:26:46 INFO mapred.Task: Task 'attempt_local588307001_0001_m_000001_0' done.
21/04/26 21:26:46 INFO mapred.Task: Final Counters for attempt_local588307001_0001_m_000001_0: Counters: 17
	File System Counters
		FILE: Number of bytes read=215405945
		FILE: Number of bytes written=296866361
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Map input records=2941058
		Map output records=7009866
		Map output bytes=60123364
		Map output materialized bytes=74143102
		Input split bytes=98
		Combine input records=0
		Spilled Records=14019732
		Failed Shuffles=0
		Merged Map outputs=0
		GC time elapsed (ms)=115
		Total committed heap usage (bytes)=1507852288
	File Input Format Counters 
		Bytes Read=33558528
21/04/26 21:26:46 INFO mapred.LocalJobRunner: Finishing task: attempt_local588307001_0001_m_000001_0
21/04/26 21:26:46 INFO mapred.LocalJobRunner: Starting task: attempt_local588307001_0001_m_000002_0
21/04/26 21:26:46 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 21:26:46 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
21/04/26 21:26:46 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@5a8242f3
21/04/26 21:26:46 INFO mapred.MapTask: Processing split: file:/E:/bigdata/2021.4.26/wc.txt:67108864+33554432
21/04/26 21:26:46 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
21/04/26 21:26:46 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
21/04/26 21:26:46 INFO mapred.MapTask: soft limit at 83886080
21/04/26 21:26:46 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
21/04/26 21:26:46 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
21/04/26 21:26:46 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
21/04/26 21:26:46 INFO mapreduce.Job:  map 100% reduce 0%
21/04/26 21:26:48 INFO mapred.MapTask: Spilling map output
21/04/26 21:26:48 INFO mapred.MapTask: bufstart = 0; bufend = 29274882; bufvoid = 104857600
21/04/26 21:26:48 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 12561604(50246416); length = 13652793/6553600
21/04/26 21:26:48 INFO mapred.MapTask: (EQUATOR) 39760641 kvi 9940156(39760624)
21/04/26 21:26:49 INFO mapred.MapTask: Finished spill 0
21/04/26 21:26:49 INFO mapred.MapTask: (RESET) equator 39760641 kv 9940156(39760624) kvi 7318728(29274912)
21/04/26 21:26:51 INFO mapred.MapTask: Spilling map output
21/04/26 21:26:51 INFO mapred.MapTask: bufstart = 39760641; bufend = 69035523; bufvoid = 104857600
21/04/26 21:26:51 INFO mapred.MapTask: kvstart = 9940156(39760624); kvend = 22501764(90007056); length = 13652793/6553600
21/04/26 21:26:51 INFO mapred.MapTask: (EQUATOR) 79521281 kvi 19880316(79521264)
21/04/26 21:26:51 INFO mapred.LocalJobRunner: 
21/04/26 21:26:51 INFO mapred.MapTask: Starting flush of map output
21/04/26 21:26:51 INFO mapreduce.Job:  map 17% reduce 0%
21/04/26 21:26:52 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:26:52 INFO mapreduce.Job:  map 22% reduce 0%
21/04/26 21:26:52 INFO mapred.MapTask: Finished spill 1
21/04/26 21:26:52 INFO mapred.MapTask: (RESET) equator 79521281 kv 19880316(79521264) kvi 19146448(76585792)
21/04/26 21:26:52 INFO mapred.MapTask: Spilling map output
21/04/26 21:26:52 INFO mapred.MapTask: bufstart = 79521281; bufend = 81094884; bufvoid = 104857600
21/04/26 21:26:52 INFO mapred.MapTask: kvstart = 19880316(79521264); kvend = 19146452(76585808); length = 733865/6553600
21/04/26 21:26:53 INFO mapred.MapTask: Finished spill 2
21/04/26 21:26:53 INFO mapred.Merger: Merging 3 sorted segments
21/04/26 21:26:53 INFO mapred.Merger: Down to the last merge-pass, with 3 segments left of total size: 74143106 bytes
21/04/26 21:26:55 INFO mapred.LocalJobRunner: map > sort > 
21/04/26 21:26:55 INFO mapreduce.Job:  map 24% reduce 0%
21/04/26 21:26:56 INFO mapred.Task: Task:attempt_local588307001_0001_m_000002_0 is done. And is in the process of committing
21/04/26 21:26:56 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:26:56 INFO mapred.Task: Task 'attempt_local588307001_0001_m_000002_0' done.
21/04/26 21:26:56 INFO mapred.Task: Final Counters for attempt_local588307001_0001_m_000002_0: Counters: 17
	File System Counters
		FILE: Number of bytes read=323108791
		FILE: Number of bytes written=445152611
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Map input records=2941054
		Map output records=7009865
		Map output bytes=60123367
		Map output materialized bytes=74143103
		Input split bytes=98
		Combine input records=0
		Spilled Records=14019730
		Failed Shuffles=0
		Merged Map outputs=0
		GC time elapsed (ms)=77
		Total committed heap usage (bytes)=1313865728
	File Input Format Counters 
		Bytes Read=33558528
21/04/26 21:26:56 INFO mapred.LocalJobRunner: Finishing task: attempt_local588307001_0001_m_000002_0
21/04/26 21:26:56 INFO mapred.LocalJobRunner: Starting task: attempt_local588307001_0001_m_000003_0
21/04/26 21:26:56 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 21:26:56 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
21/04/26 21:26:56 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@be049c8
21/04/26 21:26:56 INFO mapred.MapTask: Processing split: file:/E:/bigdata/2021.4.26/wc.txt:100663296+33554432
21/04/26 21:26:56 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
21/04/26 21:26:56 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
21/04/26 21:26:56 INFO mapred.MapTask: soft limit at 83886080
21/04/26 21:26:56 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
21/04/26 21:26:56 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
21/04/26 21:26:56 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
21/04/26 21:26:57 INFO mapreduce.Job:  map 100% reduce 0%
21/04/26 21:26:58 INFO mapred.MapTask: Spilling map output
21/04/26 21:26:58 INFO mapred.MapTask: bufstart = 0; bufend = 29274885; bufvoid = 104857600
21/04/26 21:26:58 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 12561600(50246400); length = 13652797/6553600
21/04/26 21:26:58 INFO mapred.MapTask: (EQUATOR) 39760634 kvi 9940152(39760608)
21/04/26 21:27:00 INFO mapred.MapTask: Finished spill 0
21/04/26 21:27:00 INFO mapred.MapTask: (RESET) equator 39760634 kv 9940152(39760608) kvi 7318728(29274912)
21/04/26 21:27:02 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:02 INFO mapred.MapTask: bufstart = 39760634; bufend = 69035520; bufvoid = 104857600
21/04/26 21:27:02 INFO mapred.MapTask: kvstart = 9940152(39760608); kvend = 22501760(90007040); length = 13652793/6553600
21/04/26 21:27:02 INFO mapred.MapTask: (EQUATOR) 79521272 kvi 19880312(79521248)
21/04/26 21:27:02 INFO mapred.LocalJobRunner: 
21/04/26 21:27:02 INFO mapred.MapTask: Starting flush of map output
21/04/26 21:27:02 INFO mapreduce.Job:  map 25% reduce 0%
21/04/26 21:27:02 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:27:03 INFO mapreduce.Job:  map 31% reduce 0%
21/04/26 21:27:03 INFO mapred.MapTask: Finished spill 1
21/04/26 21:27:03 INFO mapred.MapTask: (RESET) equator 79521272 kv 19880312(79521248) kvi 19146444(76585776)
21/04/26 21:27:03 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:03 INFO mapred.MapTask: bufstart = 79521272; bufend = 81094865; bufvoid = 104857600
21/04/26 21:27:03 INFO mapred.MapTask: kvstart = 19880312(79521248); kvend = 19146448(76585792); length = 733865/6553600
21/04/26 21:27:03 INFO mapred.MapTask: Finished spill 2
21/04/26 21:27:03 INFO mapred.Merger: Merging 3 sorted segments
21/04/26 21:27:03 INFO mapred.Merger: Down to the last merge-pass, with 3 segments left of total size: 74143105 bytes
21/04/26 21:27:05 INFO mapred.LocalJobRunner: map > sort > 
21/04/26 21:27:06 INFO mapreduce.Job:  map 32% reduce 0%
21/04/26 21:27:08 INFO mapred.Task: Task:attempt_local588307001_0001_m_000003_0 is done. And is in the process of committing
21/04/26 21:27:08 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:27:08 INFO mapred.Task: Task 'attempt_local588307001_0001_m_000003_0' done.
21/04/26 21:27:08 INFO mapred.Task: Final Counters for attempt_local588307001_0001_m_000003_0: Counters: 17
	File System Counters
		FILE: Number of bytes read=430811636
		FILE: Number of bytes written=593438859
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Map input records=2941058
		Map output records=7009866
		Map output bytes=60123364
		Map output materialized bytes=74143102
		Input split bytes=98
		Combine input records=0
		Spilled Records=14019732
		Failed Shuffles=0
		Merged Map outputs=0
		GC time elapsed (ms)=67
		Total committed heap usage (bytes)=1136132096
	File Input Format Counters 
		Bytes Read=33558528
21/04/26 21:27:08 INFO mapred.LocalJobRunner: Finishing task: attempt_local588307001_0001_m_000003_0
21/04/26 21:27:08 INFO mapred.LocalJobRunner: Starting task: attempt_local588307001_0001_m_000004_0
21/04/26 21:27:08 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 21:27:08 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
21/04/26 21:27:08 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@6918296b
21/04/26 21:27:08 INFO mapred.MapTask: Processing split: file:/E:/bigdata/2021.4.26/wc.txt:134217728+33554432
21/04/26 21:27:08 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
21/04/26 21:27:08 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
21/04/26 21:27:08 INFO mapred.MapTask: soft limit at 83886080
21/04/26 21:27:08 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
21/04/26 21:27:08 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
21/04/26 21:27:08 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
21/04/26 21:27:08 INFO mapreduce.Job:  map 100% reduce 0%
21/04/26 21:27:10 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:10 INFO mapred.MapTask: bufstart = 0; bufend = 29274888; bufvoid = 104857600
21/04/26 21:27:10 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 12561600(50246400); length = 13652797/6553600
21/04/26 21:27:10 INFO mapred.MapTask: (EQUATOR) 39760636 kvi 9940152(39760608)
21/04/26 21:27:11 INFO mapred.MapTask: Finished spill 0
21/04/26 21:27:11 INFO mapred.MapTask: (RESET) equator 39760636 kv 9940152(39760608) kvi 7318728(29274912)
21/04/26 21:27:13 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:13 INFO mapred.MapTask: bufstart = 39760636; bufend = 69035514; bufvoid = 104857600
21/04/26 21:27:13 INFO mapred.MapTask: kvstart = 9940152(39760608); kvend = 22501760(90007040); length = 13652793/6553600
21/04/26 21:27:13 INFO mapred.MapTask: (EQUATOR) 79521269 kvi 19880312(79521248)
21/04/26 21:27:13 INFO mapred.LocalJobRunner: 
21/04/26 21:27:13 INFO mapred.MapTask: Starting flush of map output
21/04/26 21:27:13 INFO mapreduce.Job:  map 33% reduce 0%
21/04/26 21:27:14 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:27:14 INFO mapreduce.Job:  map 39% reduce 0%
21/04/26 21:27:14 INFO mapred.MapTask: Finished spill 1
21/04/26 21:27:14 INFO mapred.MapTask: (RESET) equator 79521269 kv 19880312(79521248) kvi 19146428(76585712)
21/04/26 21:27:14 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:14 INFO mapred.MapTask: bufstart = 79521269; bufend = 81094906; bufvoid = 104857600
21/04/26 21:27:14 INFO mapred.MapTask: kvstart = 19880312(79521248); kvend = 19146432(76585728); length = 733881/6553600
21/04/26 21:27:14 INFO mapred.MapTask: Finished spill 2
21/04/26 21:27:14 INFO mapred.Merger: Merging 3 sorted segments
21/04/26 21:27:14 INFO mapred.Merger: Down to the last merge-pass, with 3 segments left of total size: 74143152 bytes
21/04/26 21:27:17 INFO mapred.LocalJobRunner: map > sort > 
21/04/26 21:27:17 INFO mapreduce.Job:  map 41% reduce 0%
21/04/26 21:27:17 INFO mapred.Task: Task:attempt_local588307001_0001_m_000004_0 is done. And is in the process of committing
21/04/26 21:27:17 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:27:17 INFO mapred.Task: Task 'attempt_local588307001_0001_m_000004_0' done.
21/04/26 21:27:17 INFO mapred.Task: Final Counters for attempt_local588307001_0001_m_000004_0: Counters: 17
	File System Counters
		FILE: Number of bytes read=538514528
		FILE: Number of bytes written=741725201
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Map input records=2941058
		Map output records=7009870
		Map output bytes=60123403
		Map output materialized bytes=74143149
		Input split bytes=98
		Combine input records=0
		Spilled Records=14019740
		Failed Shuffles=0
		Merged Map outputs=0
		GC time elapsed (ms)=67
		Total committed heap usage (bytes)=953155584
	File Input Format Counters 
		Bytes Read=33558528
21/04/26 21:27:17 INFO mapred.LocalJobRunner: Finishing task: attempt_local588307001_0001_m_000004_0
21/04/26 21:27:17 INFO mapred.LocalJobRunner: Starting task: attempt_local588307001_0001_m_000005_0
21/04/26 21:27:17 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 21:27:17 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
21/04/26 21:27:17 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@4b678c5a
21/04/26 21:27:17 INFO mapred.MapTask: Processing split: file:/E:/bigdata/2021.4.26/wc.txt:167772160+33554432
21/04/26 21:27:17 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
21/04/26 21:27:17 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
21/04/26 21:27:17 INFO mapred.MapTask: soft limit at 83886080
21/04/26 21:27:17 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
21/04/26 21:27:17 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
21/04/26 21:27:17 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
21/04/26 21:27:18 INFO mapreduce.Job:  map 100% reduce 0%
21/04/26 21:27:19 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:19 INFO mapred.MapTask: bufstart = 0; bufend = 29274881; bufvoid = 104857600
21/04/26 21:27:19 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 12561604(50246416); length = 13652793/6553600
21/04/26 21:27:19 INFO mapred.MapTask: (EQUATOR) 39760640 kvi 9940156(39760624)
21/04/26 21:27:21 INFO mapred.MapTask: Finished spill 0
21/04/26 21:27:21 INFO mapred.MapTask: (RESET) equator 39760640 kv 9940156(39760624) kvi 7318728(29274912)
21/04/26 21:27:22 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:22 INFO mapred.MapTask: bufstart = 39760640; bufend = 69035526; bufvoid = 104857600
21/04/26 21:27:22 INFO mapred.MapTask: kvstart = 9940156(39760624); kvend = 22501764(90007056); length = 13652793/6553600
21/04/26 21:27:22 INFO mapred.MapTask: (EQUATOR) 79521283 kvi 19880316(79521264)
21/04/26 21:27:22 INFO mapred.LocalJobRunner: 
21/04/26 21:27:22 INFO mapred.MapTask: Starting flush of map output
21/04/26 21:27:23 INFO mapreduce.Job:  map 42% reduce 0%
21/04/26 21:27:23 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:27:24 INFO mapred.MapTask: Finished spill 1
21/04/26 21:27:24 INFO mapred.MapTask: (RESET) equator 79521283 kv 19880316(79521264) kvi 19146452(76585808)
21/04/26 21:27:24 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:24 INFO mapred.MapTask: bufstart = 79521283; bufend = 81094858; bufvoid = 104857600
21/04/26 21:27:24 INFO mapred.MapTask: kvstart = 19880316(79521264); kvend = 19146456(76585824); length = 733861/6553600
21/04/26 21:27:24 INFO mapred.MapTask: Finished spill 2
21/04/26 21:27:24 INFO mapred.Merger: Merging 3 sorted segments
21/04/26 21:27:24 INFO mapred.Merger: Down to the last merge-pass, with 3 segments left of total size: 74143079 bytes
21/04/26 21:27:24 INFO mapreduce.Job:  map 47% reduce 0%
21/04/26 21:27:26 INFO mapred.LocalJobRunner: map > sort > 
21/04/26 21:27:27 INFO mapred.Task: Task:attempt_local588307001_0001_m_000005_0 is done. And is in the process of committing
21/04/26 21:27:27 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:27:27 INFO mapred.Task: Task 'attempt_local588307001_0001_m_000005_0' done.
21/04/26 21:27:27 INFO mapred.Task: Final Counters for attempt_local588307001_0001_m_000005_0: Counters: 17
	File System Counters
		FILE: Number of bytes read=646217347
		FILE: Number of bytes written=890011397
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Map input records=2941056
		Map output records=7009864
		Map output bytes=60123342
		Map output materialized bytes=74143076
		Input split bytes=98
		Combine input records=0
		Spilled Records=14019728
		Failed Shuffles=0
		Merged Map outputs=0
		GC time elapsed (ms)=75
		Total committed heap usage (bytes)=825753600
	File Input Format Counters 
		Bytes Read=33558528
21/04/26 21:27:27 INFO mapred.LocalJobRunner: Finishing task: attempt_local588307001_0001_m_000005_0
21/04/26 21:27:27 INFO mapred.LocalJobRunner: Starting task: attempt_local588307001_0001_m_000006_0
21/04/26 21:27:27 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 21:27:27 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
21/04/26 21:27:27 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@33ada1f
21/04/26 21:27:27 INFO mapred.MapTask: Processing split: file:/E:/bigdata/2021.4.26/wc.txt:201326592+33554432
21/04/26 21:27:27 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
21/04/26 21:27:27 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
21/04/26 21:27:27 INFO mapred.MapTask: soft limit at 83886080
21/04/26 21:27:27 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
21/04/26 21:27:27 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
21/04/26 21:27:27 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
21/04/26 21:27:27 INFO mapreduce.Job:  map 100% reduce 0%
21/04/26 21:27:29 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:29 INFO mapred.MapTask: bufstart = 0; bufend = 29274896; bufvoid = 104857600
21/04/26 21:27:29 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 12561604(50246416); length = 13652793/6553600
21/04/26 21:27:29 INFO mapred.MapTask: (EQUATOR) 39760648 kvi 9940156(39760624)
21/04/26 21:27:30 INFO mapred.MapTask: Finished spill 0
21/04/26 21:27:30 INFO mapred.MapTask: (RESET) equator 39760648 kv 9940156(39760624) kvi 7318728(29274912)
21/04/26 21:27:32 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:32 INFO mapred.MapTask: bufstart = 39760648; bufend = 69035532; bufvoid = 104857600
21/04/26 21:27:32 INFO mapred.MapTask: kvstart = 9940156(39760624); kvend = 22501764(90007056); length = 13652793/6553600
21/04/26 21:27:32 INFO mapred.MapTask: (EQUATOR) 79521286 kvi 19880316(79521264)
21/04/26 21:27:32 INFO mapred.LocalJobRunner: 
21/04/26 21:27:32 INFO mapred.MapTask: Starting flush of map output
21/04/26 21:27:32 INFO mapreduce.Job:  map 50% reduce 0%
21/04/26 21:27:33 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:27:33 INFO mapred.MapTask: Finished spill 1
21/04/26 21:27:33 INFO mapred.MapTask: (RESET) equator 79521286 kv 19880316(79521264) kvi 19146440(76585760)
21/04/26 21:27:33 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:33 INFO mapred.MapTask: bufstart = 79521286; bufend = 81094880; bufvoid = 104857600
21/04/26 21:27:33 INFO mapred.MapTask: kvstart = 19880316(79521264); kvend = 19146444(76585776); length = 733873/6553600
21/04/26 21:27:33 INFO mapred.MapTask: Finished spill 2
21/04/26 21:27:33 INFO mapred.Merger: Merging 3 sorted segments
21/04/26 21:27:33 INFO mapred.Merger: Down to the last merge-pass, with 3 segments left of total size: 74143117 bytes
21/04/26 21:27:33 INFO mapreduce.Job:  map 56% reduce 0%
21/04/26 21:27:36 INFO mapred.LocalJobRunner: map > sort > 
21/04/26 21:27:36 INFO mapreduce.Job:  map 58% reduce 0%
21/04/26 21:27:37 INFO mapred.Task: Task:attempt_local588307001_0001_m_000006_0 is done. And is in the process of committing
21/04/26 21:27:37 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:27:37 INFO mapred.Task: Task 'attempt_local588307001_0001_m_000006_0' done.
21/04/26 21:27:37 INFO mapred.Task: Final Counters for attempt_local588307001_0001_m_000006_0: Counters: 17
	File System Counters
		FILE: Number of bytes read=753919692
		FILE: Number of bytes written=1038297669
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Map input records=2941058
		Map output records=7009867
		Map output bytes=60123374
		Map output materialized bytes=74143114
		Input split bytes=98
		Combine input records=0
		Spilled Records=14019734
		Failed Shuffles=0
		Merged Map outputs=0
		GC time elapsed (ms)=71
		Total committed heap usage (bytes)=791150592
	File Input Format Counters 
		Bytes Read=33558528
21/04/26 21:27:37 INFO mapred.LocalJobRunner: Finishing task: attempt_local588307001_0001_m_000006_0
21/04/26 21:27:37 INFO mapred.LocalJobRunner: Starting task: attempt_local588307001_0001_m_000007_0
21/04/26 21:27:37 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 21:27:37 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
21/04/26 21:27:37 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@a1d7db9
21/04/26 21:27:37 INFO mapred.MapTask: Processing split: file:/E:/bigdata/2021.4.26/wc.txt:234881024+33554432
21/04/26 21:27:37 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
21/04/26 21:27:37 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
21/04/26 21:27:37 INFO mapred.MapTask: soft limit at 83886080
21/04/26 21:27:37 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
21/04/26 21:27:37 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
21/04/26 21:27:37 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
21/04/26 21:27:37 INFO mapreduce.Job:  map 100% reduce 0%
21/04/26 21:27:39 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:39 INFO mapred.MapTask: bufstart = 0; bufend = 29274882; bufvoid = 104857600
21/04/26 21:27:39 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 12561604(50246416); length = 13652793/6553600
21/04/26 21:27:39 INFO mapred.MapTask: (EQUATOR) 39760641 kvi 9940156(39760624)
21/04/26 21:27:40 INFO mapred.MapTask: Finished spill 0
21/04/26 21:27:40 INFO mapred.MapTask: (RESET) equator 39760641 kv 9940156(39760624) kvi 7318728(29274912)
21/04/26 21:27:42 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:42 INFO mapred.MapTask: bufstart = 39760641; bufend = 69035537; bufvoid = 104857600
21/04/26 21:27:42 INFO mapred.MapTask: kvstart = 9940156(39760624); kvend = 22501764(90007056); length = 13652793/6553600
21/04/26 21:27:42 INFO mapred.MapTask: (EQUATOR) 79521288 kvi 19880316(79521264)
21/04/26 21:27:42 INFO mapred.LocalJobRunner: 
21/04/26 21:27:42 INFO mapred.MapTask: Starting flush of map output
21/04/26 21:27:42 INFO mapreduce.Job:  map 58% reduce 0%
21/04/26 21:27:43 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:27:43 INFO mapred.MapTask: Finished spill 1
21/04/26 21:27:43 INFO mapred.MapTask: (RESET) equator 79521288 kv 19880316(79521264) kvi 19146444(76585776)
21/04/26 21:27:43 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:43 INFO mapred.MapTask: bufstart = 79521288; bufend = 81094880; bufvoid = 104857600
21/04/26 21:27:43 INFO mapred.MapTask: kvstart = 19880316(79521264); kvend = 19146448(76585792); length = 733869/6553600
21/04/26 21:27:43 INFO mapred.MapTask: Finished spill 2
21/04/26 21:27:43 INFO mapred.Merger: Merging 3 sorted segments
21/04/26 21:27:43 INFO mapred.Merger: Down to the last merge-pass, with 3 segments left of total size: 74143111 bytes
21/04/26 21:27:43 INFO mapreduce.Job:  map 64% reduce 0%
21/04/26 21:27:46 INFO mapred.LocalJobRunner: map > sort > 
21/04/26 21:27:46 INFO mapreduce.Job:  map 66% reduce 0%
21/04/26 21:27:46 INFO mapred.Task: Task:attempt_local588307001_0001_m_000007_0 is done. And is in the process of committing
21/04/26 21:27:46 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:27:46 INFO mapred.Task: Task 'attempt_local588307001_0001_m_000007_0' done.
21/04/26 21:27:46 INFO mapred.Task: Final Counters for attempt_local588307001_0001_m_000007_0: Counters: 17
	File System Counters
		FILE: Number of bytes read=861622031
		FILE: Number of bytes written=1186583929
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Map input records=2941058
		Map output records=7009866
		Map output bytes=60123370
		Map output materialized bytes=74143108
		Input split bytes=98
		Combine input records=0
		Spilled Records=14019732
		Failed Shuffles=0
		Merged Map outputs=0
		GC time elapsed (ms)=68
		Total committed heap usage (bytes)=1449656320
	File Input Format Counters 
		Bytes Read=33558528
21/04/26 21:27:46 INFO mapred.LocalJobRunner: Finishing task: attempt_local588307001_0001_m_000007_0
21/04/26 21:27:46 INFO mapred.LocalJobRunner: Starting task: attempt_local588307001_0001_m_000008_0
21/04/26 21:27:46 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 21:27:46 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
21/04/26 21:27:46 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@22664ee0
21/04/26 21:27:46 INFO mapred.MapTask: Processing split: file:/E:/bigdata/2021.4.26/wc.txt:268435456+33554432
21/04/26 21:27:46 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
21/04/26 21:27:46 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
21/04/26 21:27:46 INFO mapred.MapTask: soft limit at 83886080
21/04/26 21:27:46 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
21/04/26 21:27:46 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
21/04/26 21:27:46 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
21/04/26 21:27:47 INFO mapreduce.Job:  map 100% reduce 0%
21/04/26 21:27:48 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:48 INFO mapred.MapTask: bufstart = 0; bufend = 29274881; bufvoid = 104857600
21/04/26 21:27:48 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 12561604(50246416); length = 13652793/6553600
21/04/26 21:27:48 INFO mapred.MapTask: (EQUATOR) 39760640 kvi 9940156(39760624)
21/04/26 21:27:50 INFO mapred.MapTask: Finished spill 0
21/04/26 21:27:50 INFO mapred.MapTask: (RESET) equator 39760640 kv 9940156(39760624) kvi 7318728(29274912)
21/04/26 21:27:51 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:51 INFO mapred.MapTask: bufstart = 39760640; bufend = 69035523; bufvoid = 104857600
21/04/26 21:27:51 INFO mapred.MapTask: kvstart = 9940156(39760624); kvend = 22501764(90007056); length = 13652793/6553600
21/04/26 21:27:51 INFO mapred.MapTask: (EQUATOR) 79521281 kvi 19880316(79521264)
21/04/26 21:27:51 INFO mapred.LocalJobRunner: 
21/04/26 21:27:51 INFO mapred.MapTask: Starting flush of map output
21/04/26 21:27:52 INFO mapreduce.Job:  map 67% reduce 0%
21/04/26 21:27:52 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:27:53 INFO mapred.MapTask: Finished spill 1
21/04/26 21:27:53 INFO mapred.MapTask: (RESET) equator 79521281 kv 19880316(79521264) kvi 19146440(76585760)
21/04/26 21:27:53 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:53 INFO mapred.MapTask: bufstart = 79521281; bufend = 81094894; bufvoid = 104857600
21/04/26 21:27:53 INFO mapred.MapTask: kvstart = 19880316(79521264); kvend = 19146444(76585776); length = 733873/6553600
21/04/26 21:27:53 INFO mapred.MapTask: Finished spill 2
21/04/26 21:27:53 INFO mapred.Merger: Merging 3 sorted segments
21/04/26 21:27:53 INFO mapred.Merger: Down to the last merge-pass, with 3 segments left of total size: 74143120 bytes
21/04/26 21:27:53 INFO mapreduce.Job:  map 72% reduce 0%
21/04/26 21:27:55 INFO mapred.LocalJobRunner: map > sort > 
21/04/26 21:27:56 INFO mapreduce.Job:  map 74% reduce 0%
21/04/26 21:27:56 INFO mapred.Task: Task:attempt_local588307001_0001_m_000008_0 is done. And is in the process of committing
21/04/26 21:27:56 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:27:56 INFO mapred.Task: Task 'attempt_local588307001_0001_m_000008_0' done.
21/04/26 21:27:56 INFO mapred.Task: Final Counters for attempt_local588307001_0001_m_000008_0: Counters: 17
	File System Counters
		FILE: Number of bytes read=969324379
		FILE: Number of bytes written=1334870207
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Map input records=2941056
		Map output records=7009867
		Map output bytes=60123377
		Map output materialized bytes=74143117
		Input split bytes=98
		Combine input records=0
		Spilled Records=14019734
		Failed Shuffles=0
		Merged Map outputs=0
		GC time elapsed (ms)=57
		Total committed heap usage (bytes)=1227358208
	File Input Format Counters 
		Bytes Read=33558528
21/04/26 21:27:56 INFO mapred.LocalJobRunner: Finishing task: attempt_local588307001_0001_m_000008_0
21/04/26 21:27:56 INFO mapred.LocalJobRunner: Starting task: attempt_local588307001_0001_m_000009_0
21/04/26 21:27:56 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 21:27:56 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
21/04/26 21:27:56 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@559403aa
21/04/26 21:27:56 INFO mapred.MapTask: Processing split: file:/E:/bigdata/2021.4.26/wc.txt:301989888+33554432
21/04/26 21:27:56 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
21/04/26 21:27:56 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
21/04/26 21:27:56 INFO mapred.MapTask: soft limit at 83886080
21/04/26 21:27:56 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
21/04/26 21:27:56 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
21/04/26 21:27:56 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
21/04/26 21:27:57 INFO mapreduce.Job:  map 100% reduce 0%
21/04/26 21:27:58 INFO mapred.MapTask: Spilling map output
21/04/26 21:27:58 INFO mapred.MapTask: bufstart = 0; bufend = 29274888; bufvoid = 104857600
21/04/26 21:27:58 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 12561600(50246400); length = 13652797/6553600
21/04/26 21:27:58 INFO mapred.MapTask: (EQUATOR) 39760636 kvi 9940152(39760608)
21/04/26 21:28:00 INFO mapred.MapTask: Finished spill 0
21/04/26 21:28:00 INFO mapred.MapTask: (RESET) equator 39760636 kv 9940152(39760608) kvi 7318728(29274912)
21/04/26 21:28:02 INFO mapred.MapTask: Spilling map output
21/04/26 21:28:02 INFO mapred.MapTask: bufstart = 39760636; bufend = 69035517; bufvoid = 104857600
21/04/26 21:28:02 INFO mapred.MapTask: kvstart = 9940152(39760608); kvend = 22501760(90007040); length = 13652793/6553600
21/04/26 21:28:02 INFO mapred.MapTask: (EQUATOR) 79521270 kvi 19880312(79521248)
21/04/26 21:28:02 INFO mapred.LocalJobRunner: 
21/04/26 21:28:02 INFO mapred.MapTask: Starting flush of map output
21/04/26 21:28:02 INFO mapreduce.Job:  map 75% reduce 0%
21/04/26 21:28:02 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:28:03 INFO mapreduce.Job:  map 81% reduce 0%
21/04/26 21:28:03 INFO mapred.MapTask: Finished spill 1
21/04/26 21:28:03 INFO mapred.MapTask: (RESET) equator 79521270 kv 19880312(79521248) kvi 19146444(76585776)
21/04/26 21:28:03 INFO mapred.MapTask: Spilling map output
21/04/26 21:28:03 INFO mapred.MapTask: bufstart = 79521270; bufend = 81094872; bufvoid = 104857600
21/04/26 21:28:03 INFO mapred.MapTask: kvstart = 19880312(79521248); kvend = 19146448(76585792); length = 733865/6553600
21/04/26 21:28:03 INFO mapred.MapTask: Finished spill 2
21/04/26 21:28:03 INFO mapred.Merger: Merging 3 sorted segments
21/04/26 21:28:03 INFO mapred.Merger: Down to the last merge-pass, with 3 segments left of total size: 74143112 bytes
21/04/26 21:28:05 INFO mapred.LocalJobRunner: map > sort > 
21/04/26 21:28:06 INFO mapreduce.Job:  map 82% reduce 0%
21/04/26 21:28:07 INFO mapred.Task: Task:attempt_local588307001_0001_m_000009_0 is done. And is in the process of committing
21/04/26 21:28:07 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:28:07 INFO mapred.Task: Task 'attempt_local588307001_0001_m_000009_0' done.
21/04/26 21:28:07 INFO mapred.Task: Final Counters for attempt_local588307001_0001_m_000009_0: Counters: 17
	File System Counters
		FILE: Number of bytes read=1077026719
		FILE: Number of bytes written=1483156469
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Map input records=2941054
		Map output records=7009866
		Map output bytes=60123371
		Map output materialized bytes=74143109
		Input split bytes=98
		Combine input records=0
		Spilled Records=14019732
		Failed Shuffles=0
		Merged Map outputs=0
		GC time elapsed (ms)=61
		Total committed heap usage (bytes)=1042808832
	File Input Format Counters 
		Bytes Read=33558528
21/04/26 21:28:07 INFO mapred.LocalJobRunner: Finishing task: attempt_local588307001_0001_m_000009_0
21/04/26 21:28:07 INFO mapred.LocalJobRunner: Starting task: attempt_local588307001_0001_m_000010_0
21/04/26 21:28:07 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 21:28:07 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
21/04/26 21:28:07 INFO mapreduce.Job:  map 100% reduce 0%
21/04/26 21:28:07 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@abc125
21/04/26 21:28:07 INFO mapred.MapTask: Processing split: file:/E:/bigdata/2021.4.26/wc.txt:335544320+33554432
21/04/26 21:28:07 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
21/04/26 21:28:07 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
21/04/26 21:28:07 INFO mapred.MapTask: soft limit at 83886080
21/04/26 21:28:07 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
21/04/26 21:28:07 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
21/04/26 21:28:07 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
21/04/26 21:28:09 INFO mapred.MapTask: Spilling map output
21/04/26 21:28:09 INFO mapred.MapTask: bufstart = 0; bufend = 29274881; bufvoid = 104857600
21/04/26 21:28:09 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 12561604(50246416); length = 13652793/6553600
21/04/26 21:28:09 INFO mapred.MapTask: (EQUATOR) 39760640 kvi 9940156(39760624)
21/04/26 21:28:11 INFO mapred.MapTask: Finished spill 0
21/04/26 21:28:11 INFO mapred.MapTask: (RESET) equator 39760640 kv 9940156(39760624) kvi 7318728(29274912)
21/04/26 21:28:12 INFO mapred.MapTask: Spilling map output
21/04/26 21:28:12 INFO mapred.MapTask: bufstart = 39760640; bufend = 69035522; bufvoid = 104857600
21/04/26 21:28:12 INFO mapred.MapTask: kvstart = 9940156(39760624); kvend = 22501764(90007056); length = 13652793/6553600
21/04/26 21:28:12 INFO mapred.MapTask: (EQUATOR) 79521281 kvi 19880316(79521264)
21/04/26 21:28:13 INFO mapred.LocalJobRunner: 
21/04/26 21:28:13 INFO mapred.MapTask: Starting flush of map output
21/04/26 21:28:13 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:28:13 INFO mapreduce.Job:  map 89% reduce 0%
21/04/26 21:28:14 INFO mapred.MapTask: Finished spill 1
21/04/26 21:28:14 INFO mapred.MapTask: (RESET) equator 79521281 kv 19880316(79521264) kvi 19146444(76585776)
21/04/26 21:28:14 INFO mapred.MapTask: Spilling map output
21/04/26 21:28:14 INFO mapred.MapTask: bufstart = 79521281; bufend = 81094888; bufvoid = 104857600
21/04/26 21:28:14 INFO mapred.MapTask: kvstart = 19880316(79521264); kvend = 19146448(76585792); length = 733869/6553600
21/04/26 21:28:14 INFO mapred.MapTask: Finished spill 2
21/04/26 21:28:14 INFO mapred.Merger: Merging 3 sorted segments
21/04/26 21:28:14 INFO mapred.Merger: Down to the last merge-pass, with 3 segments left of total size: 74143111 bytes
21/04/26 21:28:16 INFO mapred.LocalJobRunner: map > sort > 
21/04/26 21:28:16 INFO mapreduce.Job:  map 90% reduce 0%
21/04/26 21:28:18 INFO mapred.Task: Task:attempt_local588307001_0001_m_000010_0 is done. And is in the process of committing
21/04/26 21:28:18 INFO mapred.LocalJobRunner: map > sort
21/04/26 21:28:18 INFO mapred.Task: Task 'attempt_local588307001_0001_m_000010_0' done.
21/04/26 21:28:18 INFO mapred.Task: Final Counters for attempt_local588307001_0001_m_000010_0: Counters: 17
	File System Counters
		FILE: Number of bytes read=1184729058
		FILE: Number of bytes written=1631442729
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Map input records=2941058
		Map output records=7009866
		Map output bytes=60123370
		Map output materialized bytes=74143108
		Input split bytes=98
		Combine input records=0
		Spilled Records=14019732
		Failed Shuffles=0
		Merged Map outputs=0
		GC time elapsed (ms)=65
		Total committed heap usage (bytes)=855638016
	File Input Format Counters 
		Bytes Read=33558528
21/04/26 21:28:18 INFO mapred.LocalJobRunner: Finishing task: attempt_local588307001_0001_m_000010_0
21/04/26 21:28:18 INFO mapred.LocalJobRunner: Starting task: attempt_local588307001_0001_m_000011_0
21/04/26 21:28:18 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 21:28:18 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
21/04/26 21:28:18 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@613a943c
21/04/26 21:28:18 INFO mapred.MapTask: Processing split: file:/E:/bigdata/2021.4.26/wc.txt:369098752+10911248
21/04/26 21:28:18 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
21/04/26 21:28:18 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
21/04/26 21:28:18 INFO mapred.MapTask: soft limit at 83886080
21/04/26 21:28:18 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
21/04/26 21:28:18 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
21/04/26 21:28:18 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
21/04/26 21:28:18 INFO mapreduce.Job:  map 100% reduce 0%
21/04/26 21:28:19 INFO mapred.LocalJobRunner: 
21/04/26 21:28:19 INFO mapred.MapTask: Starting flush of map output
21/04/26 21:28:19 INFO mapred.MapTask: Spilling map output
21/04/26 21:28:19 INFO mapred.MapTask: bufstart = 0; bufend = 19550923; bufvoid = 104857600
21/04/26 21:28:19 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 17096516(68386064); length = 9117881/6553600
21/04/26 21:28:19 INFO mapreduce.Job:  map 92% reduce 0%
21/04/26 21:28:20 INFO mapred.MapTask: Finished spill 0
21/04/26 21:28:20 INFO mapred.Task: Task:attempt_local588307001_0001_m_000011_0 is done. And is in the process of committing
21/04/26 21:28:20 INFO mapred.LocalJobRunner: map
21/04/26 21:28:20 INFO mapred.Task: Task 'attempt_local588307001_0001_m_000011_0' done.
21/04/26 21:28:20 INFO mapred.Task: Final Counters for attempt_local588307001_0001_m_000011_0: Counters: 17
	File System Counters
		FILE: Number of bytes read=1195640485
		FILE: Number of bytes written=1655552632
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Map input records=956375
		Map output records=2279471
		Map output bytes=19550923
		Map output materialized bytes=24109871
		Input split bytes=98
		Combine input records=0
		Spilled Records=2279471
		Failed Shuffles=0
		Merged Map outputs=0
		GC time elapsed (ms)=50
		Total committed heap usage (bytes)=781713408
	File Input Format Counters 
		Bytes Read=10911248
21/04/26 21:28:20 INFO mapred.LocalJobRunner: Finishing task: attempt_local588307001_0001_m_000011_0
21/04/26 21:28:20 INFO mapred.LocalJobRunner: map task executor complete.
21/04/26 21:28:20 INFO mapred.LocalJobRunner: Waiting for reduce tasks
21/04/26 21:28:20 INFO mapred.LocalJobRunner: Starting task: attempt_local588307001_0001_r_000000_0
21/04/26 21:28:20 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 21:28:20 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
21/04/26 21:28:20 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@4ea8102b
21/04/26 21:28:20 INFO mapred.ReduceTask: Using ShuffleConsumerPlugin: org.apache.hadoop.mapreduce.task.reduce.Shuffle@3b3fc2dc
21/04/26 21:28:20 INFO reduce.MergeManagerImpl: MergerManager: memoryLimit=3995913472, maxSingleShuffleLimit=998978368, mergeThreshold=2637303040, ioSortFactor=10, memToMemMergeOutputsThreshold=10
21/04/26 21:28:20 INFO reduce.EventFetcher: attempt_local588307001_0001_r_000000_0 Thread started: EventFetcher for fetching Map Completion Events
21/04/26 21:28:20 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local588307001_0001_m_000006_0 decomp: 74143110 len: 74143114 to MEMORY
21/04/26 21:28:20 INFO reduce.InMemoryMapOutput: Read 74143110 bytes from map-output for attempt_local588307001_0001_m_000006_0
21/04/26 21:28:20 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 74143110, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->74143110
21/04/26 21:28:20 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local588307001_0001_m_000009_0 decomp: 74143105 len: 74143109 to MEMORY
21/04/26 21:28:20 INFO reduce.InMemoryMapOutput: Read 74143105 bytes from map-output for attempt_local588307001_0001_m_000009_0
21/04/26 21:28:20 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 74143105, inMemoryMapOutputs.size() -> 2, commitMemory -> 74143110, usedMemory ->148286215
21/04/26 21:28:20 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local588307001_0001_m_000002_0 decomp: 74143099 len: 74143103 to MEMORY
21/04/26 21:28:20 INFO reduce.InMemoryMapOutput: Read 74143099 bytes from map-output for attempt_local588307001_0001_m_000002_0
21/04/26 21:28:20 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 74143099, inMemoryMapOutputs.size() -> 3, commitMemory -> 148286215, usedMemory ->222429314
21/04/26 21:28:20 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local588307001_0001_m_000005_0 decomp: 74143072 len: 74143076 to MEMORY
21/04/26 21:28:20 INFO mapreduce.Job:  map 100% reduce 0%
21/04/26 21:28:20 INFO reduce.InMemoryMapOutput: Read 74143072 bytes from map-output for attempt_local588307001_0001_m_000005_0
21/04/26 21:28:20 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 74143072, inMemoryMapOutputs.size() -> 4, commitMemory -> 222429314, usedMemory ->296572386
21/04/26 21:28:20 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local588307001_0001_m_000008_0 decomp: 74143113 len: 74143117 to MEMORY
21/04/26 21:28:20 INFO reduce.InMemoryMapOutput: Read 74143113 bytes from map-output for attempt_local588307001_0001_m_000008_0
21/04/26 21:28:20 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 74143113, inMemoryMapOutputs.size() -> 5, commitMemory -> 296572386, usedMemory ->370715499
21/04/26 21:28:20 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local588307001_0001_m_000011_0 decomp: 24109867 len: 24109871 to MEMORY
21/04/26 21:28:20 INFO reduce.InMemoryMapOutput: Read 24109867 bytes from map-output for attempt_local588307001_0001_m_000011_0
21/04/26 21:28:20 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 24109867, inMemoryMapOutputs.size() -> 6, commitMemory -> 370715499, usedMemory ->394825366
21/04/26 21:28:20 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local588307001_0001_m_000001_0 decomp: 74143098 len: 74143102 to MEMORY
21/04/26 21:28:20 INFO reduce.InMemoryMapOutput: Read 74143098 bytes from map-output for attempt_local588307001_0001_m_000001_0
21/04/26 21:28:20 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 74143098, inMemoryMapOutputs.size() -> 7, commitMemory -> 394825366, usedMemory ->468968464
21/04/26 21:28:21 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local588307001_0001_m_000004_0 decomp: 74143145 len: 74143149 to MEMORY
21/04/26 21:28:21 INFO reduce.InMemoryMapOutput: Read 74143145 bytes from map-output for attempt_local588307001_0001_m_000004_0
21/04/26 21:28:21 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 74143145, inMemoryMapOutputs.size() -> 8, commitMemory -> 468968464, usedMemory ->543111609
21/04/26 21:28:21 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local588307001_0001_m_000007_0 decomp: 74143104 len: 74143108 to MEMORY
21/04/26 21:28:21 INFO reduce.InMemoryMapOutput: Read 74143104 bytes from map-output for attempt_local588307001_0001_m_000007_0
21/04/26 21:28:21 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 74143104, inMemoryMapOutputs.size() -> 9, commitMemory -> 543111609, usedMemory ->617254713
21/04/26 21:28:21 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local588307001_0001_m_000010_0 decomp: 74143104 len: 74143108 to MEMORY
21/04/26 21:28:21 INFO reduce.InMemoryMapOutput: Read 74143104 bytes from map-output for attempt_local588307001_0001_m_000010_0
21/04/26 21:28:21 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 74143104, inMemoryMapOutputs.size() -> 10, commitMemory -> 617254713, usedMemory ->691397817
21/04/26 21:28:21 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local588307001_0001_m_000000_0 decomp: 74143109 len: 74143113 to MEMORY
21/04/26 21:28:21 INFO reduce.InMemoryMapOutput: Read 74143109 bytes from map-output for attempt_local588307001_0001_m_000000_0
21/04/26 21:28:21 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 74143109, inMemoryMapOutputs.size() -> 11, commitMemory -> 691397817, usedMemory ->765540926
21/04/26 21:28:21 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local588307001_0001_m_000003_0 decomp: 74143098 len: 74143102 to MEMORY
21/04/26 21:28:21 INFO reduce.InMemoryMapOutput: Read 74143098 bytes from map-output for attempt_local588307001_0001_m_000003_0
21/04/26 21:28:21 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 74143098, inMemoryMapOutputs.size() -> 12, commitMemory -> 765540926, usedMemory ->839684024
21/04/26 21:28:21 INFO reduce.EventFetcher: EventFetcher is interrupted.. Returning
21/04/26 21:28:21 INFO mapred.LocalJobRunner: 12 / 12 copied.
21/04/26 21:28:21 INFO reduce.MergeManagerImpl: finalMerge called with 12 in-memory map-outputs and 0 on-disk map-outputs
21/04/26 21:28:21 INFO mapred.Merger: Merging 12 sorted segments
21/04/26 21:28:21 INFO mapred.Merger: Down to the last merge-pass, with 12 segments left of total size: 839683988 bytes
21/04/26 21:28:26 INFO mapred.LocalJobRunner: reduce > sort
21/04/26 21:28:26 INFO mapreduce.Job:  map 100% reduce 38%
21/04/26 21:28:29 INFO mapred.LocalJobRunner: reduce > sort
21/04/26 21:28:29 INFO mapreduce.Job:  map 100% reduce 43%
21/04/26 21:28:32 INFO mapred.LocalJobRunner: reduce > sort
21/04/26 21:28:32 INFO mapreduce.Job:  map 100% reduce 48%
21/04/26 21:28:35 INFO mapred.LocalJobRunner: reduce > sort
21/04/26 21:28:35 INFO mapreduce.Job:  map 100% reduce 52%
21/04/26 21:28:38 INFO mapred.LocalJobRunner: reduce > sort
21/04/26 21:28:38 INFO mapreduce.Job:  map 100% reduce 56%
21/04/26 21:28:41 INFO mapred.LocalJobRunner: reduce > sort
21/04/26 21:28:41 INFO mapreduce.Job:  map 100% reduce 59%
21/04/26 21:28:44 INFO mapred.LocalJobRunner: reduce > sort
21/04/26 21:28:44 INFO mapreduce.Job:  map 100% reduce 63%
21/04/26 21:28:46 INFO reduce.MergeManagerImpl: Merged 12 segments, 839684024 bytes to disk to satisfy reduce memory limit
21/04/26 21:28:46 INFO reduce.MergeManagerImpl: Merging 1 files, 839684006 bytes from disk
21/04/26 21:28:46 INFO reduce.MergeManagerImpl: Merging 0 segments, 0 bytes from memory into reduce
21/04/26 21:28:46 INFO mapred.Merger: Merging 1 sorted segments
21/04/26 21:28:46 INFO mapred.Merger: Down to the last merge-pass, with 1 segments left of total size: 839683999 bytes
21/04/26 21:28:46 INFO mapred.LocalJobRunner: reduce > sort
21/04/26 21:28:46 INFO Configuration.deprecation: mapred.skip.on is deprecated. Instead, use mapreduce.job.skiprecords
21/04/26 21:28:47 INFO mapred.LocalJobRunner: reduce > reduce
21/04/26 21:28:47 INFO mapreduce.Job:  map 100% reduce 67%
21/04/26 21:28:50 INFO mapred.LocalJobRunner: reduce > reduce
21/04/26 21:28:50 INFO mapreduce.Job:  map 100% reduce 69%
21/04/26 21:28:53 INFO mapred.LocalJobRunner: reduce > reduce
21/04/26 21:28:53 INFO mapreduce.Job:  map 100% reduce 72%
21/04/26 21:28:56 INFO mapred.LocalJobRunner: reduce > reduce
21/04/26 21:28:56 INFO mapreduce.Job:  map 100% reduce 76%
21/04/26 21:28:59 INFO mapred.LocalJobRunner: reduce > reduce
21/04/26 21:28:59 INFO mapreduce.Job:  map 100% reduce 79%
21/04/26 21:29:02 INFO mapred.LocalJobRunner: reduce > reduce
21/04/26 21:29:02 INFO mapreduce.Job:  map 100% reduce 83%
21/04/26 21:29:05 INFO mapred.LocalJobRunner: reduce > reduce
21/04/26 21:29:05 INFO mapreduce.Job:  map 100% reduce 86%
21/04/26 21:29:08 INFO mapred.LocalJobRunner: reduce > reduce
21/04/26 21:29:08 INFO mapreduce.Job:  map 100% reduce 89%
21/04/26 21:29:11 INFO mapred.LocalJobRunner: reduce > reduce
21/04/26 21:29:11 INFO mapreduce.Job:  map 100% reduce 92%
21/04/26 21:29:14 INFO mapred.LocalJobRunner: reduce > reduce
21/04/26 21:29:14 INFO mapreduce.Job:  map 100% reduce 95%
21/04/26 21:29:17 INFO mapred.LocalJobRunner: reduce > reduce
21/04/26 21:29:17 INFO mapreduce.Job:  map 100% reduce 99%
21/04/26 21:29:18 INFO mapred.Task: Task:attempt_local588307001_0001_r_000000_0 is done. And is in the process of committing
21/04/26 21:29:18 INFO mapred.LocalJobRunner: reduce > reduce
21/04/26 21:29:18 INFO mapred.Task: Task attempt_local588307001_0001_r_000000_0 is allowed to commit now
21/04/26 21:29:18 INFO output.FileOutputCommitter: Saved output of task 'attempt_local588307001_0001_r_000000_0' to file:/E:/bigdata/2021.4.26/result/_temporary/0/task_local588307001_0001_r_000000
21/04/26 21:29:18 INFO mapred.LocalJobRunner: reduce > reduce
21/04/26 21:29:18 INFO mapred.Task: Task 'attempt_local588307001_0001_r_000000_0' done.
21/04/26 21:29:18 INFO mapred.Task: Final Counters for attempt_local588307001_0001_r_000000_0: Counters: 24
	File System Counters
		FILE: Number of bytes read=2875008947
		FILE: Number of bytes written=2495236732
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Combine input records=0
		Combine output records=0
		Reduce input groups=6
		Reduce shuffle bytes=839684072
		Reduce input records=79388000
		Reduce output records=6
		Spilled Records=79388000
		Shuffled Maps =12
		Failed Shuffles=0
		Merged Map outputs=12
		GC time elapsed (ms)=370
		Total committed heap usage (bytes)=2379218944
	Shuffle Errors
		BAD_ID=0
		CONNECTION=0
		IO_ERROR=0
		WRONG_LENGTH=0
		WRONG_MAP=0
		WRONG_REDUCE=0
	File Output Format Counters 
		Bytes Written=94
21/04/26 21:29:18 INFO mapred.LocalJobRunner: Finishing task: attempt_local588307001_0001_r_000000_0
21/04/26 21:29:18 INFO mapred.LocalJobRunner: reduce task executor complete.
21/04/26 21:29:18 INFO mapreduce.Job:  map 100% reduce 100%
21/04/26 21:29:18 INFO mapreduce.Job: Job job_local588307001_0001 completed successfully
21/04/26 21:29:18 INFO mapreduce.Job: Counters: 30
	File System Counters
		FILE: Number of bytes read=11179032658
		FILE: Number of bytes written=13940914909
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Map input records=33308000
		Map output records=79388000
		Map output bytes=680908000
		Map output materialized bytes=839684072
		Input split bytes=1176
		Combine input records=0
		Combine output records=0
		Reduce input groups=6
		Reduce shuffle bytes=839684072
		Reduce input records=79388000
		Reduce output records=6
		Spilled Records=235884529
		Shuffled Maps =12
		Failed Shuffles=0
		Merged Map outputs=12
		GC time elapsed (ms)=1198
		Total committed heap usage (bytes)=15761670144
	Shuffle Errors
		BAD_ID=0
		CONNECTION=0
		IO_ERROR=0
		WRONG_LENGTH=0
		WRONG_MAP=0
		WRONG_REDUCE=0
	File Input Format Counters 
		Bytes Read=380055056
	File Output Format Counters 
		Bytes Written=94
status = 0

Process finished with exit code 0

比虚拟机集群还要慢！！！步骤还长。。。简直不能忍：

但结果是正确的。

二手房数量统计小项目

package com.aa.secondhouse;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import java.io.IOException;

public class SecondHouseCount extends Configured implements Tool {
     

    //构建、配置、提交
    @Override
    public int run(String[] strings) throws Exception {
     
        //实例化一个MapReduce的Job对象
        Job job = Job.getInstance(this.getConf(), "second house numb");
        //指定允许Jar包运行的类
        job.setJarByClass(SecondHouseCount.class);

        //InPut
        //指定输入类的类型
        job.setInputFormatClass(TextInputFormat.class);//可以不指定，默认是TextInputFormat
        //指定输入流
        Path inputPath = new Path("E:\\bigdata\\2021.4.26\\secondhouse.csv");//使用本地路径作为程序输入

        TextInputFormat.setInputPaths(job, inputPath);

        //Map：配置Map
        job.setMapperClass(SecondMapper.class);//用来设置Mapper类
        job.setMapOutputKeyClass(Text.class);//设置K2的类型
        job.setMapOutputValueClass(IntWritable.class);//设置V2的类型

        //shuffle：配置shuffle
        //job.setPartitionerClass(null);//设置分区器
        //job.setSortComparatorClass(null);//设置排序器
        //job.setGroupingComparatorClass(null);//设置分组器
        //job.setCombinerClass(null);//设置Map端聚合

        //Reduce：配置Reduce
        job.setReducerClass(SecondReducer.class);//设置调用Reduce的类
        job.setOutputKeyClass(Text.class);//设置K3的类型
        job.setOutputValueClass(IntWritable.class);//设置V3的类型
        job.setNumReduceTasks(1);//设置ReduceTask的个数，默认为1个

        //Output：配置输出
        job.setOutputFormatClass(TextOutputFormat.class);//默认是使用TextOutputFormat
        //设置输出的路径
        Path outputPath = new Path("E:\\bigdata\\2021.4.26\\result1");
        //判断输出路径是否存在，存在就删除
        FileSystem fs = FileSystem.get(this.getConf());
        if (fs.exists(outputPath)) {
     
            fs.delete(outputPath, true);
        }
        TextOutputFormat.setOutputPath(job, outputPath);

        //提交job
        return job.waitForCompletion(true) ? 0 : -1;
    }

    public static void main(String[] args) throws Exception {
     
        //构建配置管理对象
        Configuration conf = new Configuration();
        //通过工具类的run方法调用当前类的实例的run方法
        int status = ToolRunner.run(conf, new SecondHouseCount(), args);
        System.out.println("status = " + status);
        //退出程序
        System.exit(status);
    }

    public static class SecondMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
     
        //输出K2
        Text outputKey = new Text();
        //输出V2
        IntWritable outputValue = new IntWritable(1);//不设置会导致结果=0

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
     
            //取出地区
            String region = value.toString().split(",")[3];
            //取出作为K2
            this.outputKey.set(region);
            //输出
            context.write(this.outputKey,this.outputValue);
        }
    }

    public static class SecondReducer extends Reducer<Text,IntWritable,Text,IntWritable>{
     
        //输出V3
        IntWritable outputValue=new IntWritable();

        @Override
        protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
     
            int sum=0;
            for (IntWritable value : values) {
     
                sum+=value.get();
            }
            this.outputValue.set(sum);
            context.write(key,this.outputValue);
        }
    }

}

直接给定绝对路径。。。最好新建一个文件夹，不然这段程序执行后会把第二个路径清空。。。HDD还能抢救一下，SSD欲哭无泪。。。

csv是逗号分割的文本格式，大概长这副德行：

传感器之类的下位机有时候会保存文件为csv。

运行过程：

21/04/26 22:28:27 INFO Configuration.deprecation: session.id is deprecated. Instead, use dfs.metrics.session-id
21/04/26 22:28:27 INFO jvm.JvmMetrics: Initializing JVM Metrics with processName=JobTracker, sessionId=
21/04/26 22:28:28 WARN mapreduce.JobResourceUploader: No job jar file set.  User classes may not be found. See Job or Job#setJar(String).
21/04/26 22:28:28 INFO input.FileInputFormat: Total input paths to process : 1
21/04/26 22:28:28 INFO mapreduce.JobSubmitter: number of splits:1
21/04/26 22:28:29 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_local1236341923_0001
21/04/26 22:28:29 INFO mapreduce.Job: The url to track the job: http://localhost:8080/
21/04/26 22:28:29 INFO mapreduce.Job: Running job: job_local1236341923_0001
21/04/26 22:28:29 INFO mapred.LocalJobRunner: OutputCommitter set in config null
21/04/26 22:28:29 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 22:28:29 INFO mapred.LocalJobRunner: OutputCommitter is org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
21/04/26 22:28:29 INFO mapred.LocalJobRunner: Waiting for map tasks
21/04/26 22:28:29 INFO mapred.LocalJobRunner: Starting task: attempt_local1236341923_0001_m_000000_0
21/04/26 22:28:29 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 22:28:29 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
21/04/26 22:28:29 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@3e639b3a
21/04/26 22:28:29 INFO mapred.MapTask: Processing split: file:/E:/bigdata/2021.4.26/secondhouse.csv:0+2225948
21/04/26 22:28:29 INFO mapred.MapTask: (EQUATOR) 0 kvi 26214396(104857584)
21/04/26 22:28:29 INFO mapred.MapTask: mapreduce.task.io.sort.mb: 100
21/04/26 22:28:29 INFO mapred.MapTask: soft limit at 83886080
21/04/26 22:28:29 INFO mapred.MapTask: bufstart = 0; bufvoid = 104857600
21/04/26 22:28:29 INFO mapred.MapTask: kvstart = 26214396; length = 6553600
21/04/26 22:28:29 INFO mapred.MapTask: Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
21/04/26 22:28:29 INFO mapred.LocalJobRunner: 
21/04/26 22:28:29 INFO mapred.MapTask: Starting flush of map output
21/04/26 22:28:29 INFO mapred.MapTask: Spilling map output
21/04/26 22:28:29 INFO mapred.MapTask: bufstart = 0; bufend = 310211; bufvoid = 104857600
21/04/26 22:28:29 INFO mapred.MapTask: kvstart = 26214396(104857584); kvend = 26101596(104406384); length = 112801/6553600
21/04/26 22:28:29 INFO mapred.MapTask: Finished spill 0
21/04/26 22:28:29 INFO mapred.Task: Task:attempt_local1236341923_0001_m_000000_0 is done. And is in the process of committing
21/04/26 22:28:29 INFO mapred.LocalJobRunner: map
21/04/26 22:28:29 INFO mapred.Task: Task 'attempt_local1236341923_0001_m_000000_0' done.
21/04/26 22:28:29 INFO mapred.Task: Final Counters for attempt_local1236341923_0001_m_000000_0: Counters: 17
	File System Counters
		FILE: Number of bytes read=2226112
		FILE: Number of bytes written=660803
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Map input records=28201
		Map output records=28201
		Map output bytes=310211
		Map output materialized bytes=366619
		Input split bytes=107
		Combine input records=0
		Spilled Records=28201
		Failed Shuffles=0
		Merged Map outputs=0
		GC time elapsed (ms)=0
		Total committed heap usage (bytes)=385875968
	File Input Format Counters 
		Bytes Read=2225948
21/04/26 22:28:29 INFO mapred.LocalJobRunner: Finishing task: attempt_local1236341923_0001_m_000000_0
21/04/26 22:28:29 INFO mapred.LocalJobRunner: map task executor complete.
21/04/26 22:28:29 INFO mapred.LocalJobRunner: Waiting for reduce tasks
21/04/26 22:28:29 INFO mapred.LocalJobRunner: Starting task: attempt_local1236341923_0001_r_000000_0
21/04/26 22:28:29 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 1
21/04/26 22:28:29 INFO util.ProcfsBasedProcessTree: ProcfsBasedProcessTree currently is supported only on Linux.
21/04/26 22:28:29 INFO mapred.Task:  Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@50329503
21/04/26 22:28:29 INFO mapred.ReduceTask: Using ShuffleConsumerPlugin: org.apache.hadoop.mapreduce.task.reduce.Shuffle@7a43db10
21/04/26 22:28:29 INFO reduce.MergeManagerImpl: MergerManager: memoryLimit=3995913472, maxSingleShuffleLimit=998978368, mergeThreshold=2637303040, ioSortFactor=10, memToMemMergeOutputsThreshold=10
21/04/26 22:28:29 INFO reduce.EventFetcher: attempt_local1236341923_0001_r_000000_0 Thread started: EventFetcher for fetching Map Completion Events
21/04/26 22:28:29 INFO reduce.LocalFetcher: localfetcher#1 about to shuffle output of map attempt_local1236341923_0001_m_000000_0 decomp: 366615 len: 366619 to MEMORY
21/04/26 22:28:29 INFO reduce.InMemoryMapOutput: Read 366615 bytes from map-output for attempt_local1236341923_0001_m_000000_0
21/04/26 22:28:29 INFO reduce.MergeManagerImpl: closeInMemoryFile -> map-output of size: 366615, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->366615
21/04/26 22:28:29 INFO reduce.EventFetcher: EventFetcher is interrupted.. Returning
21/04/26 22:28:29 INFO mapred.LocalJobRunner: 1 / 1 copied.
21/04/26 22:28:29 INFO reduce.MergeManagerImpl: finalMerge called with 1 in-memory map-outputs and 0 on-disk map-outputs
21/04/26 22:28:29 INFO mapred.Merger: Merging 1 sorted segments
21/04/26 22:28:29 INFO mapred.Merger: Down to the last merge-pass, with 1 segments left of total size: 366606 bytes
21/04/26 22:28:29 INFO reduce.MergeManagerImpl: Merged 1 segments, 366615 bytes to disk to satisfy reduce memory limit
21/04/26 22:28:29 INFO reduce.MergeManagerImpl: Merging 1 files, 366619 bytes from disk
21/04/26 22:28:29 INFO reduce.MergeManagerImpl: Merging 0 segments, 0 bytes from memory into reduce
21/04/26 22:28:29 INFO mapred.Merger: Merging 1 sorted segments
21/04/26 22:28:29 INFO mapred.Merger: Down to the last merge-pass, with 1 segments left of total size: 366606 bytes
21/04/26 22:28:29 INFO mapred.LocalJobRunner: 1 / 1 copied.
21/04/26 22:28:29 INFO Configuration.deprecation: mapred.skip.on is deprecated. Instead, use mapreduce.job.skiprecords
21/04/26 22:28:29 INFO mapred.Task: Task:attempt_local1236341923_0001_r_000000_0 is done. And is in the process of committing
21/04/26 22:28:29 INFO mapred.LocalJobRunner: 1 / 1 copied.
21/04/26 22:28:29 INFO mapred.Task: Task attempt_local1236341923_0001_r_000000_0 is allowed to commit now
21/04/26 22:28:29 INFO output.FileOutputCommitter: Saved output of task 'attempt_local1236341923_0001_r_000000_0' to file:/E:/bigdata/2021.4.26/result1/_temporary/0/task_local1236341923_0001_r_000000
21/04/26 22:28:29 INFO mapred.LocalJobRunner: reduce > reduce
21/04/26 22:28:29 INFO mapred.Task: Task 'attempt_local1236341923_0001_r_000000_0' done.
21/04/26 22:28:29 INFO mapred.Task: Final Counters for attempt_local1236341923_0001_r_000000_0: Counters: 24
	File System Counters
		FILE: Number of bytes read=2959382
		FILE: Number of bytes written=1027631
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Combine input records=0
		Combine output records=0
		Reduce input groups=17
		Reduce shuffle bytes=366619
		Reduce input records=28201
		Reduce output records=17
		Spilled Records=28201
		Shuffled Maps =1
		Failed Shuffles=0
		Merged Map outputs=1
		GC time elapsed (ms)=9
		Total committed heap usage (bytes)=385875968
	Shuffle Errors
		BAD_ID=0
		CONNECTION=0
		IO_ERROR=0
		WRONG_LENGTH=0
		WRONG_MAP=0
		WRONG_REDUCE=0
	File Output Format Counters 
		Bytes Written=209
21/04/26 22:28:29 INFO mapred.LocalJobRunner: Finishing task: attempt_local1236341923_0001_r_000000_0
21/04/26 22:28:29 INFO mapred.LocalJobRunner: reduce task executor complete.
21/04/26 22:28:30 INFO mapreduce.Job: Job job_local1236341923_0001 running in uber mode : false
21/04/26 22:28:30 INFO mapreduce.Job:  map 100% reduce 100%
21/04/26 22:28:30 INFO mapreduce.Job: Job job_local1236341923_0001 completed successfully
21/04/26 22:28:30 INFO mapreduce.Job: Counters: 30
	File System Counters
		FILE: Number of bytes read=5185494
		FILE: Number of bytes written=1688434
		FILE: Number of read operations=0
		FILE: Number of large read operations=0
		FILE: Number of write operations=0
	Map-Reduce Framework
		Map input records=28201
		Map output records=28201
		Map output bytes=310211
		Map output materialized bytes=366619
		Input split bytes=107
		Combine input records=0
		Combine output records=0
		Reduce input groups=17
		Reduce shuffle bytes=366619
		Reduce input records=28201
		Reduce output records=17
		Spilled Records=56402
		Shuffled Maps =1
		Failed Shuffles=0
		Merged Map outputs=1
		GC time elapsed (ms)=9
		Total committed heap usage (bytes)=771751936
	Shuffle Errors
		BAD_ID=0
		CONNECTION=0
		IO_ERROR=0
		WRONG_LENGTH=0
		WRONG_MAP=0
		WRONG_REDUCE=0
	File Input Format Counters 
		Bytes Read=2225948
	File Output Format Counters 
		Bytes Written=209
status = 0

Process finished with exit code 0

执行后：

至此，已初步掌握MapReduce。

MapReduce进阶继续介绍MapReduce的更多用法。

你可能感兴趣的:(云计算大数据,笔记,mapreduce,java,hadoop,intellij,idea,maven)

10月|愿你的青春不负梦想-读书笔记-01 Tracy的小书斋
本书的作者是俞敏洪，大家都很熟悉他了吧。俞敏洪老师是我行业的领头羊吧，也是我事业上的偶像。本日摘录他书中第一章中的金句：『一个人如果什么目标都没有，就会浑浑噩噩，感觉生命中缺少能量。能给我们能量的，是对未来的期待。第一件事，我始终为了进步而努力。与其追寻全世界的骏马，不如种植丰美的草原，到时骏马自然会来。第二件事，我始终有阶段性的目标。什么东西能给我能量？答案是对未来的期待。』读到这里的时候，我便
《投行人生》读书笔记小蘑菇的树洞
《投行人生》----作者詹姆斯-A-朗德摩根斯坦利副主席40年的职业洞见-很短小精悍的篇幅，比较适合初入职场的新人。第一部分成功的职业生涯需要规划1.情商归为适应能力分享与协作同理心适应能力，更多的是自我意识，你有能力识别自己的情并分辨这些情绪如何影响你的思想和行为。2.对于初入职场的人的建议，细节，截止日期和数据很重要截止日期，一种有效的方法是请老板为你所有的任务进行优先级排序。和老板喝咖啡的好
Long类型前后端数据不一致 igotyback 前端
响应给前端的数据浏览器控制台中response中看到的Long类型的数据是正常的到前端数据不一致前后端数据类型不匹配是一个常见问题，尤其是当后端使用Java的Long类型（64位）与前端JavaScript的Number类型（最大安全整数为2^53-1，即16位）进行数据交互时，很容易出现精度丢失的问题。这是因为JavaScript中的Number类型无法安全地表示超过16位的整数。为了解决这个问
LocalDateTime 转 String igotyback java 开发语言
importjava.time.LocalDateTime;importjava.time.format.DateTimeFormatter;publicclassMain{publicstaticvoidmain(String[]args){//获取当前时间LocalDateTimenow=LocalDateTime.now();//定义日期格式化器DateTimeFormatterformat
Linux下QT开发的动态库界面弹出操作（SDL2） 13jjyao QT类 qt 开发语言 sdl2 linux
需求：操作系统为linux，开发框架为qt，做成需带界面的qt动态库，调用方为java等非qt程序难点：调用方为java等非qt程序，也就是说调用方肯定不带QApplication::exec()，缺少了这个，QTimer等事件和QT创建的窗口将不能弹出(包括opencv也是不能弹出)；这与qt调用本身qt库是有本质的区别的思路：1.调用方缺QApplication::exec()，那么我们在接口
【一起学Rust | 设计模式】习惯语法——使用借用类型作为参数、格式化拼接字符串、构造函数广龙宇一起学Rust #Rust设计模式 rust 设计模式开发语言
提示：文章写完后，目录可以自动生成，如何生成可参考右边的帮助文档文章目录前言一、使用借用类型作为参数二、格式化拼接字符串三、使用构造函数总结前言Rust不是传统的面向对象编程语言，它的所有特性，使其独一无二。因此，学习特定于Rust的设计模式是必要的。本系列文章为作者学习《Rust设计模式》的学习笔记以及自己的见解。因此，本系列文章的结构也与此书的结构相同（后续可能会调成结构），基本上分为三个部分
git常用命令笔记咩酱-小羊 git 笔记
###用习惯了idea总是不记得git的一些常见命令，需要用到的时候总是担心旁边站了人~~~记个笔记@_@，告诉自己看笔记不丢人初始化初始化一个新的Git仓库gitinit配置配置用户信息gitconfig--globaluser.name"YourName"gitconfig--globaluser.email"[email protected]"基本操作克隆远程仓库gitclone查看
509. 斐波那契数(每日一题) lzyprime
lzyprime博客(github)创建时间：2021.01.04qq及邮箱：2383518170leetcode笔记题目描述斐波那契数，通常用F(n)表示，形成的序列称为斐波那契数列。该数列由0和1开始，后面的每一项数字都是前面两项数字的和。也就是：F(0)=0，F(1)=1F(n)=F(n-1)+F(n-2)，其中n>1给你n，请计算F(n)。示例1：输入：2输出：1解释：F(2)=F(1)+
拥有断舍离的心态，过精简生活--《断舍离》读书笔记爱吃丸子的小樱桃
不知不觉间房间里的东西越来越多，虽然摆放整齐，但也时常会觉得空间逼仄，令人心生烦闷。抱着断舍离的态度，我开始阅读《断舍离》这本书，希望从书中能找到一些有效的方法，帮助我实现空间、物品上的断舍离。《断舍离》是日本作家山下英子通过自己的经历、思考和实践总结而成的，整体内涵也从刚开始的私人生活哲学的“断舍离”升华成了“人生实践哲学”，接着又成为每个人都能实行的“改变人生的断舍离”，从“哲学”逐渐升华成“
四章-32-点要素的聚合彩云飘过
本文基于腾讯课堂老胡的课《跟我学Openlayers--基础实例详解》做的学习笔记，使用的openlayers5.3.xapi。源码见1032.html，对应的官网示例https://openlayers.org/en/latest/examples/cluster.htmlhttps://openlayers.org/en/latest/examples/earthquake-clusters.
DIV+CSS+JavaScript技术制作网页（旅游主题网页设计与制作）云南大理 STU学生网页设计网页设计期末网页作业 html静态网页 html5期末大作业网页设计 web大作业
️精彩专栏推荐作者主页:【进入主页—获取更多源码】web前端期末大作业：【HTML5网页期末作业(1000套)】程序员有趣的告白方式：【HTML七夕情人节表白网页制作(110套)】文章目录二、网站介绍三、网站效果▶️1.视频演示2.图片演示四、网站代码HTML结构代码CSS样式代码五、更多源码二、网站介绍网站布局方面：计划采用目前主流的、能兼容各大主流浏览器、显示效果稳定的浮动网页布局结构。网站程
【华为OD机试真题2023B卷 JAVA&JS】We Are A Team 若博豆 java 算法华为 javascript
华为OD2023（B卷）机试题库全覆盖，刷题指南点这里WeAreATeam时间限制：1秒|内存限制：32768K|语言限制：不限题目描述：总共有n个人在机房，每个人有一个标号（1<=标号<=n），他们分成了多个团队，需要你根据收到的m条消息判定指定的两个人是否在一个团队中，具体的：1、消息构成为：abc，整数a、b分别代
高端密码学院笔记285 柚子_b4b4
高端幸福密码学院（高级班）幸福使者：李华第（598）期《幸福》之回归内在深层生命原动力基础篇——揭秘“激励”成长的喜悦心理案例分析主讲：刘莉一，知识扩充:成功=艰苦劳动+正确方法+少说空话。贪图省力的船夫，目标永远下游。智者的梦再美，也不如愚人实干的脚印。幸福早课堂2020.10.16星期五一笔记:1，重视和珍惜的前提是知道它的价值非常重要，当你珍惜了，你就真正定下来，真正的学到身上。2，大家需要
关于城市旅游的HTML网页设计——(旅游风景云南 5页)HTML+CSS+JavaScript 二挡起步 web前端期末大作业 javascript html css 旅游风景
⛵源码获取文末联系✈Web前端开发技术描述网页设计题材，DIV+CSS布局制作,HTML+CSS网页设计期末课程大作业|游景点介绍|旅游风景区|家乡介绍|等网站的设计与制作|HTML期末大学生网页设计作业，Web大学生网页HTML：结构CSS：样式在操作方面上运用了html5和css3，采用了div+css结构、表单、超链接、浮动、绝对定位、相对定位、字体样式、引用视频等基础知识JavaScrip
HTML网页设计制作大作业（div+css）云南我的家乡旅游景点带文字滚动二挡起步 web前端期末大作业 web设计网页规划与设计 html css javascript dreamweaver 前端
Web前端开发技术描述网页设计题材，DIV+CSS布局制作,HTML+CSS网页设计期末课程大作业游景点介绍|旅游风景区|家乡介绍|等网站的设计与制作HTML期末大学生网页设计作业HTML：结构CSS：样式在操作方面上运用了html5和css3，采用了div+css结构、表单、超链接、浮动、绝对定位、相对定位、字体样式、引用视频等基础知识JavaScript：做与用户的交互行为文章目录前端学习路线
Day17笔记-高阶函数 ~在杰难逃~ Python 笔记 python 开发语言 pycharm 数据分析
高阶函数【重点掌握】函数的本质：函数是一个变量，函数名是一个变量名，一个函数可以作为另一个函数的参数或返回值使用如果A函数作为B函数的参数，B函数调用完成之后，会得到一个结果，则B函数被称为高阶函数常用的高阶函数：map(),reduce(),filter(),sorted()1.map()map(func,iterable)，返回值是一个iterator【容器，迭代器】func:函数iterab
Day1笔记-Python简介&标识符和关键字&输入输出 ~在杰难逃~ Python python 开发语言大数据数据分析数据挖掘
大家好，从今天开始呢，杰哥开展一个新的专栏，当然，数据分析部分也会不定时更新的，这个新的专栏主要是讲解一些Python的基础语法和知识，帮助0基础的小伙伴入门和学习Python，感兴趣的小伙伴可以开始认真学习啦！一、Python简介【了解】1.计算机工作原理编程语言就是用来定义计算机程序的形式语言。我们通过编程语言来编写程序代码，再通过语言处理程序执行向计算机发送指令，让计算机完成对应的工作，编程
node.js学习小猿L node.js node.js 学习 vim
node.js学习实操及笔记温故node.js，node.js学习实操过程及笔记~node.js学习视频node.js官网node.js中文网实操笔记githubcsdn笔记为什么学node.js可以让别人访问我们编写的网页为后续的框架学习打下基础，三大框架vuereactangular离不开node.jsnode.js是什么官网：node.js是一个开源的、跨平台的运行JavaScript的运行
数据仓库——维度表一致性墨染丶eye 背诵数据仓库
数据仓库基础笔记思维导图已经整理完毕，完整连接为：数据仓库基础知识笔记思维导图维度一致性问题从逻辑层面来看，当一系列星型模型共享一组公共维度时，所涉及的维度称为一致性维度。当维度表存在不一致时，短期的成功难以弥补长期的错误。维度时确保不同过程中信息集成起来实现横向钻取货活动的关键。造成横向钻取失败的原因维度结构的差别，因为维度的差别，分析工作涉及的领域从简单到复杂，但是都是通过复杂的报表来弥补设计
【Git】常见命令(仅笔记) 好想有猫猫 Git Linux学习笔记 git 笔记 elasticsearch linux c++
文章目录创建/初始化本地仓库添加本地仓库配置项提交文件查看仓库状态回退仓库查看日志分支删除文件暂存工作区代码远程仓库使用`.gitigore`文件让git不追踪一些文件标签创建/初始化本地仓库gitinit添加本地仓库配置项gitconfig-l#以列表形式显示配置项gitconfiguser.name"ljh"#配置user.namegitconfiguser.email"[email protected]
Java 重写(Override)与重载(Overload) 叨唧唧的
Java重写(Override)与重载(Overload)重写(Override)重写是子类对父类的允许访问的方法的实现过程进行重新编写,返回值和形参都不能改变。即外壳不变，核心重写！重写的好处在于子类可以根据需要，定义特定于自己的行为。也就是说子类能够根据需要实现父类的方法。重写方法不能抛出新的检查异常或者比被重写方法申明更加宽泛的异常。例如：父类的一个方法申明了一个检查异常IOExceptio
简单了解 JVM 记得开心一点啊 jvm
目录♫什么是JVM♫JVM的运行流程♫JVM运行时数据区♪虚拟机栈♪本地方法栈♪堆♪程序计数器♪方法区/元数据区♫类加载的过程♫双亲委派模型♫垃圾回收机制♫什么是JVMJVM是JavaVirtualMachine的简称，意为Java虚拟机。虚拟机是指通过软件模拟的具有完整硬件功能的、运行在一个完全隔离的环境中的完整计算机系统（如：JVM、VMwave、VirtualBox）。JVM和其他两个虚拟机
1分钟解决 -bash: mvn: command not found，在Centos 7中安装Maven Energet!c 开发语言
1分钟解决-bash:mvn:commandnotfound，在Centos7中安装Maven检查Java环境1下载Maven2解压Maven3配置环境变量4验证安装5常见问题与注意事项6总结检查Java环境Maven依赖Java环境，请确保系统已经安装了Java并配置了环境变量。可以通过以下命令检查：java-version如果未安装，请先安装Java。1下载Maven从官网下载：前往Apach
为什么你总是对下属不满意? ZhaoWu1050
【ZhaoWu的听课笔记】大多数公司，都存在两种问题。我创业四年，更是体会深切。这两种问题就是：老板经常不满意下属的表现；下属总是不知道老板想要什么；虽然这两种问题普遍存在，其实解决方法并不复杂。这节课，我们再聊聊第一个问题：为什么老板经常不满意下属表现?其实，这背后也是一条管理常识。管理学家德鲁克先生早就说过：管理者的任务，不是去改变人。*来自《卓有成效的管理者》只是大多数老板和我一样，都是一边
母亲节如何做小红书营销美橙传媒
小红书的一举一动引起了外界的高度关注。通过爆款笔记和流行话题，我们可以看到“干货”类型的内容在小红书中偏向实用的生活经验共享和生活指南非常受欢迎。根据运营社的分析，这种现象是由小红书用户心智和内容社区背后机制共同决定的。首先，小红书将使用“强搜索”逻辑为用户提供特定的“搜索场景”。在“我必须这样生活”中，大量使用了满足小红书站用户喜好和需求的内容。内容社区自制的高质量内容也吸引了寻找营销新途径的品
Java企业面试题3 马龙强_ java
1.break和continue的作用(智*图)break：用于完全退出一个循环（如for,while）或一个switch语句。当在循环体内遇到break语句时，程序会立即跳出当前循环体，继续执行循环之后的代码。continue：用于跳过当前循环体中剩余的部分，并开始下一次循环。如果是在for循环中使用continue，则会直接进行条件判断以决定是否执行下一轮循环。2.if分支语句和switch分
JVM、JRE和 JDK：理解Java开发的三大核心组件 Y雨何时停T Java java
Java是一门跨平台的编程语言，它的成功离不开背后强大的运行环境与开发工具的支持。在Java的生态中，JVM（Java虚拟机）、JRE（Java运行时环境）和JDK（Java开发工具包）是三个至关重要的核心组件。本文将探讨JVM、JDK和JRE的区别，帮助你更好地理解Java的运行机制。1.JVM：Java虚拟机（JavaVirtualMachine）什么是JVM？JVM，即Java虚拟机，是Ja
读书笔记|《遇见孩子，遇见更好的自己》5 抹茶社长
为人父母意味着放弃自己的过去，不要对以往没有实现的心愿耿耿于怀，只有这样，孩子们才能做回自己。985909803.jpg孩子在与父母保持亲密的同时更需要独立，唯有这样，孩子才会成为孩子，父母才会成其为父母。有耐心的人生往往更幸福，给孩子留点余地。认识到养儿育女是对耐心的考验。为失败做好心理准备，教会孩子控制情绪。了解自己的底线，说到底线，有一点很重要，父母之所以发脾气，真正的原因往往在于他们自己，
Java面试题精选：消息队列(二) 芒果不是芒 Java面试题精选 java kafka
一、Kafka的特性1.消息持久化：消息存储在磁盘，所以消息不会丢失2.高吞吐量：可以轻松实现单机百万级别的并发3.扩展性：扩展性强，还是动态扩展4.多客户端支持：支持多种语言（Java、C、C++、GO、）5.KafkaStreams（一个天生的流处理）:在双十一或者销售大屏就会用到这种流处理。使用KafkaStreams可以快速的把销售额统计出来6.安全机制：Kafka进行生产或者消费的时候会
基于Python给出的PDF文档转Markdown文档的方法程序媛了了 python pdf 开发语言
注：网上有很多将Markdown文档转为PDF文档的方法，但是却很少有将PDF文档转为Markdown文档的方法。就算有，比如某些网站声称可以将PDF文档转为Markdown文档，尝试过，不太符合自己的要求，而且无法保证文档没有泄露风险。于是本人为了解决这个问题，借助GPT（能使用GPT镜像或者有条件直接使用GPT的，反正能调用GPT接口就行）生成Python代码来完成这个功能。笔记、代码难免存在
HttpClient 4.3与4.3版本以下版本比较 spjich java httpclient
网上利用java发送http请求的代码很多，一搜一大把，有的利用的是java.net.*下的HttpURLConnection，有的用httpclient，而且发送的代码也分门别类。今天我们主要来说的是利用httpclient发送请求。 httpclient又可分为 httpclient3.x httpclient4.x到httpclient4.3以下 httpclient4.3
Essential Studio Enterprise Edition 2015 v1新功能体验 Axiba .net
概述：Essential Studio已全线升级至2015 v1版本了！新版本为JavaScript和ASP.NET MVC添加了新的文件资源管理器控件，还有其他一些控件功能升级，精彩不容错过，让我们一起来看看吧！ syncfusion公司是世界领先的Windows开发组件提供商，该公司正式对外发布Essential Studio Enterprise Edition 2015 v1版本。新版本
[宇宙与天文]微波背景辐射值与地球温度 comsci 背景
宇宙这个庞大,无边无际的空间是否存在某种确定的,变化的温度呢? 如果宇宙微波背景辐射值是表示宇宙空间温度的参数之一,那么测量这些数值,并观测周围的恒星能量输出值,我们是否获得地球的长期气候变化的情况呢? &nbs
lvs-server 男人50 server
#!/bin/bash # # LVS script for VS/DR # #./etc/rc.d/init.d/functions # VIP=10.10.6.252 RIP1=10.10.6.101 RIP2=10.10.6.13 PORT=80 case $1 in start) /sbin/ifconfig eth2:0 $VIP broadca
java的WebCollector爬虫框架 oloz 爬虫
WebCollector主页： https://github.com/CrawlScript/WebCollector 下载：webcollector-版本号-bin.zip将解压后文件夹中的所有jar包添加到工程既可。接下来看demo package org.spider.myspider; import cn.edu.hfut.dmic.webcollector.cra
jQuery append 与 after 的区别小猪猪08
1、after函数定义和用法： after() 方法在被选元素后插入指定的内容。语法： $(selector).after(content) 实例： <html> <head> <script type="text/javascript" src="/jquery/jquery.js"></scr
mysql知识充电香水浓 mysql
索引索引是在存储引擎中实现的，因此每种存储引擎的索引都不一定完全相同，并且每种存储引擎也不一定支持所有索引类型。根据存储引擎定义每个表的最大索引数和最大索引长度。所有存储引擎支持每个表至少16个索引，总索引长度至少为256字节。大多数存储引擎有更高的限制。MYSQL中索引的存储类型有两种：BTREE和HASH，具体和表的存储引擎相关； MYISAM和InnoDB存储引擎
我的架构经验系列文章索引 agevs 架构
下面是一些个人架构上的总结，本来想只在公司内部进行共享的，因此内容写的口语化一点，也没什么图示，所有内容没有查任何资料是脑子里面的东西吐出来的因此可能会不准确不全，希望抛砖引玉，大家互相讨论。要注意，我这些文章是一个总体的架构经验不针对具体的语言和平台，因此也不一定是适用所有的语言和平台的。（内容是前几天写的，现附上索引）前端架构 http://www.
Android so lib库远程http下载和动态注册 aijuans andorid
一、背景在开发Android应用程序的实现，有时候需要引入第三方so lib库，但第三方so库比较大，例如开源第三方播放组件ffmpeg库, 如果直接打包的apk包里面, 整个应用程序会大很多.经过查阅资料和实验，发现通过远程下载so文件，然后再动态注册so文件时可行的。主要需要解决下载so文件存放位置以及文件读写权限问题。二、主要
linux中svn配置出错 conf/svnserve.conf:12: Option expected 解决方法 baalwolf option
在客户端访问subversion版本库时出现这个错误： svnserve.conf:12: Option expected 为什么会出现这个错误呢，就是因为subversion读取配置文件svnserve.conf时，无法识别有前置空格的配置文件，如### This file controls the configuration of the svnserve daemon, if you##
MongoDB的连接池和连接管理 BigCat2013 mongodb
在关系型数据库中，我们总是需要关闭使用的数据库连接，不然大量的创建连接会导致资源的浪费甚至于数据库宕机。这篇文章主要想解释一下mongoDB的连接池以及连接管理机制，如果正对此有疑惑的朋友可以看一下。通常我们习惯于new 一个connection并且通常在finally语句中调用connection的close()方法将其关闭。正巧，mongoDB中当我们new一个Mongo的时候，会发现它也
AngularJS使用Socket.IO bijian1013 JavaScript AngularJS Socket.IO
目前，web应用普遍被要求是实时web应用，即服务端的数据更新之后，应用能立即更新。以前使用的技术（例如polling）存在一些局限性，而且有时我们需要在客户端打开一个socket，然后进行通信。 Socket.IO(http://socket.io/)是一个非常优秀的库，它可以帮你实
[Maven学习笔记四]Maven依赖特性 bit1129 maven
三个模块为了说明问题，以用户登陆小web应用为例。通常一个web应用分为三个模块，模型和数据持久化层user-core, 业务逻辑层user-service以及web展现层user-web， user-service依赖于user-core user-web依赖于user-core和user-service 依赖作用范围 Maven的dependency定义
【Akka一】Akka入门 bit1129 akka
什么是Akka Message-Driven Runtime is the Foundation to Reactive Applications In Akka, your business logic is driven through message-based communication patterns that are independent of physical locatio
zabbix_api之perl语言写法 ronin47 zabbix_api之perl
zabbix_api网上比较多的写法是python或curl。上次我用java－－http://bossr.iteye.com/blog/2195679，这次用perl。for example: #!/usr/bin/perl use 5.010 ; use strict ; use warnings ; use JSON :: RPC :: Client ; use
比优衣库跟牛掰的视频流出了，兄弟连Linux运维工程师课堂实录，更加刺激，更加实在！ brotherlamp linux运维工程师 linux运维工程师教程 linux运维工程师视频 linux运维工程师资料 linux运维工程师自学
比优衣库跟牛掰的视频流出了，兄弟连Linux运维工程师课堂实录，更加刺激，更加实在！ ----------------------------------------------------- 兄弟连Linux运维工程师课堂实录-计算机基础-1-课程体系介绍1 链接：http://pan.baidu.com/s/1i3GQtGL 密码：bl65 兄弟连Lin
bitmap求哈密顿距离-给定N（1<=N<=100000）个五维的点A(x1,x2,x3,x4,x5)，求两个点X(x1,x2,x3,x4,x5)和Y( bylijinnan java
import java.util.Random; /** * 题目： * 给定N（1<=N<=100000）个五维的点A(x1,x2,x3,x4,x5)，求两个点X(x1,x2,x3,x4,x5)和Y(y1,y2,y3,y4,y5)， * 使得他们的哈密顿距离（d=|x1-y1| + |x2-y2| + |x3-y3| + |x4-y4| + |x5-y5|）最大
map的三种遍历方法 chicony map
package com.test; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Set; public class TestMap { public static v
Linux安装mysql的一些坑 chenchao051 linux
1、mysql不建议在root用户下运行 2、出现服务启动不了，111错误，注意要用chown来赋予权限，我在root用户下装的mysql，我就把usr/share/mysql/mysql.server复制到/etc/init.d/mysqld, (同时把my-huge.cnf复制/etc/my.cnf) chown -R cc /etc/init.d/mysql
Sublime Text 3 配置 daizj 配置 Sublime Text
Sublime Text 3 配置解释(默认){// 设置主题文件“color_scheme”: “Packages/Color Scheme – Default/Monokai.tmTheme”,// 设置字体和大小“font_face”: “Consolas”,“font_size”: 12,// 字体选项：no_bold不显示粗体字，no_italic不显示斜体字，no_antialias和
MySQL server has gone away 问题的解决方法 dcj3sjt126com SQL Server
MySQL server has gone away 问题解决方法，需要的朋友可以参考下。应用程序（比如PHP）长时间的执行批量的MYSQL语句。执行一个SQL，但SQL语句过大或者语句中含有BLOB或者longblob字段。比如，图片数据的处理。都容易引起MySQL server has gone away。今天遇到类似的情景，MySQL只是冷冷的说：MySQL server h
javascript/dom:固定居中效果 dcj3sjt126com JavaScript
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> <html xmlns="http://www.w3.org/1999/xhtml&
使用 Spring 2.5 注释驱动的 IoC 功能 e200702084 spring bean 配置管理 IOC Office
使用 Spring 2.5 注释驱动的 IoC 功能 developerWorks 文档选项将打印机的版面设置成横向打印模式打印本页将此页作为电子邮件发送将此页作为电子邮件发送级别：初级陈雄华 ([email protected]), 技术总监, 宝宝淘网络科技有限公司 2008 年 2 月 28 日 &nb
MongoDB常用操作命令 geeksun mongodb
1. 基本操作 db.AddUser(username,password) 添加用户 db.auth(usrename,password) 设置数据库连接验证 db.cloneDataBase(fromhost)
php写守护进程（Daemon） hongtoushizi PHP
转载自： http://blog.csdn.net/tengzhaorong/article/details/9764655 守护进程（Daemon）是运行在后台的一种特殊进程。它独立于控制终端并且周期性地执行某种任务或等待处理某些发生的事件。守护进程是一种很有用的进程。php也可以实现守护进程的功能。 1、基本概念 &nbs
spring整合mybatis,关于注入Dao对象出错问题 jonsvien DAO spring bean mybatis prototype
今天在公司测试功能时发现一问题：先进行代码说明： 1，controller配置了Scope="prototype"（表明每一次请求都是原子型） @resource/@autowired service对象都可以（两种注解都可以）。 2，service 配置了Scope="prototype"（表明每一次请求都是原子型）
对象关系行为模式之标识映射 home198979 PHP 架构企业应用对象关系标识映射
HELLO!架构一、概念 identity Map:通过在映射中保存每个已经加载的对象，确保每个对象只加载一次，当要访问对象的时候，通过映射来查找它们。其实在数据源架构模式之数据映射器代码中有提及到标识映射，Mapper类的getFromMap方法就是实现标识映射的实现。二、为什么要使用标识映射？在数据源架构模式之数据映射器中 //c
Linux下hosts文件详解 pda158 linux
　1、主机名：　　无论在局域网还是INTERNET上，每台主机都有一个IP地址，是为了区分此台主机和彼台主机，也就是说IP地址就是主机的门牌号。　　公网：IP地址不方便记忆，所以又有了域名。域名只是在公网（INtERNET)中存在，每个域名都对应一个IP地址，但一个IP地址可有对应多个域名。　　局域网：每台机器都有一个主机名，用于主机与主机之间的便于区分，就可以为每台机器设置主机
nginx配置文件粗解 spjich java nginx
#运行用户#user nobody;#启动进程,通常设置成和cpu的数量相等worker_processes 2;#全局错误日志及PID文件#error_log logs/error.log;#error_log logs/error.log notice;#error_log logs/error.log inf
数学函数 w54653520 java
public class S { // 传入两个整数，进行比较，返回两个数中的最大值的方法。 public int get( int num1, int nu