Hadoop Day06~MapReduce计算json文件的电影总分

计算json文件的电影总分

package com.doit.sumrate;

public class UserRateBean {
    private String movie;
    private Integer rate;
    private String timeStamp;
    private String uid;

    public String getMovie() {
        return movie;
    }

    public void setMovie(String movie) {
        this.movie = movie;
    }

    public Integer getRate() {
        return rate;
    }

    public void setRate(Integer rate) {
        this.rate = rate;
    }

    public String getTimeStamp() {
        return timeStamp;
    }

    public void setTimeStamp(String timeStamp) {
        this.timeStamp = timeStamp;
    }

    public String getUid() {
        return uid;
    }

    public void setUid(String uid) {
        this.uid = uid;
    }

    @Override
    public String toString() {
        return "UserRateBean{" +
                "movie='" + movie + '\'' +
                ", rate=" + rate +
                ", timeStamp='" + timeStamp + '\'' +
                ", uid='" + uid + '\'' +
                '}';

    }
}

package com.doit.sumrate;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.codehaus.jackson.map.ObjectMapper;
import java.io.IOException;
public class UserRateDriver {
    public static class UserRateMap extends Mapper{
        ObjectMapper objectMapper = new ObjectMapper();
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String line = value.toString();
            //{"movie":"1193","rate":"5","timeStamp":"978300760","uid":"1"}
            UserRateBean userRateBean = objectMapper.readValue(line, UserRateBean.class);
            String movie = userRateBean.getMovie();
            Integer rate = userRateBean.getRate();
            context.write(new Text(movie),new IntWritable(rate));
        }
    }
    public static class UserRateReducer extends Reducer {
        protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException {
            int sum = 0;
            for (IntWritable v : values) {
                sum = sum + v.get();
            }
            context.write(key, new IntWritable(sum));
        }
    }
    public static void main(String[] args) throws InterruptedException, IOException, ClassNotFoundException {

        Configuration conf = new Configuration();
        conf.set("yarn.resorcemanager.hostname", "node01");
        conf.set("fs.deafutFS", "hdfs://node01:9000/");
        Job job = Job.getInstance(conf);

        job.setJarByClass(UserRateDriver.class);

        job.setMapperClass(UserRateMap.class);
        job.setReducerClass(UserRateReducer.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);


        FileInputFormat.setInputPaths(job,new Path(args[0]));
        FileOutputFormat.setOutputPath(job,new Path(args[1]));

        job.submit();
        boolean b = job.waitForCompletion(true);
        System.exit(b?0:1);

    }

}

[root@node01 home]# hadoop fs -mkdir -p /lll/yyy/input/
[root@node01 home]# hadoop fs -put rating.json /lll/yyy/input/
[root@node01 home]# hadoop fs -cat /lll/yyy/input/rating.json
[root@node01 home]# hadoop jar movitrate.jar /lll/yyy/input/rating.json /lll/yyy/output/
[root@node01 home]# hadoop fs -ls /lll/yyy/input/rating.json/
[root@node01 home]# hadoop fs -cat /lll/yyy/output/part-r-00000/
469	204
47	4669
470	66
471	2175
472	119
473	336
474	3718
475	1217
476	94
477	676
478	59
479	372
48	1137
480	10057
481	994
482	621
483	277
484	77
485	1254
486	180
487	162
488	132
489	240
49	101
490	615
491	703
492	1246
493	1075
494	1392
495	142
496	119
497	2668
498	129
499	135
5	890
50	8054
500	2866
501	339
502	300
503	48
504	526
505	172
506	398
507	1085
508	2185
509	2234
510	101
511	472
512	514
513	209
514	1222
515	1763
516	712
517	1213
518	366
519	392
52	1569
520	1096
521	394
522	296
523	428
524	1721
525	67
526	20
527	10392
528	171
529	2864
53	38
530	7
531	1274
532	1186
533	555
534	1097
535	1149
536	174
537	637
538	1204
539	3708
54	105
540	375
541	7692
542	551
543	2308
544	341
545	2
546	656
547	357
548	349
549	600
55	138
550	537
551	3703
552	1714
553	2142
554	68
555	2585
556	558
557	9
558	365
559	15
56	18
560	29
561	47
562	1816
563	55
564	102
565	191
566	38
567	50
568	135
569	293
57	337
570	74
571	122
572	6
573	74
574	603
575	302
576	3
577	163
578	9
579	2
58	2051
580	201
581	571
582	6
583	103
584	4
585	1215
586	2116
587	3930
588	5118
589	10751
59	22
590	5681
591	4
592	5153
593	11219
594	2934
595	4119
596	1868
597	3809
598	34
599	1079
6	3646
60	1147
600	90
601	3
602	169
603	174
605	749
606	207
607	4
608	10692
609	186
61	201
610	2192
611	534
612	190
613	298
614	24
615	271
616	932
617	62
618	19
619	83
62	2000
621	156
623	9
624	4
626	43
627	436
628	1890
63	317
630	20
631	156
632	52
633	72
634	5
635	279
637	356
638	61
639	204
64	179
640	190
641	1
642	2
643	6
644	2
645	72
647	1567
648	5242
649	67
65	295
650	309
651	2
652	26
653	1974
655	1
656	100
657	6
658	3
659	93
66	162
660	3
661	1819
662	382
663	902
664	24
665	128
666	10
667	96
668	207
669	96
67	13
670	247
671	1678
672	4
673	1475
674	1164
678	1299
679	7
68	207
680	215
681	59
682	14
684	1
685	99
687	9
688	485
69	1166
690	4
691	375
692	224
694	373
695	429
696	13
697	217
698	11
7	1562
70	2885
700	482
701	4
702	99
703	16
704	97
705	138
706	5
707	849
708	2332
709	438
71	202
710	119
711	155
712	7
714	595
715	188
716	75
717	4
718	115
719	684
72	338
720	1939
722	103
724	1284
725	254
726	27
728	817
729	11
73	855
730	1
731	134
732	17
733	4989
734	9
735	339
736	3523
737	609
74	357
741	1284
742	440
743	275
744	6
745	2970
746	124
747	110
748	1314
749	11
75	23
750	6083
751	53
753	101
754	38
755	34
756	12
757	15
758	4
759	242
76	508
760	116
761	343
762	651
763	3
764	242
765	450
766	357
767	77
769	39
77	128
771	68
774	4
775	29
776	12
778	2974
779	140
78	163
780	6073
781	348
782	370
783	1257
784	1051
785	2661
786	1437
787	15
788	2840
789	6
79	297
790	3
791	7
792	4
793	26
796	10
798	552
799	1132
8	205
80	199
800	2572
801	375
802	1493
803	210
804	694
805	1235
806	190
807	42
808	146
809	270
81	525
810	176
811	35
813	118
814	3
815	12
818	484
82	345
820	13
821	4
823	13
824	110
826	1
827	6
828	157
829	300
83	122
830	871
831	77
832	1962
833	170
834	8
835	322
836	629
837	758
838	1944
839	299
84	65
840	128
841	85
842	391
843	1
844	25
846	142
847	13
848	536
849	1283
85	660
850	110
851	444
852	1981
853	12
854	29
858	10059
859	2
86	801
860	36
861	959
862	113
863	35
864	43
865	2
866	2187
867	53
868	3
869	130
87	168
870	107
872	3
874	25
875	149
876	60
877	79
878	4
879	670
88	565
880	713
881	202
882	160
884	6
885	111
886	306
887	5
888	106
889	5
89	704
891	432
892	587
893	255
895	2
896	175
897	368
898	2503
899	3217
9	271
90	22
900	1321
901	634
902	2642
903	3867
904	4700
905	1601
906	766
907	412
908	5765
909	1721
910	3578
911	1280
912	7365
913	4585
914	2642
915	1447
916	1792
917	283
918	717
919	7298
92	206
920	4621
921	1176
922	2111
923	4898
924	6982
925	39
926	1715
927	334
928	1623
929	352
93	199
930	1911
931	806
932	747
933	1817
934	1112
935	258
936	538
937	165
938	677
939	35
94	678
940	1502
941	389
942	1227
943	681
944	368
945	1041
946	620
947	772
948	488
949	855
95	1835
950	1238
951	1687
952	969
953	3134
954	1624
955	1722
956	142
957	120
958	44
959	217
96	51
960	69
961	80
962	39
963	193
964	173
965	1031
966	24
967	75
968	2625
969	4494
97	165
970	126
971	1097
972	19
973	321
974	60
975	3
976	200
977	4
978	394
98	12
980	14
981	11
982	298
984	88
985	15
986	836
987	116
988	233
989	5
99	173
990	139
991	599
992	52
993	28
994	1843
996	744
997	94
998	280
999	939
........
........
........