+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18package org .apache .hadoop .examples ; 19 20import java .io .IOException ; 21import java .util .StringTokenizer ; 22 23import org .apache .hadoop .conf .Configuration ; 24import org .apache .hadoop .fs .Path ; 25import org .apache .hadoop .io .IntWritable ; 26import org .apache .hadoop .io .Text ; 27import org .apache .hadoop .mapreduce .Job ; 28import org .apache .hadoop .mapreduce .Mapper ; 29import org .apache .hadoop .mapreduce .Reducer ; 30import org .apache .hadoop .mapreduce .lib .input .FileInputFormat ; 31import org .apache .hadoop .mapreduce .lib .output .FileOutputFormat ; 32import org .apache .hadoop .util .GenericOptionsParser ; 33 34public class WordCount { 35 36public static class TokenizerMapper 37extends Mapper <Object ,Text ,Text ,IntWritable >{ 38 39private final static IntWritable one =new IntWritable (1 ); 40private Text word =new Text (); 41 42public void map (Object key ,Text value ,Context context 43 )throws IOException ,InterruptedException { 44StringTokenizer itr =new StringTokenizer (value .toString ()); 45while (itr .hasMoreTokens ()) { 46word .set (itr .nextToken ()); 47context .write (word ,one ); 48 } 49 } 50 } 51 52public static class IntSumReducer 53extends Reducer <Text ,IntWritable ,Text ,IntWritable > { 54private IntWritable result =new IntWritable (); 55 56public void reduce (Text key ,Iterable <IntWritable >values , 57Context context 58 )throws IOException ,InterruptedException { 59int sum =0 ; 60for (IntWritable val :values ) { 61sum +=val .get (); 62 } 63result .set (sum ); 64context .write (key ,result ); 65 } 66 } 67 68public static void main (String []args )throws Exception { 69Configuration conf =new Configuration (); 70String []otherArgs =new GenericOptionsParser (conf ,args ).getRemainingArgs (); 71if (otherArgs .length <2 ) { 72System .err .println ("Usage: wordcount <in> [<in>...] <out>" ); 73System .exit (2 ); 74 } 75Job job =Job .getInstance (conf ,"word count" ); 76job .setJarByClass (WordCount .class ); 77job .setMapperClass (TokenizerMapper .class ); 78job .setCombinerClass (IntSumReducer .class ); 79job .setReducerClass (IntSumReducer .class ); 80job .setOutputKeyClass (Text .class ); 81job .setOutputValueClass (IntWritable .class ); 82for (int i =0 ;i <otherArgs .length -1 ; ++i ) { 83FileInputFormat .addInputPath (job ,new Path (otherArgs [i ])); 84 } 85FileOutputFormat .setOutputPath (job , 86new Path (otherArgs [otherArgs .length -1 ])); 87System .exit (job .waitForCompletion (true ) ?0 :1 ); 88 } 89}