+/EE360P

distributed systems

git clone https://git.julia.blue/+/EE360P

vijaygarg1init07ca65d

master
3.2 KiB89 linesraw
1/**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements.  See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership.  The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License.  You may obtain a copy of the License at
9 *
10 *     http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18package org.apache.hadoop.examples;
19
20import java.io.IOException;
21import java.util.StringTokenizer;
22
23import org.apache.hadoop.conf.Configuration;
24import org.apache.hadoop.fs.Path;
25import org.apache.hadoop.io.IntWritable;
26import org.apache.hadoop.io.Text;
27import org.apache.hadoop.mapreduce.Job;
28import org.apache.hadoop.mapreduce.Mapper;
29import org.apache.hadoop.mapreduce.Reducer;
30import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
31import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
32import org.apache.hadoop.util.GenericOptionsParser;
33
34public class WordCount {
35
36  public static class TokenizerMapper 
37       extends Mapper<Object, Text, Text, IntWritable>{
38    
39    private final static IntWritable one = new IntWritable(1);
40    private Text word = new Text();
41      
42    public void map(Object key, Text value, Context context
43                    ) throws IOException, InterruptedException {
44      StringTokenizer itr = new StringTokenizer(value.toString());
45      while (itr.hasMoreTokens()) {
46        word.set(itr.nextToken());
47        context.write(word, one);
48      }
49    }
50  }
51  
52  public static class IntSumReducer 
53       extends Reducer<Text,IntWritable,Text,IntWritable> {
54    private IntWritable result = new IntWritable();
55
56    public void reduce(Text key, Iterable<IntWritable> values, 
57                       Context context
58                       ) throws IOException, InterruptedException {
59      int sum = 0;
60      for (IntWritable val : values) {
61        sum += val.get();
62      }
63      result.set(sum);
64      context.write(key, result);
65    }
66  }
67
68  public static void main(String[] args) throws Exception {
69    Configuration conf = new Configuration();
70    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
71    if (otherArgs.length < 2) {
72      System.err.println("Usage: wordcount <in> [<in>...] <out>");
73      System.exit(2);
74    }
75    Job job = Job.getInstance(conf, "word count");
76    job.setJarByClass(WordCount.class);
77    job.setMapperClass(TokenizerMapper.class);
78    job.setCombinerClass(IntSumReducer.class);
79    job.setReducerClass(IntSumReducer.class);
80    job.setOutputKeyClass(Text.class);
81    job.setOutputValueClass(IntWritable.class);
82    for (int i = 0; i < otherArgs.length - 1; ++i) {
83      FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
84    }
85    FileOutputFormat.setOutputPath(job,
86      new Path(otherArgs[otherArgs.length - 1]));
87    System.exit(job.waitForCompletion(true) ? 0 : 1);
88  }
89}