+/EE360P

distributed systems

git clone https://git.julia.blue/+/EE360P

vijaygarg1init07ca65d

master
2.8 KiB78 linesraw
1/**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements.  See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership.  The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License.  You may obtain a copy of the License at
9 *
10 *     http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19package org.apache.hadoop.examples;
20
21import java.io.IOException;
22import java.util.ArrayList;
23import java.util.StringTokenizer;
24import java.util.Map.Entry;
25
26import org.apache.hadoop.io.Text;
27import org.apache.hadoop.mapreduce.Job;
28import org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorBaseDescriptor;
29import org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorJob;
30
31/**
32 * This is an example Aggregated Hadoop Map/Reduce application. It reads the
33 * text input files, breaks each line into words and counts them. The output is
34 * a locally sorted list of words and the count of how often they occurred.
35 * 
36 * To run: bin/hadoop jar hadoop-*-examples.jar aggregatewordcount 
37 * <i>in-dir</i> <i>out-dir</i> <i>numOfReducers</i> textinputformat
38 * 
39 */
40public class AggregateWordCount {
41
42  public static class WordCountPlugInClass extends
43      ValueAggregatorBaseDescriptor {
44    @Override
45    public ArrayList<Entry<Text, Text>> generateKeyValPairs(Object key,
46                                                            Object val) {
47      String countType = LONG_VALUE_SUM;
48      ArrayList<Entry<Text, Text>> retv = new ArrayList<Entry<Text, Text>>();
49      String line = val.toString();
50      StringTokenizer itr = new StringTokenizer(line);
51      while (itr.hasMoreTokens()) {
52        Entry<Text, Text> e = generateEntry(countType, itr.nextToken(), ONE);
53        if (e != null) {
54          retv.add(e);
55        }
56      }
57      return retv;
58    }
59  }
60
61  /**
62   * The main driver for word count map/reduce program. Invoke this method to
63   * submit the map/reduce job.
64   * 
65   * @throws IOException
66   *           When there is communication problems with the job tracker.
67   */
68  @SuppressWarnings("unchecked")
69  public static void main(String[] args) 
70    throws IOException, InterruptedException, ClassNotFoundException  {
71    Job job = ValueAggregatorJob.createValueAggregatorJob(args
72        , new Class[] {WordCountPlugInClass.class});
73    job.setJarByClass(AggregateWordCount.class);
74    int ret = job.waitForCompletion(true) ? 0 : 1;
75    System.exit(ret);
76  }
77
78}