+/EE360P

distributed systems

git clone https://git.julia.blue/+/EE360P

vijaygarg1init07ca65d

master
2.9 KiB82 linesraw
1/**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements.  See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership.  The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License.  You may obtain a copy of the License at
9 *
10 *     http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19package org.apache.hadoop.examples;
20
21import java.io.IOException;
22import java.util.ArrayList;
23import java.util.Map.Entry;
24
25import org.apache.hadoop.io.Text;
26import org.apache.hadoop.mapreduce.Job;
27import org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorBaseDescriptor;
28import org.apache.hadoop.mapreduce.lib.aggregate.ValueAggregatorJob;
29
30/**
31 * This is an example Aggregated Hadoop Map/Reduce application. Computes the
32 * histogram of the words in the input texts.
33 * 
34 * To run: bin/hadoop jar hadoop-*-examples.jar aggregatewordhist <i>in-dir</i>
35 * <i>out-dir</i> <i>numOfReducers</i> textinputformat
36 * 
37 */
38public class AggregateWordHistogram {
39
40  public static class AggregateWordHistogramPlugin 
41    extends ValueAggregatorBaseDescriptor {
42    
43    /**
44     * Parse the given value, generate an aggregation-id/value pair per word.
45     * The ID is of type VALUE_HISTOGRAM, with WORD_HISTOGRAM as the real id.
46     * The value is WORD\t1.
47     *
48     * @return a list of the generated pairs.
49     */
50    @Override
51    public ArrayList<Entry<Text, Text>> generateKeyValPairs(Object key,
52                                          Object val) {
53      String words[] = val.toString().split(" |\t");
54      ArrayList<Entry<Text, Text>> retv = new ArrayList<Entry<Text, Text>>();
55      for (int i = 0; i < words.length; i++) {
56        Text valCount = new Text(words[i] + "\t" + "1");
57        Entry<Text, Text> en = generateEntry(VALUE_HISTOGRAM, "WORD_HISTOGRAM",
58                                 valCount);
59        retv.add(en);
60      }
61      return retv;
62    }
63    
64  }
65  
66  /**
67   * The main driver for word count map/reduce program. Invoke this method to
68   * submit the map/reduce job.
69   * 
70   * @throws IOException
71   *           When there is communication problems with the job tracker.
72   */
73  @SuppressWarnings("unchecked")
74  public static void main(String[] args) 
75    throws IOException, InterruptedException, ClassNotFoundException  {
76    Job job = ValueAggregatorJob.createValueAggregatorJob(args
77        , new Class[] {AggregateWordHistogramPlugin.class});
78    job.setJarByClass(AggregateWordCount.class);
79    int ret = job.waitForCompletion(true) ? 0 : 1;
80    System.exit(ret);
81  }
82}