+/EE360P

distributed systems

git clone https://git.julia.blue/+/EE360P

vijaygarg1init07ca65d

master
6.2 KiB199 linesraw
1package org.apache.hadoop.examples;
2
3/**
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements.  See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership.  The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License.  You may obtain a copy of the License at
11 *
12 *     http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 */
20
21import java.io.BufferedReader;
22import java.io.IOException;
23import java.io.InputStreamReader;
24import java.util.StringTokenizer;
25
26import org.apache.hadoop.conf.Configuration;
27import org.apache.hadoop.conf.Configured;
28import org.apache.hadoop.fs.FileSystem;
29import org.apache.hadoop.fs.Path;
30import org.apache.hadoop.io.LongWritable;
31import org.apache.hadoop.io.Text;
32import org.apache.hadoop.mapreduce.Job;
33import org.apache.hadoop.mapreduce.Mapper;
34import org.apache.hadoop.mapreduce.Reducer;
35import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
36import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
37import org.apache.hadoop.util.Tool;
38import org.apache.hadoop.util.ToolRunner;
39
40import com.google.common.base.Charsets;
41
42public class WordMean extends Configured implements Tool {
43
44  private double mean = 0;
45
46  private final static Text COUNT = new Text("count");
47  private final static Text LENGTH = new Text("length");
48  private final static LongWritable ONE = new LongWritable(1);
49
50  /**
51   * Maps words from line of text into 2 key-value pairs; one key-value pair for
52   * counting the word, another for counting its length.
53   */
54  public static class WordMeanMapper extends
55      Mapper<Object, Text, Text, LongWritable> {
56
57    private LongWritable wordLen = new LongWritable();
58
59    /**
60     * Emits 2 key-value pairs for counting the word and its length. Outputs are
61     * (Text, LongWritable).
62     * 
63     * @param value
64     *          This will be a line of text coming in from our input file.
65     */
66    public void map(Object key, Text value, Context context)
67        throws IOException, InterruptedException {
68      StringTokenizer itr = new StringTokenizer(value.toString());
69      while (itr.hasMoreTokens()) {
70        String string = itr.nextToken();
71        this.wordLen.set(string.length());
72        context.write(LENGTH, this.wordLen);
73        context.write(COUNT, ONE);
74      }
75    }
76  }
77
78  /**
79   * Performs integer summation of all the values for each key.
80   */
81  public static class WordMeanReducer extends
82      Reducer<Text, LongWritable, Text, LongWritable> {
83
84    private LongWritable sum = new LongWritable();
85
86    /**
87     * Sums all the individual values within the iterator and writes them to the
88     * same key.
89     * 
90     * @param key
91     *          This will be one of 2 constants: LENGTH_STR or COUNT_STR.
92     * @param values
93     *          This will be an iterator of all the values associated with that
94     *          key.
95     */
96    public void reduce(Text key, Iterable<LongWritable> values, Context context)
97        throws IOException, InterruptedException {
98
99      int theSum = 0;
100      for (LongWritable val : values) {
101        theSum += val.get();
102      }
103      sum.set(theSum);
104      context.write(key, sum);
105    }
106  }
107
108  /**
109   * Reads the output file and parses the summation of lengths, and the word
110   * count, to perform a quick calculation of the mean.
111   * 
112   * @param path
113   *          The path to find the output file in. Set in main to the output
114   *          directory.
115   * @throws IOException
116   *           If it cannot access the output directory, we throw an exception.
117   */
118  private double readAndCalcMean(Path path, Configuration conf)
119      throws IOException {
120    FileSystem fs = FileSystem.get(conf);
121    Path file = new Path(path, "part-r-00000");
122
123    if (!fs.exists(file))
124      throw new IOException("Output not found!");
125
126    BufferedReader br = null;
127
128    // average = total sum / number of elements;
129    try {
130      br = new BufferedReader(new InputStreamReader(fs.open(file), Charsets.UTF_8));
131
132      long count = 0;
133      long length = 0;
134
135      String line;
136      while ((line = br.readLine()) != null) {
137        StringTokenizer st = new StringTokenizer(line);
138
139        // grab type
140        String type = st.nextToken();
141
142        // differentiate
143        if (type.equals(COUNT.toString())) {
144          String countLit = st.nextToken();
145          count = Long.parseLong(countLit);
146        } else if (type.equals(LENGTH.toString())) {
147          String lengthLit = st.nextToken();
148          length = Long.parseLong(lengthLit);
149        }
150      }
151
152      double theMean = (((double) length) / ((double) count));
153      System.out.println("The mean is: " + theMean);
154      return theMean;
155    } finally {
156      if (br != null) {
157        br.close();
158      }
159    }
160  }
161
162  public static void main(String[] args) throws Exception {
163    ToolRunner.run(new Configuration(), new WordMean(), args);
164  }
165
166  @Override
167  public int run(String[] args) throws Exception {
168    if (args.length != 2) {
169      System.err.println("Usage: wordmean <in> <out>");
170      return 0;
171    }
172
173    Configuration conf = getConf();
174
175    Job job = Job.getInstance(conf, "word mean");
176    job.setJarByClass(WordMean.class);
177    job.setMapperClass(WordMeanMapper.class);
178    job.setCombinerClass(WordMeanReducer.class);
179    job.setReducerClass(WordMeanReducer.class);
180    job.setOutputKeyClass(Text.class);
181    job.setOutputValueClass(LongWritable.class);
182    FileInputFormat.addInputPath(job, new Path(args[0]));
183    Path outputpath = new Path(args[1]);
184    FileOutputFormat.setOutputPath(job, outputpath);
185    boolean result = job.waitForCompletion(true);
186    mean = readAndCalcMean(outputpath, conf);
187
188    return (result ? 0 : 1);
189  }
190
191  /**
192   * Only valuable after run() called.
193   * 
194   * @return Returns the mean value.
195   */
196  public double getMean() {
197    return mean;
198  }
199}