+/EE360P

distributed systems

git clone https://git.julia.blue/+/EE360P

vijaygarg1init07ca65d

master
7.1 KiB213 linesraw
1package org.apache.hadoop.examples;
2
3/**
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements.  See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership.  The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License.  You may obtain a copy of the License at
11 *
12 *     http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 */
20
21import java.io.BufferedReader;
22import java.io.IOException;
23import java.io.InputStreamReader;
24import java.util.StringTokenizer;
25
26import org.apache.hadoop.conf.Configuration;
27import org.apache.hadoop.conf.Configured;
28import org.apache.hadoop.fs.FileSystem;
29import org.apache.hadoop.fs.Path;
30import org.apache.hadoop.io.LongWritable;
31import org.apache.hadoop.io.Text;
32import org.apache.hadoop.mapreduce.Job;
33import org.apache.hadoop.mapreduce.Mapper;
34import org.apache.hadoop.mapreduce.Reducer;
35import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
36import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
37import org.apache.hadoop.util.Tool;
38import org.apache.hadoop.util.ToolRunner;
39
40import com.google.common.base.Charsets;
41
42public class WordStandardDeviation extends Configured implements Tool {
43
44  private double stddev = 0;
45
46  private final static Text LENGTH = new Text("length");
47  private final static Text SQUARE = new Text("square");
48  private final static Text COUNT = new Text("count");
49  private final static LongWritable ONE = new LongWritable(1);
50
51  /**
52   * Maps words from line of text into 3 key-value pairs; one key-value pair for
53   * counting the word, one for counting its length, and one for counting the
54   * square of its length.
55   */
56  public static class WordStandardDeviationMapper extends
57      Mapper<Object, Text, Text, LongWritable> {
58
59    private LongWritable wordLen = new LongWritable();
60    private LongWritable wordLenSq = new LongWritable();
61
62    /**
63     * Emits 3 key-value pairs for counting the word, its length, and the
64     * squares of its length. Outputs are (Text, LongWritable).
65     * 
66     * @param value
67     *          This will be a line of text coming in from our input file.
68     */
69    public void map(Object key, Text value, Context context)
70        throws IOException, InterruptedException {
71      StringTokenizer itr = new StringTokenizer(value.toString());
72      while (itr.hasMoreTokens()) {
73        String string = itr.nextToken();
74
75        this.wordLen.set(string.length());
76
77        // the square of an integer is an integer...
78        this.wordLenSq.set((long) Math.pow(string.length(), 2.0));
79
80        context.write(LENGTH, this.wordLen);
81        context.write(SQUARE, this.wordLenSq);
82        context.write(COUNT, ONE);
83      }
84    }
85  }
86
87  /**
88   * Performs integer summation of all the values for each key.
89   */
90  public static class WordStandardDeviationReducer extends
91      Reducer<Text, LongWritable, Text, LongWritable> {
92
93    private LongWritable val = new LongWritable();
94
95    /**
96     * Sums all the individual values within the iterator and writes them to the
97     * same key.
98     * 
99     * @param key
100     *          This will be one of 2 constants: LENGTH_STR, COUNT_STR, or
101     *          SQUARE_STR.
102     * @param values
103     *          This will be an iterator of all the values associated with that
104     *          key.
105     */
106    public void reduce(Text key, Iterable<LongWritable> values, Context context)
107        throws IOException, InterruptedException {
108
109      int sum = 0;
110      for (LongWritable value : values) {
111        sum += value.get();
112      }
113      val.set(sum);
114      context.write(key, val);
115    }
116  }
117
118  /**
119   * Reads the output file and parses the summation of lengths, the word count,
120   * and the lengths squared, to perform a quick calculation of the standard
121   * deviation.
122   * 
123   * @param path
124   *          The path to find the output file in. Set in main to the output
125   *          directory.
126   * @throws IOException
127   *           If it cannot access the output directory, we throw an exception.
128   */
129  private double readAndCalcStdDev(Path path, Configuration conf)
130      throws IOException {
131    FileSystem fs = FileSystem.get(conf);
132    Path file = new Path(path, "part-r-00000");
133
134    if (!fs.exists(file))
135      throw new IOException("Output not found!");
136
137    double stddev = 0;
138    BufferedReader br = null;
139    try {
140      br = new BufferedReader(new InputStreamReader(fs.open(file), Charsets.UTF_8));
141      long count = 0;
142      long length = 0;
143      long square = 0;
144      String line;
145      while ((line = br.readLine()) != null) {
146        StringTokenizer st = new StringTokenizer(line);
147
148        // grab type
149        String type = st.nextToken();
150
151        // differentiate
152        if (type.equals(COUNT.toString())) {
153          String countLit = st.nextToken();
154          count = Long.parseLong(countLit);
155        } else if (type.equals(LENGTH.toString())) {
156          String lengthLit = st.nextToken();
157          length = Long.parseLong(lengthLit);
158        } else if (type.equals(SQUARE.toString())) {
159          String squareLit = st.nextToken();
160          square = Long.parseLong(squareLit);
161        }
162      }
163      // average = total sum / number of elements;
164      double mean = (((double) length) / ((double) count));
165      // standard deviation = sqrt((sum(lengths ^ 2)/count) - (mean ^ 2))
166      mean = Math.pow(mean, 2.0);
167      double term = (((double) square / ((double) count)));
168      stddev = Math.sqrt((term - mean));
169      System.out.println("The standard deviation is: " + stddev);
170    } finally {
171      if (br != null) {
172        br.close();
173      }
174    }
175    return stddev;
176  }
177
178  public static void main(String[] args) throws Exception {
179    ToolRunner.run(new Configuration(), new WordStandardDeviation(),
180        args);
181  }
182
183  @Override
184  public int run(String[] args) throws Exception {
185    if (args.length != 2) {
186      System.err.println("Usage: wordstddev <in> <out>");
187      return 0;
188    }
189
190    Configuration conf = getConf();
191
192    Job job = Job.getInstance(conf, "word stddev");
193    job.setJarByClass(WordStandardDeviation.class);
194    job.setMapperClass(WordStandardDeviationMapper.class);
195    job.setCombinerClass(WordStandardDeviationReducer.class);
196    job.setReducerClass(WordStandardDeviationReducer.class);
197    job.setOutputKeyClass(Text.class);
198    job.setOutputValueClass(LongWritable.class);
199    FileInputFormat.addInputPath(job, new Path(args[0]));
200    Path outputpath = new Path(args[1]);
201    FileOutputFormat.setOutputPath(job, outputpath);
202    boolean result = job.waitForCompletion(true);
203
204    // read output and calculate standard deviation
205    stddev = readAndCalcStdDev(outputpath, conf);
206
207    return (result ? 0 : 1);
208  }
209
210  public double getStandardDeviation() {
211    return stddev;
212  }
213}