+/EE360P

distributed systems

git clone https://git.julia.blue/+/EE360P

vijaygarg1init07ca65d

master
6.9 KiB211 linesraw
1package org.apache.hadoop.examples;
2
3/**
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements.  See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership.  The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License.  You may obtain a copy of the License at
11 *
12 *     http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 */
20
21import java.io.BufferedReader;
22import java.io.IOException;
23import java.io.InputStreamReader;
24import java.util.StringTokenizer;
25
26import org.apache.hadoop.conf.Configuration;
27import org.apache.hadoop.conf.Configured;
28import org.apache.hadoop.fs.FileSystem;
29import org.apache.hadoop.fs.Path;
30import org.apache.hadoop.io.IntWritable;
31import org.apache.hadoop.io.Text;
32import org.apache.hadoop.mapreduce.Job;
33import org.apache.hadoop.mapreduce.Mapper;
34import org.apache.hadoop.mapreduce.Reducer;
35import org.apache.hadoop.mapreduce.TaskCounter;
36import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
37import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
38import org.apache.hadoop.util.Tool;
39import org.apache.hadoop.util.ToolRunner;
40
41import com.google.common.base.Charsets;
42
43public class WordMedian extends Configured implements Tool {
44
45  private double median = 0;
46  private final static IntWritable ONE = new IntWritable(1);
47
48  /**
49   * Maps words from line of text into a key-value pair; the length of the word
50   * as the key, and 1 as the value.
51   */
52  public static class WordMedianMapper extends
53      Mapper<Object, Text, IntWritable, IntWritable> {
54
55    private IntWritable length = new IntWritable();
56
57    /**
58     * Emits a key-value pair for counting the word. Outputs are (IntWritable,
59     * IntWritable).
60     * 
61     * @param value
62     *          This will be a line of text coming in from our input file.
63     */
64    public void map(Object key, Text value, Context context)
65        throws IOException, InterruptedException {
66      StringTokenizer itr = new StringTokenizer(value.toString());
67      while (itr.hasMoreTokens()) {
68        String string = itr.nextToken();
69        length.set(string.length());
70        context.write(length, ONE);
71      }
72    }
73  }
74
75  /**
76   * Performs integer summation of all the values for each key.
77   */
78  public static class WordMedianReducer extends
79      Reducer<IntWritable, IntWritable, IntWritable, IntWritable> {
80
81    private IntWritable val = new IntWritable();
82
83    /**
84     * Sums all the individual values within the iterator and writes them to the
85     * same key.
86     * 
87     * @param key
88     *          This will be a length of a word that was read.
89     * @param values
90     *          This will be an iterator of all the values associated with that
91     *          key.
92     */
93    public void reduce(IntWritable key, Iterable<IntWritable> values,
94        Context context) throws IOException, InterruptedException {
95
96      int sum = 0;
97      for (IntWritable value : values) {
98        sum += value.get();
99      }
100      val.set(sum);
101      context.write(key, val);
102    }
103  }
104
105  /**
106   * This is a standard program to read and find a median value based on a file
107   * of word counts such as: 1 456, 2 132, 3 56... Where the first values are
108   * the word lengths and the following values are the number of times that
109   * words of that length appear.
110   * 
111   * @param path
112   *          The path to read the HDFS file from (part-r-00000...00001...etc).
113   * @param medianIndex1
114   *          The first length value to look for.
115   * @param medianIndex2
116   *          The second length value to look for (will be the same as the first
117   *          if there are an even number of words total).
118   * @throws IOException
119   *           If file cannot be found, we throw an exception.
120   * */
121  private double readAndFindMedian(String path, int medianIndex1,
122      int medianIndex2, Configuration conf) throws IOException {
123    FileSystem fs = FileSystem.get(conf);
124    Path file = new Path(path, "part-r-00000");
125
126    if (!fs.exists(file))
127      throw new IOException("Output not found!");
128
129    BufferedReader br = null;
130
131    try {
132      br = new BufferedReader(new InputStreamReader(fs.open(file), Charsets.UTF_8));
133      int num = 0;
134
135      String line;
136      while ((line = br.readLine()) != null) {
137        StringTokenizer st = new StringTokenizer(line);
138
139        // grab length
140        String currLen = st.nextToken();
141
142        // grab count
143        String lengthFreq = st.nextToken();
144
145        int prevNum = num;
146        num += Integer.parseInt(lengthFreq);
147
148        if (medianIndex2 >= prevNum && medianIndex1 <= num) {
149          System.out.println("The median is: " + currLen);
150          br.close();
151          return Double.parseDouble(currLen);
152        } else if (medianIndex2 >= prevNum && medianIndex1 < num) {
153          String nextCurrLen = st.nextToken();
154          double theMedian = (Integer.parseInt(currLen) + Integer
155              .parseInt(nextCurrLen)) / 2.0;
156          System.out.println("The median is: " + theMedian);
157          br.close();
158          return theMedian;
159        }
160      }
161    } finally {
162      if (br != null) {
163        br.close();
164      }
165    }
166    // error, no median found
167    return -1;
168  }
169
170  public static void main(String[] args) throws Exception {
171    ToolRunner.run(new Configuration(), new WordMedian(), args);
172  }
173
174  @Override
175  public int run(String[] args) throws Exception {
176    if (args.length != 2) {
177      System.err.println("Usage: wordmedian <in> <out>");
178      return 0;
179    }
180
181    setConf(new Configuration());
182    Configuration conf = getConf();
183
184    Job job = Job.getInstance(conf, "word median");
185    job.setJarByClass(WordMedian.class);
186    job.setMapperClass(WordMedianMapper.class);
187    job.setCombinerClass(WordMedianReducer.class);
188    job.setReducerClass(WordMedianReducer.class);
189    job.setOutputKeyClass(IntWritable.class);
190    job.setOutputValueClass(IntWritable.class);
191    FileInputFormat.addInputPath(job, new Path(args[0]));
192    FileOutputFormat.setOutputPath(job, new Path(args[1]));
193    boolean result = job.waitForCompletion(true);
194
195    // Wait for JOB 1 -- get middle value to check for Median
196
197    long totalWords = job.getCounters()
198        .getGroup(TaskCounter.class.getCanonicalName())
199        .findCounter("MAP_OUTPUT_RECORDS", "Map output records").getValue();
200    int medianIndex1 = (int) Math.ceil((totalWords / 2.0));
201    int medianIndex2 = (int) Math.floor((totalWords / 2.0));
202
203    median = readAndFindMedian(args[1], medianIndex1, medianIndex2, conf);
204
205    return (result ? 0 : 1);
206  }
207
208  public double getMedian() {
209    return median;
210  }
211}