+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1package org .apache .hadoop .examples ; 2 3/** 4* Licensed to the Apache Software Foundation (ASF) under one 5* or more contributor license agreements. See the NOTICE file 6* distributed with this work for additional information 7* regarding copyright ownership. The ASF licenses this file 8* to you under the Apache License, Version 2.0 (the 9* "License"); you may not use this file except in compliance 10* with the License. You may obtain a copy of the License at 11* 12* http://www.apache.org/licenses/LICENSE-2.0 13* 14* Unless required by applicable law or agreed to in writing, software 15* distributed under the License is distributed on an "AS IS" BASIS, 16* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17* See the License for the specific language governing permissions and 18* limitations under the License. 19*/ 20 21import java .io .BufferedReader ; 22import java .io .IOException ; 23import java .io .InputStreamReader ; 24import java .util .StringTokenizer ; 25 26import org .apache .hadoop .conf .Configuration ; 27import org .apache .hadoop .conf .Configured ; 28import org .apache .hadoop .fs .FileSystem ; 29import org .apache .hadoop .fs .Path ; 30import org .apache .hadoop .io .LongWritable ; 31import org .apache .hadoop .io .Text ; 32import org .apache .hadoop .mapreduce .Job ; 33import org .apache .hadoop .mapreduce .Mapper ; 34import org .apache .hadoop .mapreduce .Reducer ; 35import org .apache .hadoop .mapreduce .lib .input .FileInputFormat ; 36import org .apache .hadoop .mapreduce .lib .output .FileOutputFormat ; 37import org .apache .hadoop .util .Tool ; 38import org .apache .hadoop .util .ToolRunner ; 39 40import com .common .base .Charsets ; 41 42public class WordStandardDeviation extends Configured implements Tool { 43 44private double stddev =0 ; 45 46private final static Text LENGTH =new Text ("length" ); 47private final static Text SQUARE =new Text ("square" ); 48private final static Text COUNT =new Text ("count" ); 49private final static LongWritable ONE =new LongWritable (1 ); 50 51/** 52* Maps words from line of text into 3 key-value pairs; one key-value pair for 53* counting the word, one for counting its length, and one for counting the 54* square of its length. 55*/ 56public static class WordStandardDeviationMapper extends 57Mapper <Object ,Text ,Text ,LongWritable > { 58 59private LongWritable wordLen =new LongWritable (); 60private LongWritable wordLenSq =new LongWritable (); 61 62/** 63* Emits 3 key-value pairs for counting the word, its length, and the 64* squares of its length. Outputs are (Text, LongWritable). 65* 66* @param value 67* This will be a line of text coming in from our input file. 68*/ 69public void map (Object key ,Text value ,Context context ) 70throws IOException ,InterruptedException { 71StringTokenizer itr =new StringTokenizer (value .toString ()); 72while (itr .hasMoreTokens ()) { 73String string =itr .nextToken (); 74 75this .wordLen .set (string .length ()); 76 77// the square of an integer is an integer... 78this .wordLenSq .set ((long )Math .pow (string .length (),2.0 )); 79 80context .write (LENGTH ,this .wordLen ); 81context .write (SQUARE ,this .wordLenSq ); 82context .write (COUNT ,ONE ); 83 } 84 } 85 } 86 87/** 88* Performs integer summation of all the values for each key. 89*/ 90public static class WordStandardDeviationReducer extends 91Reducer <Text ,LongWritable ,Text ,LongWritable > { 92 93private LongWritable val =new LongWritable (); 94 95/** 96* Sums all the individual values within the iterator and writes them to the 97* same key. 98* 99* @param key 100* This will be one of 2 constants: LENGTH_STR, COUNT_STR, or 101* SQUARE_STR. 102* @param values 103* This will be an iterator of all the values associated with that 104* key. 105*/ 106public void reduce (Text key ,Iterable <LongWritable >values ,Context context ) 107throws IOException ,InterruptedException { 108 109int sum =0 ; 110for (LongWritable value :values ) { 111sum +=value .get (); 112 } 113val .set (sum ); 114context .write (key ,val ); 115 } 116 } 117 118/** 119* Reads the output file and parses the summation of lengths, the word count, 120* and the lengths squared, to perform a quick calculation of the standard 121* deviation. 122* 123* @param path 124* The path to find the output file in. Set in main to the output 125* directory. 126* @throws IOException 127* If it cannot access the output directory, we throw an exception. 128*/ 129private double readAndCalcStdDev (Path path ,Configuration conf ) 130throws IOException { 131FileSystem fs =FileSystem .get (conf ); 132Path file =new Path (path ,"part-r-00000" ); 133 134if (!fs .exists (file )) 135throw new IOException ("Output not found!" ); 136 137double stddev =0 ; 138BufferedReader br =null ; 139try { 140br =new BufferedReader (new InputStreamReader (fs .open (file ),Charsets .UTF_8 )); 141long count =0 ; 142long length =0 ; 143long square =0 ; 144String line ; 145while ((line =br .readLine ()) !=null ) { 146StringTokenizer st =new StringTokenizer (line ); 147 148// grab type 149String type =st .nextToken (); 150 151// differentiate 152if (type .equals (COUNT .toString ())) { 153String countLit =st .nextToken (); 154count =Long .parseLong (countLit ); 155 }else if (type .equals (LENGTH .toString ())) { 156String lengthLit =st .nextToken (); 157length =Long .parseLong (lengthLit ); 158 }else if (type .equals (SQUARE .toString ())) { 159String squareLit =st .nextToken (); 160square =Long .parseLong (squareLit ); 161 } 162 } 163// average = total sum / number of elements; 164double mean = (((double )length ) / ((double )count )); 165// standard deviation = sqrt((sum(lengths ^ 2)/count) - (mean ^ 2)) 166mean =Math .pow (mean ,2.0 ); 167double term = (((double )square / ((double )count ))); 168stddev =Math .sqrt ((term -mean )); 169System .out .println ("The standard deviation is: " +stddev ); 170 }finally { 171if (br !=null ) { 172br .close (); 173 } 174 } 175return stddev ; 176 } 177 178public static void main (String []args )throws Exception { 179ToolRunner .run (new Configuration (),new WordStandardDeviation (), 180args ); 181 } 182 183@ Override 184public int run (String []args )throws Exception { 185if (args .length !=2 ) { 186System .err .println ("Usage: wordstddev <in> <out>" ); 187return 0 ; 188 } 189 190Configuration conf =getConf (); 191 192Job job =Job .getInstance (conf ,"word stddev" ); 193job .setJarByClass (WordStandardDeviation .class ); 194job .setMapperClass (WordStandardDeviationMapper .class ); 195job .setCombinerClass (WordStandardDeviationReducer .class ); 196job .setReducerClass (WordStandardDeviationReducer .class ); 197job .setOutputKeyClass (Text .class ); 198job .setOutputValueClass (LongWritable .class ); 199FileInputFormat .addInputPath (job ,new Path (args [0 ])); 200Path outputpath =new Path (args [1 ]); 201FileOutputFormat .setOutputPath (job ,outputpath ); 202boolean result =job .waitForCompletion (true ); 203 204// read output and calculate standard deviation 205stddev =readAndCalcStdDev (outputpath ,conf ); 206 207return (result ?0 :1 ); 208 } 209 210public double getStandardDeviation () { 211return stddev ; 212 } 213}