+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1package org .apache .hadoop .examples ; 2 3/** 4* Licensed to the Apache Software Foundation (ASF) under one 5* or more contributor license agreements. See the NOTICE file 6* distributed with this work for additional information 7* regarding copyright ownership. The ASF licenses this file 8* to you under the Apache License, Version 2.0 (the 9* "License"); you may not use this file except in compliance 10* with the License. You may obtain a copy of the License at 11* 12* http://www.apache.org/licenses/LICENSE-2.0 13* 14* Unless required by applicable law or agreed to in writing, software 15* distributed under the License is distributed on an "AS IS" BASIS, 16* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17* See the License for the specific language governing permissions and 18* limitations under the License. 19*/ 20 21import java .io .BufferedReader ; 22import java .io .IOException ; 23import java .io .InputStreamReader ; 24import java .util .StringTokenizer ; 25 26import org .apache .hadoop .conf .Configuration ; 27import org .apache .hadoop .conf .Configured ; 28import org .apache .hadoop .fs .FileSystem ; 29import org .apache .hadoop .fs .Path ; 30import org .apache .hadoop .io .LongWritable ; 31import org .apache .hadoop .io .Text ; 32import org .apache .hadoop .mapreduce .Job ; 33import org .apache .hadoop .mapreduce .Mapper ; 34import org .apache .hadoop .mapreduce .Reducer ; 35import org .apache .hadoop .mapreduce .lib .input .FileInputFormat ; 36import org .apache .hadoop .mapreduce .lib .output .FileOutputFormat ; 37import org .apache .hadoop .util .Tool ; 38import org .apache .hadoop .util .ToolRunner ; 39 40import com .common .base .Charsets ; 41 42public class WordMean extends Configured implements Tool { 43 44private double mean =0 ; 45 46private final static Text COUNT =new Text ("count" ); 47private final static Text LENGTH =new Text ("length" ); 48private final static LongWritable ONE =new LongWritable (1 ); 49 50/** 51* Maps words from line of text into 2 key-value pairs; one key-value pair for 52* counting the word, another for counting its length. 53*/ 54public static class WordMeanMapper extends 55Mapper <Object ,Text ,Text ,LongWritable > { 56 57private LongWritable wordLen =new LongWritable (); 58 59/** 60* Emits 2 key-value pairs for counting the word and its length. Outputs are 61* (Text, LongWritable). 62* 63* @param value 64* This will be a line of text coming in from our input file. 65*/ 66public void map (Object key ,Text value ,Context context ) 67throws IOException ,InterruptedException { 68StringTokenizer itr =new StringTokenizer (value .toString ()); 69while (itr .hasMoreTokens ()) { 70String string =itr .nextToken (); 71this .wordLen .set (string .length ()); 72context .write (LENGTH ,this .wordLen ); 73context .write (COUNT ,ONE ); 74 } 75 } 76 } 77 78/** 79* Performs integer summation of all the values for each key. 80*/ 81public static class WordMeanReducer extends 82Reducer <Text ,LongWritable ,Text ,LongWritable > { 83 84private LongWritable sum =new LongWritable (); 85 86/** 87* Sums all the individual values within the iterator and writes them to the 88* same key. 89* 90* @param key 91* This will be one of 2 constants: LENGTH_STR or COUNT_STR. 92* @param values 93* This will be an iterator of all the values associated with that 94* key. 95*/ 96public void reduce (Text key ,Iterable <LongWritable >values ,Context context ) 97throws IOException ,InterruptedException { 98 99int theSum =0 ; 100for (LongWritable val :values ) { 101theSum +=val .get (); 102 } 103sum .set (theSum ); 104context .write (key ,sum ); 105 } 106 } 107 108/** 109* Reads the output file and parses the summation of lengths, and the word 110* count, to perform a quick calculation of the mean. 111* 112* @param path 113* The path to find the output file in. Set in main to the output 114* directory. 115* @throws IOException 116* If it cannot access the output directory, we throw an exception. 117*/ 118private double readAndCalcMean (Path path ,Configuration conf ) 119throws IOException { 120FileSystem fs =FileSystem .get (conf ); 121Path file =new Path (path ,"part-r-00000" ); 122 123if (!fs .exists (file )) 124throw new IOException ("Output not found!" ); 125 126BufferedReader br =null ; 127 128// average = total sum / number of elements; 129try { 130br =new BufferedReader (new InputStreamReader (fs .open (file ),Charsets .UTF_8 )); 131 132long count =0 ; 133long length =0 ; 134 135String line ; 136while ((line =br .readLine ()) !=null ) { 137StringTokenizer st =new StringTokenizer (line ); 138 139// grab type 140String type =st .nextToken (); 141 142// differentiate 143if (type .equals (COUNT .toString ())) { 144String countLit =st .nextToken (); 145count =Long .parseLong (countLit ); 146 }else if (type .equals (LENGTH .toString ())) { 147String lengthLit =st .nextToken (); 148length =Long .parseLong (lengthLit ); 149 } 150 } 151 152double theMean = (((double )length ) / ((double )count )); 153System .out .println ("The mean is: " +theMean ); 154return theMean ; 155 }finally { 156if (br !=null ) { 157br .close (); 158 } 159 } 160 } 161 162public static void main (String []args )throws Exception { 163ToolRunner .run (new Configuration (),new WordMean (),args ); 164 } 165 166@ Override 167public int run (String []args )throws Exception { 168if (args .length !=2 ) { 169System .err .println ("Usage: wordmean <in> <out>" ); 170return 0 ; 171 } 172 173Configuration conf =getConf (); 174 175Job job =Job .getInstance (conf ,"word mean" ); 176job .setJarByClass (WordMean .class ); 177job .setMapperClass (WordMeanMapper .class ); 178job .setCombinerClass (WordMeanReducer .class ); 179job .setReducerClass (WordMeanReducer .class ); 180job .setOutputKeyClass (Text .class ); 181job .setOutputValueClass (LongWritable .class ); 182FileInputFormat .addInputPath (job ,new Path (args [0 ])); 183Path outputpath =new Path (args [1 ]); 184FileOutputFormat .setOutputPath (job ,outputpath ); 185boolean result =job .waitForCompletion (true ); 186mean =readAndCalcMean (outputpath ,conf ); 187 188return (result ?0 :1 ); 189 } 190 191/** 192* Only valuable after run() called. 193* 194* @return Returns the mean value. 195*/ 196public double getMean () { 197return mean ; 198 } 199}