+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1package org .apache .hadoop .examples ; 2 3/** 4* Licensed to the Apache Software Foundation (ASF) under one 5* or more contributor license agreements. See the NOTICE file 6* distributed with this work for additional information 7* regarding copyright ownership. The ASF licenses this file 8* to you under the Apache License, Version 2.0 (the 9* "License"); you may not use this file except in compliance 10* with the License. You may obtain a copy of the License at 11* 12* http://www.apache.org/licenses/LICENSE-2.0 13* 14* Unless required by applicable law or agreed to in writing, software 15* distributed under the License is distributed on an "AS IS" BASIS, 16* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17* See the License for the specific language governing permissions and 18* limitations under the License. 19*/ 20 21import java .io .BufferedReader ; 22import java .io .IOException ; 23import java .io .InputStreamReader ; 24import java .util .StringTokenizer ; 25 26import org .apache .hadoop .conf .Configuration ; 27import org .apache .hadoop .conf .Configured ; 28import org .apache .hadoop .fs .FileSystem ; 29import org .apache .hadoop .fs .Path ; 30import org .apache .hadoop .io .IntWritable ; 31import org .apache .hadoop .io .Text ; 32import org .apache .hadoop .mapreduce .Job ; 33import org .apache .hadoop .mapreduce .Mapper ; 34import org .apache .hadoop .mapreduce .Reducer ; 35import org .apache .hadoop .mapreduce .TaskCounter ; 36import org .apache .hadoop .mapreduce .lib .input .FileInputFormat ; 37import org .apache .hadoop .mapreduce .lib .output .FileOutputFormat ; 38import org .apache .hadoop .util .Tool ; 39import org .apache .hadoop .util .ToolRunner ; 40 41import com .common .base .Charsets ; 42 43public class WordMedian extends Configured implements Tool { 44 45private double median =0 ; 46private final static IntWritable ONE =new IntWritable (1 ); 47 48/** 49* Maps words from line of text into a key-value pair; the length of the word 50* as the key, and 1 as the value. 51*/ 52public static class WordMedianMapper extends 53Mapper <Object ,Text ,IntWritable ,IntWritable > { 54 55private IntWritable length =new IntWritable (); 56 57/** 58* Emits a key-value pair for counting the word. Outputs are (IntWritable, 59* IntWritable). 60* 61* @param value 62* This will be a line of text coming in from our input file. 63*/ 64public void map (Object key ,Text value ,Context context ) 65throws IOException ,InterruptedException { 66StringTokenizer itr =new StringTokenizer (value .toString ()); 67while (itr .hasMoreTokens ()) { 68String string =itr .nextToken (); 69length .set (string .length ()); 70context .write (length ,ONE ); 71 } 72 } 73 } 74 75/** 76* Performs integer summation of all the values for each key. 77*/ 78public static class WordMedianReducer extends 79Reducer <IntWritable ,IntWritable ,IntWritable ,IntWritable > { 80 81private IntWritable val =new IntWritable (); 82 83/** 84* Sums all the individual values within the iterator and writes them to the 85* same key. 86* 87* @param key 88* This will be a length of a word that was read. 89* @param values 90* This will be an iterator of all the values associated with that 91* key. 92*/ 93public void reduce (IntWritable key ,Iterable <IntWritable >values , 94Context context )throws IOException ,InterruptedException { 95 96int sum =0 ; 97for (IntWritable value :values ) { 98sum +=value .get (); 99 } 100val .set (sum ); 101context .write (key ,val ); 102 } 103 } 104 105/** 106* This is a standard program to read and find a median value based on a file 107* of word counts such as: 1 456, 2 132, 3 56... Where the first values are 108* the word lengths and the following values are the number of times that 109* words of that length appear. 110* 111* @param path 112* The path to read the HDFS file from (part-r-00000...00001...etc). 113* @param medianIndex1 114* The first length value to look for. 115* @param medianIndex2 116* The second length value to look for (will be the same as the first 117* if there are an even number of words total). 118* @throws IOException 119* If file cannot be found, we throw an exception. 120* */ 121private double readAndFindMedian (String path ,int medianIndex1 , 122int medianIndex2 ,Configuration conf )throws IOException { 123FileSystem fs =FileSystem .get (conf ); 124Path file =new Path (path ,"part-r-00000" ); 125 126if (!fs .exists (file )) 127throw new IOException ("Output not found!" ); 128 129BufferedReader br =null ; 130 131try { 132br =new BufferedReader (new InputStreamReader (fs .open (file ),Charsets .UTF_8 )); 133int num =0 ; 134 135String line ; 136while ((line =br .readLine ()) !=null ) { 137StringTokenizer st =new StringTokenizer (line ); 138 139// grab length 140String currLen =st .nextToken (); 141 142// grab count 143String lengthFreq =st .nextToken (); 144 145int prevNum =num ; 146num +=Integer .parseInt (lengthFreq ); 147 148if (medianIndex2 >=prevNum &&medianIndex1 <=num ) { 149System .out .println ("The median is: " +currLen ); 150br .close (); 151return Double .parseDouble (currLen ); 152 }else if (medianIndex2 >=prevNum &&medianIndex1 <num ) { 153String nextCurrLen =st .nextToken (); 154double theMedian = (Integer .parseInt (currLen ) +Integer 155 .parseInt (nextCurrLen )) /2.0 ; 156System .out .println ("The median is: " +theMedian ); 157br .close (); 158return theMedian ; 159 } 160 } 161 }finally { 162if (br !=null ) { 163br .close (); 164 } 165 } 166// error, no median found 167return -1 ; 168 } 169 170public static void main (String []args )throws Exception { 171ToolRunner .run (new Configuration (),new WordMedian (),args ); 172 } 173 174@ Override 175public int run (String []args )throws Exception { 176if (args .length !=2 ) { 177System .err .println ("Usage: wordmedian <in> <out>" ); 178return 0 ; 179 } 180 181setConf (new Configuration ()); 182Configuration conf =getConf (); 183 184Job job =Job .getInstance (conf ,"word median" ); 185job .setJarByClass (WordMedian .class ); 186job .setMapperClass (WordMedianMapper .class ); 187job .setCombinerClass (WordMedianReducer .class ); 188job .setReducerClass (WordMedianReducer .class ); 189job .setOutputKeyClass (IntWritable .class ); 190job .setOutputValueClass (IntWritable .class ); 191FileInputFormat .addInputPath (job ,new Path (args [0 ])); 192FileOutputFormat .setOutputPath (job ,new Path (args [1 ])); 193boolean result =job .waitForCompletion (true ); 194 195// Wait for JOB 1 -- get middle value to check for Median 196 197long totalWords =job .getCounters () 198 .getGroup (TaskCounter .class .getCanonicalName ()) 199 .findCounter ("MAP_OUTPUT_RECORDS" ,"Map output records" ).getValue (); 200int medianIndex1 = (int )Math .ceil ((totalWords /2.0 )); 201int medianIndex2 = (int )Math .floor ((totalWords /2.0 )); 202 203median =readAndFindMedian (args [1 ],medianIndex1 ,medianIndex2 ,conf ); 204 205return (result ?0 :1 ); 206 } 207 208public double getMedian () { 209return median ; 210 } 211}