+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18 19package org .apache .hadoop .examples ; 20 21import java .io .DataInput ; 22import java .io .DataOutput ; 23import java .io .IOException ; 24import java .util .StringTokenizer ; 25 26import org .apache .hadoop .conf .Configured ; 27import org .apache .hadoop .fs .FSDataInputStream ; 28import org .apache .hadoop .fs .FileSystem ; 29import org .apache .hadoop .fs .Path ; 30import org .apache .hadoop .io .IntWritable ; 31import org .apache .hadoop .io .Text ; 32import org .apache .hadoop .io .WritableComparable ; 33import org .apache .hadoop .mapreduce .InputSplit ; 34import org .apache .hadoop .mapreduce .Job ; 35import org .apache .hadoop .mapreduce .Mapper ; 36import org .apache .hadoop .mapreduce .RecordReader ; 37import org .apache .hadoop .mapreduce .TaskAttemptContext ; 38import org .apache .hadoop .mapreduce .lib .input .CombineFileInputFormat ; 39import org .apache .hadoop .mapreduce .lib .input .CombineFileRecordReader ; 40import org .apache .hadoop .mapreduce .lib .input .CombineFileSplit ; 41import org .apache .hadoop .mapreduce .lib .input .FileInputFormat ; 42import org .apache .hadoop .mapreduce .lib .output .FileOutputFormat ; 43import org .apache .hadoop .mapreduce .lib .reduce .IntSumReducer ; 44import org .apache .hadoop .util .LineReader ; 45import org .apache .hadoop .util .Tool ; 46import org .apache .hadoop .util .ToolRunner ; 47 48 49/** 50* MultiFileWordCount is an example to demonstrate the usage of 51* MultiFileInputFormat. This examples counts the occurrences of 52* words in the text files under the given input directory. 53*/ 54public class MultiFileWordCount extends Configured implements Tool { 55 56/** 57* This record keeps <filename,offset> pairs. 58*/ 59public static class WordOffset implements WritableComparable { 60 61private long offset ; 62private String fileName ; 63 64public void readFields (DataInput in )throws IOException { 65this .offset =in .readLong (); 66this .fileName =Text .readString (in ); 67 } 68 69public void write (DataOutput out )throws IOException { 70out .writeLong (offset ); 71Text .writeString (out ,fileName ); 72 } 73 74public int compareTo (Object o ) { 75WordOffset that = (WordOffset )o ; 76 77int f =this .fileName .compareTo (that .fileName ); 78if (f ==0 ) { 79return (int )Math .signum ((double )(this .offset -that .offset )); 80 } 81return f ; 82 } 83@ Override 84public boolean equals (Object obj ) { 85if (obj instanceof WordOffset ) 86return this .compareTo (obj ) ==0 ; 87return false ; 88 } 89@ Override 90public int hashCode () { 91assert false :"hashCode not designed" ; 92return 42 ;//an arbitrary constant 93 } 94 } 95 96 97/** 98* To use {@link CombineFileInputFormat}, one should extend it, to return a 99* (custom) {@link RecordReader}. CombineFileInputFormat uses 100* {@link CombineFileSplit}s. 101*/ 102public static class MyInputFormat 103extends CombineFileInputFormat <WordOffset ,Text > { 104 105public RecordReader <WordOffset ,Text >createRecordReader (InputSplit split , 106TaskAttemptContext context )throws IOException { 107return new CombineFileRecordReader <WordOffset ,Text >( 108 (CombineFileSplit )split ,context ,CombineFileLineRecordReader .class ); 109 } 110 } 111 112/** 113* RecordReader is responsible from extracting records from a chunk 114* of the CombineFileSplit. 115*/ 116public static class CombineFileLineRecordReader 117extends RecordReader <WordOffset ,Text > { 118 119private long startOffset ;//offset of the chunk; 120private long end ;//end of the chunk; 121private long pos ;// current pos 122private FileSystem fs ; 123private Path path ; 124private WordOffset key ; 125private Text value ; 126 127private FSDataInputStream fileIn ; 128private LineReader reader ; 129 130public CombineFileLineRecordReader (CombineFileSplit split , 131TaskAttemptContext context ,Integer index )throws IOException { 132 133this .path =split .getPath (index ); 134fs =this .path .getFileSystem (context .getConfiguration ()); 135this .startOffset =split .getOffset (index ); 136this .end =startOffset +split .getLength (index ); 137boolean skipFirstLine =false ; 138 139//open the file 140fileIn =fs .open (path ); 141if (startOffset !=0 ) { 142skipFirstLine =true ; 143 --startOffset ; 144fileIn .seek (startOffset ); 145 } 146reader =new LineReader (fileIn ); 147if (skipFirstLine ) {// skip first line and re-establish "startOffset". 148startOffset +=reader .readLine (new Text (),0 , 149 (int )Math .min ((long )Integer .MAX_VALUE ,end -startOffset )); 150 } 151this .pos =startOffset ; 152 } 153 154public void initialize (InputSplit split ,TaskAttemptContext context ) 155throws IOException ,InterruptedException { 156 } 157 158public void close ()throws IOException { } 159 160public float getProgress ()throws IOException { 161if (startOffset ==end ) { 162return 0.0f ; 163 }else { 164return Math .min (1.0f , (pos -startOffset ) / (float )(end -startOffset )); 165 } 166 } 167 168public boolean nextKeyValue ()throws IOException { 169if (key ==null ) { 170key =new WordOffset (); 171key .fileName =path .getName (); 172 } 173key .offset =pos ; 174if (value ==null ) { 175value =new Text (); 176 } 177int newSize =0 ; 178if (pos <end ) { 179newSize =reader .readLine (value ); 180pos +=newSize ; 181 } 182if (newSize ==0 ) { 183key =null ; 184value =null ; 185return false ; 186 }else { 187return true ; 188 } 189 } 190 191public WordOffset getCurrentKey () 192throws IOException ,InterruptedException { 193return key ; 194 } 195 196public Text getCurrentValue ()throws IOException ,InterruptedException { 197return value ; 198 } 199 } 200 201/** 202* This Mapper is similar to the one in {@link WordCount.TokenizerMapper}. 203*/ 204public static class MapClass extends 205Mapper <WordOffset ,Text ,Text ,IntWritable > { 206private final static IntWritable one =new IntWritable (1 ); 207private Text word =new Text (); 208 209public void map (WordOffset key ,Text value ,Context context ) 210throws IOException ,InterruptedException { 211 212String line =value .toString (); 213StringTokenizer itr =new StringTokenizer (line ); 214while (itr .hasMoreTokens ()) { 215word .set (itr .nextToken ()); 216context .write (word ,one ); 217 } 218 } 219 } 220 221private void printUsage () { 222System .out .println ("Usage : multifilewc <input_dir> <output>" ); 223 } 224 225public int run (String []args )throws Exception { 226 227if (args .length <2 ) { 228printUsage (); 229return 2 ; 230 } 231 232Job job =Job .getInstance (getConf ()); 233job .setJobName ("MultiFileWordCount" ); 234job .setJarByClass (MultiFileWordCount .class ); 235 236//set the InputFormat of the job to our InputFormat 237job .setInputFormatClass (MyInputFormat .class ); 238 239// the keys are words (strings) 240job .setOutputKeyClass (Text .class ); 241// the values are counts (ints) 242job .setOutputValueClass (IntWritable .class ); 243 244//use the defined mapper 245job .setMapperClass (MapClass .class ); 246//use the WordCount Reducer 247job .setCombinerClass (IntSumReducer .class ); 248job .setReducerClass (IntSumReducer .class ); 249 250FileInputFormat .addInputPaths (job ,args [0 ]); 251FileOutputFormat .setOutputPath (job ,new Path (args [1 ])); 252 253return job .waitForCompletion (true ) ?0 :1 ; 254 } 255 256public static void main (String []args )throws Exception { 257int ret =ToolRunner .run (new MultiFileWordCount (),args ); 258System .exit (ret ); 259 } 260 261}