+/EE360P

distributed systems

git clone https://git.julia.blue/+/EE360P

vijaygarg1init07ca65d

master
7.9 KiB261 linesraw
1/**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements.  See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership.  The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License.  You may obtain a copy of the License at
9 *
10 *     http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19package org.apache.hadoop.examples;
20
21import java.io.DataInput;
22import java.io.DataOutput;
23import java.io.IOException;
24import java.util.StringTokenizer;
25
26import org.apache.hadoop.conf.Configured;
27import org.apache.hadoop.fs.FSDataInputStream;
28import org.apache.hadoop.fs.FileSystem;
29import org.apache.hadoop.fs.Path;
30import org.apache.hadoop.io.IntWritable;
31import org.apache.hadoop.io.Text;
32import org.apache.hadoop.io.WritableComparable;
33import org.apache.hadoop.mapreduce.InputSplit;
34import org.apache.hadoop.mapreduce.Job;
35import org.apache.hadoop.mapreduce.Mapper;
36import org.apache.hadoop.mapreduce.RecordReader;
37import org.apache.hadoop.mapreduce.TaskAttemptContext;
38import org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat;
39import org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReader;
40import org.apache.hadoop.mapreduce.lib.input.CombineFileSplit;
41import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
42import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
43import org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer;
44import org.apache.hadoop.util.LineReader;
45import org.apache.hadoop.util.Tool;
46import org.apache.hadoop.util.ToolRunner;
47
48
49/**
50 * MultiFileWordCount is an example to demonstrate the usage of 
51 * MultiFileInputFormat. This examples counts the occurrences of
52 * words in the text files under the given input directory.
53 */
54public class MultiFileWordCount extends Configured implements Tool {
55
56  /**
57   * This record keeps <filename,offset> pairs.
58   */
59  public static class WordOffset implements WritableComparable {
60
61    private long offset;
62    private String fileName;
63
64    public void readFields(DataInput in) throws IOException {
65      this.offset = in.readLong();
66      this.fileName = Text.readString(in);
67    }
68
69    public void write(DataOutput out) throws IOException {
70      out.writeLong(offset);
71      Text.writeString(out, fileName);
72    }
73
74    public int compareTo(Object o) {
75      WordOffset that = (WordOffset)o;
76
77      int f = this.fileName.compareTo(that.fileName);
78      if(f == 0) {
79        return (int)Math.signum((double)(this.offset - that.offset));
80      }
81      return f;
82    }
83    @Override
84    public boolean equals(Object obj) {
85      if(obj instanceof WordOffset)
86        return this.compareTo(obj) == 0;
87      return false;
88    }
89    @Override
90    public int hashCode() {
91      assert false : "hashCode not designed";
92      return 42; //an arbitrary constant
93    }
94  }
95
96
97  /**
98   * To use {@link CombineFileInputFormat}, one should extend it, to return a 
99   * (custom) {@link RecordReader}. CombineFileInputFormat uses 
100   * {@link CombineFileSplit}s. 
101   */
102  public static class MyInputFormat 
103    extends CombineFileInputFormat<WordOffset, Text>  {
104
105    public RecordReader<WordOffset,Text> createRecordReader(InputSplit split,
106        TaskAttemptContext context) throws IOException {
107      return new CombineFileRecordReader<WordOffset, Text>(
108        (CombineFileSplit)split, context, CombineFileLineRecordReader.class);
109    }
110  }
111
112  /**
113   * RecordReader is responsible from extracting records from a chunk
114   * of the CombineFileSplit. 
115   */
116  public static class CombineFileLineRecordReader 
117    extends RecordReader<WordOffset, Text> {
118
119    private long startOffset; //offset of the chunk;
120    private long end; //end of the chunk;
121    private long pos; // current pos 
122    private FileSystem fs;
123    private Path path;
124    private WordOffset key;
125    private Text value;
126    
127    private FSDataInputStream fileIn;
128    private LineReader reader;
129    
130    public CombineFileLineRecordReader(CombineFileSplit split,
131        TaskAttemptContext context, Integer index) throws IOException {
132      
133      this.path = split.getPath(index);
134      fs = this.path.getFileSystem(context.getConfiguration());
135      this.startOffset = split.getOffset(index);
136      this.end = startOffset + split.getLength(index);
137      boolean skipFirstLine = false;
138      
139      //open the file
140      fileIn = fs.open(path);
141      if (startOffset != 0) {
142        skipFirstLine = true;
143        --startOffset;
144        fileIn.seek(startOffset);
145      }
146      reader = new LineReader(fileIn);
147      if (skipFirstLine) {  // skip first line and re-establish "startOffset".
148        startOffset += reader.readLine(new Text(), 0,
149                    (int)Math.min((long)Integer.MAX_VALUE, end - startOffset));
150      }
151      this.pos = startOffset;
152    }
153
154    public void initialize(InputSplit split, TaskAttemptContext context)
155        throws IOException, InterruptedException {
156    }
157
158    public void close() throws IOException { }
159
160    public float getProgress() throws IOException {
161      if (startOffset == end) {
162        return 0.0f;
163      } else {
164        return Math.min(1.0f, (pos - startOffset) / (float)(end - startOffset));
165      }
166    }
167
168    public boolean nextKeyValue() throws IOException {
169      if (key == null) {
170        key = new WordOffset();
171        key.fileName = path.getName();
172      }
173      key.offset = pos;
174      if (value == null) {
175        value = new Text();
176      }
177      int newSize = 0;
178      if (pos < end) {
179        newSize = reader.readLine(value);
180        pos += newSize;
181      }
182      if (newSize == 0) {
183        key = null;
184        value = null;
185        return false;
186      } else {
187        return true;
188      }
189    }
190
191    public WordOffset getCurrentKey() 
192        throws IOException, InterruptedException {
193      return key;
194    }
195
196    public Text getCurrentValue() throws IOException, InterruptedException {
197      return value;
198    }
199  }
200
201  /**
202   * This Mapper is similar to the one in {@link WordCount.TokenizerMapper}.
203   */
204  public static class MapClass extends 
205      Mapper<WordOffset, Text, Text, IntWritable> {
206    private final static IntWritable one = new IntWritable(1);
207    private Text word = new Text();
208    
209    public void map(WordOffset key, Text value, Context context)
210        throws IOException, InterruptedException {
211      
212      String line = value.toString();
213      StringTokenizer itr = new StringTokenizer(line);
214      while (itr.hasMoreTokens()) {
215        word.set(itr.nextToken());
216        context.write(word, one);
217      }
218    }
219  }
220  
221  private void printUsage() {
222    System.out.println("Usage : multifilewc <input_dir> <output>" );
223  }
224
225  public int run(String[] args) throws Exception {
226
227    if(args.length < 2) {
228      printUsage();
229      return 2;
230    }
231
232    Job job = Job.getInstance(getConf());
233    job.setJobName("MultiFileWordCount");
234    job.setJarByClass(MultiFileWordCount.class);
235
236    //set the InputFormat of the job to our InputFormat
237    job.setInputFormatClass(MyInputFormat.class);
238    
239    // the keys are words (strings)
240    job.setOutputKeyClass(Text.class);
241    // the values are counts (ints)
242    job.setOutputValueClass(IntWritable.class);
243
244    //use the defined mapper
245    job.setMapperClass(MapClass.class);
246    //use the WordCount Reducer
247    job.setCombinerClass(IntSumReducer.class);
248    job.setReducerClass(IntSumReducer.class);
249
250    FileInputFormat.addInputPaths(job, args[0]);
251    FileOutputFormat.setOutputPath(job, new Path(args[1]));
252
253    return job.waitForCompletion(true) ? 0 : 1;
254  }
255
256  public static void main(String[] args) throws Exception {
257    int ret = ToolRunner.run(new MultiFileWordCount(), args);
258    System.exit(ret);
259  }
260
261}