+/EE360P

distributed systems

git clone https://git.julia.blue/+/EE360P

vijaygarg1init07ca65d

master
6.6 KiB189 linesraw
1/**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements.  See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership.  The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License.  You may obtain a copy of the License at
9 *
10 *     http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19package org.apache.hadoop.examples.terasort;
20
21import java.io.IOException;
22import java.util.zip.Checksum;
23
24import org.apache.hadoop.conf.Configuration;
25import org.apache.hadoop.conf.Configured;
26import org.apache.hadoop.fs.Path;
27import org.apache.hadoop.io.BytesWritable;
28import org.apache.hadoop.io.Text;
29import org.apache.hadoop.mapreduce.Cluster;
30import org.apache.hadoop.mapreduce.Job;
31import org.apache.hadoop.mapreduce.Mapper;
32import org.apache.hadoop.mapreduce.Reducer;
33import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
34import org.apache.hadoop.mapreduce.lib.input.FileSplit;
35import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
36import org.apache.hadoop.util.PureJavaCrc32;
37import org.apache.hadoop.util.Tool;
38import org.apache.hadoop.util.ToolRunner;
39
40/**
41 * Generate 1 mapper per a file that checks to make sure the keys
42 * are sorted within each file. The mapper also generates 
43 * "$file:begin", first key and "$file:end", last key. The reduce verifies that
44 * all of the start/end items are in order.
45 * Any output from the reduce is problem report.
46 * <p>
47 * To run the program: 
48 * <b>bin/hadoop jar hadoop-*-examples.jar teravalidate out-dir report-dir</b>
49 * <p>
50 * If there is any output, something is wrong and the output of the reduce
51 * will have the problem report.
52 */
53public class TeraValidate extends Configured implements Tool {
54  private static final Text ERROR = new Text("error");
55  private static final Text CHECKSUM = new Text("checksum");
56  
57  private static String textifyBytes(Text t) {
58    BytesWritable b = new BytesWritable();
59    b.set(t.getBytes(), 0, t.getLength());
60    return b.toString();
61  }
62
63  static class ValidateMapper extends Mapper<Text,Text,Text,Text> {
64    private Text lastKey;
65    private String filename;
66    private Unsigned16 checksum = new Unsigned16();
67    private Unsigned16 tmp = new Unsigned16();
68    private Checksum crc32 = new PureJavaCrc32();
69
70    /**
71     * Get the final part of the input name
72     * @param split the input split
73     * @return the "part-r-00000" for the input
74     */
75    private String getFilename(FileSplit split) {
76      return split.getPath().getName();
77    }
78
79    public void map(Text key, Text value, Context context) 
80        throws IOException, InterruptedException {
81      if (lastKey == null) {
82        FileSplit fs = (FileSplit) context.getInputSplit();
83        filename = getFilename(fs);
84        context.write(new Text(filename + ":begin"), key);
85        lastKey = new Text();
86      } else {
87        if (key.compareTo(lastKey) < 0) {
88          context.write(ERROR, new Text("misorder in " + filename + 
89                                         " between " + textifyBytes(lastKey) + 
90                                         " and " + textifyBytes(key)));
91        }
92      }
93      // compute the crc of the key and value and add it to the sum
94      crc32.reset();
95      crc32.update(key.getBytes(), 0, key.getLength());
96      crc32.update(value.getBytes(), 0, value.getLength());
97      tmp.set(crc32.getValue());
98      checksum.add(tmp);
99      lastKey.set(key);
100    }
101    
102    public void cleanup(Context context) 
103        throws IOException, InterruptedException  {
104      if (lastKey != null) {
105        context.write(new Text(filename + ":end"), lastKey);
106        context.write(CHECKSUM, new Text(checksum.toString()));
107      }
108    }
109  }
110
111  /**
112   * Check the boundaries between the output files by making sure that the
113   * boundary keys are always increasing.
114   * Also passes any error reports along intact.
115   */
116  static class ValidateReducer extends Reducer<Text,Text,Text,Text> {
117    private boolean firstKey = true;
118    private Text lastKey = new Text();
119    private Text lastValue = new Text();
120    public void reduce(Text key, Iterable<Text> values,
121        Context context) throws IOException, InterruptedException  {
122      if (ERROR.equals(key)) {
123        for (Text val : values) {
124          context.write(key, val);
125        }
126      } else if (CHECKSUM.equals(key)) {
127        Unsigned16 tmp = new Unsigned16();
128        Unsigned16 sum = new Unsigned16();
129        for (Text val : values) {
130          tmp.set(val.toString());
131          sum.add(tmp);
132        }
133        context.write(CHECKSUM, new Text(sum.toString()));
134      } else {
135        Text value = values.iterator().next();
136        if (firstKey) {
137          firstKey = false;
138        } else {
139          if (value.compareTo(lastValue) < 0) {
140            context.write(ERROR, 
141                           new Text("bad key partitioning:\n  file " + 
142                                    lastKey + " key " + 
143                                    textifyBytes(lastValue) +
144                                    "\n  file " + key + " key " + 
145                                    textifyBytes(value)));
146          }
147        }
148        lastKey.set(key);
149        lastValue.set(value);
150      }
151    }
152    
153  }
154
155  private static void usage() throws IOException {
156    System.err.println("teravalidate <out-dir> <report-dir>");
157  }
158
159  public int run(String[] args) throws Exception {
160    Job job = Job.getInstance(getConf());
161    if (args.length != 2) {
162      usage();
163      return 1;
164    }
165    TeraInputFormat.setInputPaths(job, new Path(args[0]));
166    FileOutputFormat.setOutputPath(job, new Path(args[1]));
167    job.setJobName("TeraValidate");
168    job.setJarByClass(TeraValidate.class);
169    job.setMapperClass(ValidateMapper.class);
170    job.setReducerClass(ValidateReducer.class);
171    job.setOutputKeyClass(Text.class);
172    job.setOutputValueClass(Text.class);
173    // force a single reducer
174    job.setNumReduceTasks(1);
175    // force a single split 
176    FileInputFormat.setMinInputSplitSize(job, Long.MAX_VALUE);
177    job.setInputFormatClass(TeraInputFormat.class);
178    return job.waitForCompletion(true) ? 0 : 1;
179  }
180
181  /**
182   * @param args
183   */
184  public static void main(String[] args) throws Exception {
185    int res = ToolRunner.run(new Configuration(), new TeraValidate(), args);
186    System.exit(res);
187  }
188
189}