+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18 19package org .apache .hadoop .examples .terasort ; 20 21import java .io .IOException ; 22import java .util .zip .Checksum ; 23 24import org .apache .hadoop .conf .Configuration ; 25import org .apache .hadoop .conf .Configured ; 26import org .apache .hadoop .fs .Path ; 27import org .apache .hadoop .io .BytesWritable ; 28import org .apache .hadoop .io .Text ; 29import org .apache .hadoop .mapreduce .Cluster ; 30import org .apache .hadoop .mapreduce .Job ; 31import org .apache .hadoop .mapreduce .Mapper ; 32import org .apache .hadoop .mapreduce .Reducer ; 33import org .apache .hadoop .mapreduce .lib .input .FileInputFormat ; 34import org .apache .hadoop .mapreduce .lib .input .FileSplit ; 35import org .apache .hadoop .mapreduce .lib .output .FileOutputFormat ; 36import org .apache .hadoop .util .PureJavaCrc32 ; 37import org .apache .hadoop .util .Tool ; 38import org .apache .hadoop .util .ToolRunner ; 39 40/** 41* Generate 1 mapper per a file that checks to make sure the keys 42* are sorted within each file. The mapper also generates 43* "$file:begin", first key and "$file:end", last key. The reduce verifies that 44* all of the start/end items are in order. 45* Any output from the reduce is problem report. 46* <p> 47* To run the program: 48* <b>bin/hadoop jar hadoop-*-examples.jar teravalidate out-dir report-dir</b> 49* <p> 50* If there is any output, something is wrong and the output of the reduce 51* will have the problem report. 52*/ 53public class TeraValidate extends Configured implements Tool { 54private static final Text ERROR =new Text ("error" ); 55private static final Text CHECKSUM =new Text ("checksum" ); 56 57private static String textifyBytes (Text t ) { 58BytesWritable b =new BytesWritable (); 59b .set (t .getBytes (),0 ,t .getLength ()); 60return b .toString (); 61 } 62 63static class ValidateMapper extends Mapper <Text ,Text ,Text ,Text > { 64private Text lastKey ; 65private String filename ; 66private Unsigned16 checksum =new Unsigned16 (); 67private Unsigned16 tmp =new Unsigned16 (); 68private Checksum crc32 =new PureJavaCrc32 (); 69 70/** 71* Get the final part of the input name 72* @param split the input split 73* @return the "part-r-00000" for the input 74*/ 75private String getFilename (FileSplit split ) { 76return split .getPath ().getName (); 77 } 78 79public void map (Text key ,Text value ,Context context ) 80throws IOException ,InterruptedException { 81if (lastKey ==null ) { 82FileSplit fs = (FileSplit )context .getInputSplit (); 83filename =getFilename (fs ); 84context .write (new Text (filename +":begin" ),key ); 85lastKey =new Text (); 86 }else { 87if (key .compareTo (lastKey ) <0 ) { 88context .write (ERROR ,new Text ("misorder in " +filename + 89" between " +textifyBytes (lastKey ) + 90" and " +textifyBytes (key ))); 91 } 92 } 93// compute the crc of the key and value and add it to the sum 94crc32 .reset (); 95crc32 .update (key .getBytes (),0 ,key .getLength ()); 96crc32 .update (value .getBytes (),0 ,value .getLength ()); 97tmp .set (crc32 .getValue ()); 98checksum .add (tmp ); 99lastKey .set (key ); 100 } 101 102public void cleanup (Context context ) 103throws IOException ,InterruptedException { 104if (lastKey !=null ) { 105context .write (new Text (filename +":end" ),lastKey ); 106context .write (CHECKSUM ,new Text (checksum .toString ())); 107 } 108 } 109 } 110 111/** 112* Check the boundaries between the output files by making sure that the 113* boundary keys are always increasing. 114* Also passes any error reports along intact. 115*/ 116static class ValidateReducer extends Reducer <Text ,Text ,Text ,Text > { 117private boolean firstKey =true ; 118private Text lastKey =new Text (); 119private Text lastValue =new Text (); 120public void reduce (Text key ,Iterable <Text >values , 121Context context )throws IOException ,InterruptedException { 122if (ERROR .equals (key )) { 123for (Text val :values ) { 124context .write (key ,val ); 125 } 126 }else if (CHECKSUM .equals (key )) { 127Unsigned16 tmp =new Unsigned16 (); 128Unsigned16 sum =new Unsigned16 (); 129for (Text val :values ) { 130tmp .set (val .toString ()); 131sum .add (tmp ); 132 } 133context .write (CHECKSUM ,new Text (sum .toString ())); 134 }else { 135Text value =values .iterator ().next (); 136if (firstKey ) { 137firstKey =false ; 138 }else { 139if (value .compareTo (lastValue ) <0 ) { 140context .write (ERROR , 141new Text ("bad key partitioning:\n file " + 142lastKey +" key " + 143textifyBytes (lastValue ) + 144"\n file " +key +" key " + 145textifyBytes (value ))); 146 } 147 } 148lastKey .set (key ); 149lastValue .set (value ); 150 } 151 } 152 153 } 154 155private static void usage ()throws IOException { 156System .err .println ("teravalidate <out-dir> <report-dir>" ); 157 } 158 159public int run (String []args )throws Exception { 160Job job =Job .getInstance (getConf ()); 161if (args .length !=2 ) { 162usage (); 163return 1 ; 164 } 165TeraInputFormat .setInputPaths (job ,new Path (args [0 ])); 166FileOutputFormat .setOutputPath (job ,new Path (args [1 ])); 167job .setJobName ("TeraValidate" ); 168job .setJarByClass (TeraValidate .class ); 169job .setMapperClass (ValidateMapper .class ); 170job .setReducerClass (ValidateReducer .class ); 171job .setOutputKeyClass (Text .class ); 172job .setOutputValueClass (Text .class ); 173// force a single reducer 174job .setNumReduceTasks (1 ); 175// force a single split 176FileInputFormat .setMinInputSplitSize (job ,Long .MAX_VALUE ); 177job .setInputFormatClass (TeraInputFormat .class ); 178return job .waitForCompletion (true ) ?0 :1 ; 179 } 180 181/** 182* @param args 183*/ 184public static void main (String []args )throws Exception { 185int res =ToolRunner .run (new Configuration (),new TeraValidate (),args ); 186System .exit (res ); 187 } 188 189}