+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18 19package org .apache .hadoop .examples .terasort ; 20 21import java .io .DataInput ; 22import java .io .DataOutput ; 23import java .io .IOException ; 24import java .util .ArrayList ; 25import java .util .List ; 26import java .util .zip .Checksum ; 27 28import org .apache .commons .logging .Log ; 29import org .apache .commons .logging .LogFactory ; 30import org .apache .hadoop .conf .Configuration ; 31import org .apache .hadoop .conf .Configured ; 32import org .apache .hadoop .fs .Path ; 33import org .apache .hadoop .io .LongWritable ; 34import org .apache .hadoop .io .NullWritable ; 35import org .apache .hadoop .io .Text ; 36import org .apache .hadoop .io .Writable ; 37import org .apache .hadoop .io .WritableUtils ; 38import org .apache .hadoop .mapreduce .Cluster ; 39import org .apache .hadoop .mapreduce .Counter ; 40import org .apache .hadoop .mapreduce .InputFormat ; 41import org .apache .hadoop .mapreduce .InputSplit ; 42import org .apache .hadoop .mapreduce .Job ; 43import org .apache .hadoop .mapreduce .JobContext ; 44import org .apache .hadoop .mapreduce .MRJobConfig ; 45import org .apache .hadoop .mapreduce .Mapper ; 46import org .apache .hadoop .mapreduce .RecordReader ; 47import org .apache .hadoop .mapreduce .TaskAttemptContext ; 48import org .apache .hadoop .mapreduce .lib .output .FileOutputFormat ; 49import org .apache .hadoop .util .PureJavaCrc32 ; 50import org .apache .hadoop .util .Tool ; 51import org .apache .hadoop .util .ToolRunner ; 52 53/** 54* Generate the official GraySort input data set. 55* The user specifies the number of rows and the output directory and this 56* class runs a map/reduce program to generate the data. 57* The format of the data is: 58* <ul> 59* <li>(10 bytes key) (constant 2 bytes) (32 bytes rowid) 60* (constant 4 bytes) (48 bytes filler) (constant 4 bytes) 61* <li>The rowid is the right justified row id as a hex number. 62* </ul> 63* 64* <p> 65* To run the program: 66* <b>bin/hadoop jar hadoop-*-examples.jar teragen 10000000000 in-dir</b> 67*/ 68public class TeraGen extends Configured implements Tool { 69private static final Log LOG =LogFactory .getLog (TeraSort .class ); 70 71public static enum Counters {CHECKSUM } 72 73public static final String NUM_ROWS ="mapreduce.terasort.num-rows" ; 74/** 75* An input format that assigns ranges of longs to each mapper. 76*/ 77static class RangeInputFormat 78extends InputFormat <LongWritable ,NullWritable > { 79 80/** 81* An input split consisting of a range on numbers. 82*/ 83static class RangeInputSplit extends InputSplit implements Writable { 84long firstRow ; 85long rowCount ; 86 87public RangeInputSplit () { } 88 89public RangeInputSplit (long offset ,long length ) { 90firstRow =offset ; 91rowCount =length ; 92 } 93 94public long getLength ()throws IOException { 95return 0 ; 96 } 97 98public String []getLocations ()throws IOException { 99return new String []{}; 100 } 101 102public void readFields (DataInput in )throws IOException { 103firstRow =WritableUtils .readVLong (in ); 104rowCount =WritableUtils .readVLong (in ); 105 } 106 107public void write (DataOutput out )throws IOException { 108WritableUtils .writeVLong (out ,firstRow ); 109WritableUtils .writeVLong (out ,rowCount ); 110 } 111 } 112 113/** 114* A record reader that will generate a range of numbers. 115*/ 116static class RangeRecordReader 117extends RecordReader <LongWritable ,NullWritable > { 118long startRow ; 119long finishedRows ; 120long totalRows ; 121LongWritable key =null ; 122 123public RangeRecordReader () { 124 } 125 126public void initialize (InputSplit split ,TaskAttemptContext context ) 127throws IOException ,InterruptedException { 128startRow = ((RangeInputSplit )split ).firstRow ; 129finishedRows =0 ; 130totalRows = ((RangeInputSplit )split ).rowCount ; 131 } 132 133public void close ()throws IOException { 134// NOTHING 135 } 136 137public LongWritable getCurrentKey () { 138return key ; 139 } 140 141public NullWritable getCurrentValue () { 142return NullWritable .get (); 143 } 144 145public float getProgress ()throws IOException { 146return finishedRows / (float )totalRows ; 147 } 148 149public boolean nextKeyValue () { 150if (key ==null ) { 151key =new LongWritable (); 152 } 153if (finishedRows <totalRows ) { 154key .set (startRow +finishedRows ); 155finishedRows +=1 ; 156return true ; 157 }else { 158return false ; 159 } 160 } 161 162 } 163 164public RecordReader <LongWritable ,NullWritable > 165createRecordReader (InputSplit split ,TaskAttemptContext context ) 166throws IOException { 167return new RangeRecordReader (); 168 } 169 170/** 171* Create the desired number of splits, dividing the number of rows 172* between the mappers. 173*/ 174public List <InputSplit >getSplits (JobContext job ) { 175long totalRows =getNumberOfRows (job ); 176int numSplits =job .getConfiguration ().getInt (MRJobConfig .NUM_MAPS ,1 ); 177LOG .info ("Generating " +totalRows +" using " +numSplits ); 178List <InputSplit >splits =new ArrayList <InputSplit >(); 179long currentRow =0 ; 180for (int split =0 ;split <numSplits ; ++split ) { 181long goal = 182 (long )Math .ceil (totalRows * (double )(split +1 ) /numSplits ); 183splits .add (new RangeInputSplit (currentRow ,goal -currentRow )); 184currentRow =goal ; 185 } 186return splits ; 187 } 188 189 } 190 191static long getNumberOfRows (JobContext job ) { 192return job .getConfiguration ().getLong (NUM_ROWS ,0 ); 193 } 194 195static void setNumberOfRows (Job job ,long numRows ) { 196job .getConfiguration ().setLong (NUM_ROWS ,numRows ); 197 } 198 199/** 200* The Mapper class that given a row number, will generate the appropriate 201* output line. 202*/ 203public static class SortGenMapper 204extends Mapper <LongWritable ,NullWritable ,Text ,Text > { 205 206private Text key =new Text (); 207private Text value =new Text (); 208private Unsigned16 rand =null ; 209private Unsigned16 rowId =null ; 210private Unsigned16 checksum =new Unsigned16 (); 211private Checksum crc32 =new PureJavaCrc32 (); 212private Unsigned16 total =new Unsigned16 (); 213private static final Unsigned16 ONE =new Unsigned16 (1 ); 214private byte []buffer =new byte [TeraInputFormat .KEY_LENGTH + 215TeraInputFormat .VALUE_LENGTH ]; 216private Counter checksumCounter ; 217 218public void map (LongWritable row ,NullWritable ignored , 219Context context )throws IOException ,InterruptedException { 220if (rand ==null ) { 221rowId =new Unsigned16 (row .get ()); 222rand =Random16 .skipAhead (rowId ); 223checksumCounter =context .getCounter (Counters .CHECKSUM ); 224 } 225Random16 .nextRand (rand ); 226GenSort .generateRecord (buffer ,rand ,rowId ); 227key .set (buffer ,0 ,TeraInputFormat .KEY_LENGTH ); 228value .set (buffer ,TeraInputFormat .KEY_LENGTH , 229TeraInputFormat .VALUE_LENGTH ); 230context .write (key ,value ); 231crc32 .reset (); 232crc32 .update (buffer ,0 , 233TeraInputFormat .KEY_LENGTH +TeraInputFormat .VALUE_LENGTH ); 234checksum .set (crc32 .getValue ()); 235total .add (checksum ); 236rowId .add (ONE ); 237 } 238 239@ Override 240public void cleanup (Context context ) { 241if (checksumCounter !=null ) { 242checksumCounter .increment (total .getLow8 ()); 243 } 244 } 245 } 246 247private static void usage ()throws IOException { 248System .err .println ("teragen <num rows> <output dir>" ); 249 } 250 251/** 252* Parse a number that optionally has a postfix that denotes a base. 253* @param str an string integer with an option base {k,m,b,t}. 254* @return the expanded value 255*/ 256private static long parseHumanLong (String str ) { 257char tail =str .charAt (str .length () -1 ); 258long base =1 ; 259switch (tail ) { 260case 't' : 261base *=1000 *1000 *1000 *1000 ; 262break ; 263case 'b' : 264base *=1000 *1000 *1000 ; 265break ; 266case 'm' : 267base *=1000 *1000 ; 268break ; 269case 'k' : 270base *=1000 ; 271break ; 272default : 273 } 274if (base !=1 ) { 275str =str .substring (0 ,str .length () -1 ); 276 } 277return Long .parseLong (str ) *base ; 278 } 279 280/** 281* @param args the cli arguments 282*/ 283public int run (String []args ) 284throws IOException ,InterruptedException ,ClassNotFoundException { 285Job job =Job .getInstance (getConf ()); 286if (args .length !=2 ) { 287usage (); 288return 2 ; 289 } 290setNumberOfRows (job ,parseHumanLong (args [0 ])); 291Path outputDir =new Path (args [1 ]); 292FileOutputFormat .setOutputPath (job ,outputDir ); 293job .setJobName ("TeraGen" ); 294job .setJarByClass (TeraGen .class ); 295job .setMapperClass (SortGenMapper .class ); 296job .setNumReduceTasks (0 ); 297job .setOutputKeyClass (Text .class ); 298job .setOutputValueClass (Text .class ); 299job .setInputFormatClass (RangeInputFormat .class ); 300job .setOutputFormatClass (TeraOutputFormat .class ); 301return job .waitForCompletion (true ) ?0 :1 ; 302 } 303 304public static void main (String []args )throws Exception { 305int res =ToolRunner .run (new Configuration (),new TeraGen (),args ); 306System .exit (res ); 307 } 308}