+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18 19package org .apache .hadoop .examples ; 20 21import java .io .IOException ; 22import java .util .ArrayList ; 23import java .util .Date ; 24import java .util .List ; 25import java .util .Random ; 26 27import org .apache .hadoop .conf .Configuration ; 28import org .apache .hadoop .conf .Configured ; 29import org .apache .hadoop .fs .Path ; 30import org .apache .hadoop .io .BytesWritable ; 31import org .apache .hadoop .io .Text ; 32import org .apache .hadoop .io .Writable ; 33import org .apache .hadoop .io .WritableComparable ; 34import org .apache .hadoop .mapred .ClusterStatus ; 35import org .apache .hadoop .mapred .JobClient ; 36import org .apache .hadoop .mapreduce .*; 37import org .apache .hadoop .mapreduce .lib .input .FileSplit ; 38import org .apache .hadoop .mapreduce .lib .output .FileOutputFormat ; 39import org .apache .hadoop .mapreduce .lib .output .SequenceFileOutputFormat ; 40import org .apache .hadoop .util .GenericOptionsParser ; 41import org .apache .hadoop .util .Tool ; 42import org .apache .hadoop .util .ToolRunner ; 43 44/** 45* This program uses map/reduce to just run a distributed job where there is 46* no interaction between the tasks and each task write a large unsorted 47* random binary sequence file of BytesWritable. 48* In order for this program to generate data for terasort with 10-byte keys 49* and 90-byte values, have the following config: 50* <pre>{@code 51* <?xml version="1.0"?> 52* <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> 53* <configuration> 54* <property> 55* <name>mapreduce.randomwriter.minkey</name> 56* <value>10</value> 57* </property> 58* <property> 59* <name>mapreduce.randomwriter.maxkey</name> 60* <value>10</value> 61* </property> 62* <property> 63* <name>mapreduce.randomwriter.minvalue</name> 64* <value>90</value> 65* </property> 66* <property> 67* <name>mapreduce.randomwriter.maxvalue</name> 68* <value>90</value> 69* </property> 70* <property> 71* <name>mapreduce.randomwriter.totalbytes</name> 72* <value>1099511627776</value> 73* </property> 74* </configuration>}</pre> 75* Equivalently, {@link RandomWriter} also supports all the above options 76* and ones supported by {@link GenericOptionsParser} via the command-line. 77*/ 78public class RandomWriter extends Configured implements Tool { 79public static final String TOTAL_BYTES ="mapreduce.randomwriter.totalbytes" ; 80public static final String BYTES_PER_MAP = 81"mapreduce.randomwriter.bytespermap" ; 82public static final String MAPS_PER_HOST = 83"mapreduce.randomwriter.mapsperhost" ; 84public static final String MAX_VALUE ="mapreduce.randomwriter.maxvalue" ; 85public static final String MIN_VALUE ="mapreduce.randomwriter.minvalue" ; 86public static final String MIN_KEY ="mapreduce.randomwriter.minkey" ; 87public static final String MAX_KEY ="mapreduce.randomwriter.maxkey" ; 88 89/** 90* User counters 91*/ 92static enum Counters {RECORDS_WRITTEN ,BYTES_WRITTEN } 93 94/** 95* A custom input format that creates virtual inputs of a single string 96* for each map. 97*/ 98static class RandomInputFormat extends InputFormat <Text ,Text > { 99 100/** 101* Generate the requested number of file splits, with the filename 102* set to the filename of the output file. 103*/ 104public List <InputSplit >getSplits (JobContext job )throws IOException { 105List <InputSplit >result =new ArrayList <InputSplit >(); 106Path outDir =FileOutputFormat .getOutputPath (job ); 107int numSplits = 108job .getConfiguration ().getInt (MRJobConfig .NUM_MAPS ,1 ); 109for (int i =0 ;i <numSplits ; ++i ) { 110result .add (new FileSplit (new Path (outDir ,"dummy-split-" +i ),0 ,1 , 111 (String [])null )); 112 } 113return result ; 114 } 115 116/** 117* Return a single record (filename, "") where the filename is taken from 118* the file split. 119*/ 120static class RandomRecordReader extends RecordReader <Text ,Text > { 121Path name ; 122Text key =null ; 123Text value =new Text (); 124public RandomRecordReader (Path p ) { 125name =p ; 126 } 127 128public void initialize (InputSplit split , 129TaskAttemptContext context ) 130throws IOException ,InterruptedException { 131 132 } 133 134public boolean nextKeyValue () { 135if (name !=null ) { 136key =new Text (); 137key .set (name .getName ()); 138name =null ; 139return true ; 140 } 141return false ; 142 } 143 144public Text getCurrentKey () { 145return key ; 146 } 147 148public Text getCurrentValue () { 149return value ; 150 } 151 152public void close () {} 153 154public float getProgress () { 155return 0.0f ; 156 } 157 } 158 159public RecordReader <Text ,Text >createRecordReader (InputSplit split , 160TaskAttemptContext context )throws IOException ,InterruptedException { 161return new RandomRecordReader (((FileSplit )split ).getPath ()); 162 } 163 } 164 165static class RandomMapper extends Mapper <WritableComparable ,Writable , 166BytesWritable ,BytesWritable > { 167 168private long numBytesToWrite ; 169private int minKeySize ; 170private int keySizeRange ; 171private int minValueSize ; 172private int valueSizeRange ; 173private Random random =new Random (); 174private BytesWritable randomKey =new BytesWritable (); 175private BytesWritable randomValue =new BytesWritable (); 176 177private void randomizeBytes (byte []data ,int offset ,int length ) { 178for (int i =offset +length -1 ;i >=offset ; --i ) { 179data [i ] = (byte )random .nextInt (256 ); 180 } 181 } 182 183/** 184* Given an output filename, write a bunch of random records to it. 185*/ 186public void map (WritableComparable key , 187Writable value , 188Context context )throws IOException ,InterruptedException { 189int itemCount =0 ; 190while (numBytesToWrite >0 ) { 191int keyLength =minKeySize + 192 (keySizeRange !=0 ?random .nextInt (keySizeRange ) :0 ); 193randomKey .setSize (keyLength ); 194randomizeBytes (randomKey .getBytes (),0 ,randomKey .getLength ()); 195int valueLength =minValueSize + 196 (valueSizeRange !=0 ?random .nextInt (valueSizeRange ) :0 ); 197randomValue .setSize (valueLength ); 198randomizeBytes (randomValue .getBytes (),0 ,randomValue .getLength ()); 199context .write (randomKey ,randomValue ); 200numBytesToWrite -=keyLength +valueLength ; 201context .getCounter (Counters .BYTES_WRITTEN ).increment (keyLength +valueLength ); 202context .getCounter (Counters .RECORDS_WRITTEN ).increment (1 ); 203if (++itemCount %200 ==0 ) { 204context .setStatus ("wrote record " +itemCount +". " + 205numBytesToWrite +" bytes left." ); 206 } 207 } 208context .setStatus ("done with " +itemCount +" records." ); 209 } 210 211/** 212* Save the values out of the configuaration that we need to write 213* the data. 214*/ 215@ Override 216public void setup (Context context ) { 217Configuration conf =context .getConfiguration (); 218numBytesToWrite =conf .getLong (BYTES_PER_MAP , 2191 *1024 *1024 *1024 ); 220minKeySize =conf .getInt (MIN_KEY ,10 ); 221keySizeRange = 222conf .getInt (MAX_KEY ,1000 ) -minKeySize ; 223minValueSize =conf .getInt (MIN_VALUE ,0 ); 224valueSizeRange = 225conf .getInt (MAX_VALUE ,20000 ) -minValueSize ; 226 } 227 } 228 229/** 230* This is the main routine for launching a distributed random write job. 231* It runs 10 maps/node and each node writes 1 gig of data to a DFS file. 232* The reduce doesn't do anything. 233* 234* @throws IOException 235*/ 236public int run (String []args )throws Exception { 237if (args .length ==0 ) { 238System .out .println ("Usage: writer <out-dir>" ); 239ToolRunner .printGenericCommandUsage (System .out ); 240return 2 ; 241 } 242 243Path outDir =new Path (args [0 ]); 244Configuration conf =getConf (); 245JobClient client =new JobClient (conf ); 246ClusterStatus cluster =client .getClusterStatus (); 247int numMapsPerHost =conf .getInt (MAPS_PER_HOST ,10 ); 248long numBytesToWritePerMap =conf .getLong (BYTES_PER_MAP , 2491 *1024 *1024 *1024 ); 250if (numBytesToWritePerMap ==0 ) { 251System .err .println ("Cannot have" +BYTES_PER_MAP +" set to 0" ); 252return -2 ; 253 } 254long totalBytesToWrite =conf .getLong (TOTAL_BYTES , 255numMapsPerHost *numBytesToWritePerMap *cluster .getTaskTrackers ()); 256int numMaps = (int ) (totalBytesToWrite /numBytesToWritePerMap ); 257if (numMaps ==0 &&totalBytesToWrite >0 ) { 258numMaps =1 ; 259conf .setLong (BYTES_PER_MAP ,totalBytesToWrite ); 260 } 261conf .setInt (MRJobConfig .NUM_MAPS ,numMaps ); 262 263Job job =Job .getInstance (conf ); 264 265job .setJarByClass (RandomWriter .class ); 266job .setJobName ("random-writer" ); 267FileOutputFormat .setOutputPath (job ,outDir ); 268job .setOutputKeyClass (BytesWritable .class ); 269job .setOutputValueClass (BytesWritable .class ); 270job .setInputFormatClass (RandomInputFormat .class ); 271job .setMapperClass (RandomMapper .class ); 272job .setReducerClass (Reducer .class ); 273job .setOutputFormatClass (SequenceFileOutputFormat .class ); 274 275System .out .println ("Running " +numMaps +" maps." ); 276 277// reducer NONE 278job .setNumReduceTasks (0 ); 279 280Date startTime =new Date (); 281System .out .println ("Job started: " +startTime ); 282int ret =job .waitForCompletion (true ) ?0 :1 ; 283Date endTime =new Date (); 284System .out .println ("Job ended: " +endTime ); 285System .out .println ("The job took " + 286 (endTime .getTime () -startTime .getTime ()) /1000 + 287" seconds." ); 288 289return ret ; 290 } 291 292public static void main (String []args )throws Exception { 293int res =ToolRunner .run (new Configuration (),new RandomWriter (),args ); 294System .exit (res ); 295 } 296 297}