+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18 19package org .apache .hadoop .examples ; 20 21import java .io .IOException ; 22import java .net .URI ; 23import java .util .*; 24 25import org .apache .hadoop .conf .Configuration ; 26import org .apache .hadoop .conf .Configured ; 27import org .apache .hadoop .mapreduce .filecache .DistributedCache ; 28import org .apache .hadoop .fs .Path ; 29import org .apache .hadoop .io .BytesWritable ; 30import org .apache .hadoop .io .Writable ; 31import org .apache .hadoop .io .WritableComparable ; 32import org .apache .hadoop .mapred .ClusterStatus ; 33import org .apache .hadoop .mapred .JobClient ; 34import org .apache .hadoop .mapreduce .*; 35import org .apache .hadoop .mapreduce .lib .input .FileInputFormat ; 36import org .apache .hadoop .mapreduce .lib .input .SequenceFileInputFormat ; 37import org .apache .hadoop .mapreduce .lib .output .FileOutputFormat ; 38import org .apache .hadoop .mapreduce .lib .output .SequenceFileOutputFormat ; 39import org .apache .hadoop .mapreduce .lib .partition .InputSampler ; 40import org .apache .hadoop .mapreduce .lib .partition .TotalOrderPartitioner ; 41import org .apache .hadoop .util .Tool ; 42import org .apache .hadoop .util .ToolRunner ; 43 44/** 45* This is the trivial map/reduce program that does absolutely nothing 46* other than use the framework to fragment and sort the input values. 47* 48* To run: bin/hadoop jar build/hadoop-examples.jar sort 49* [-r <i>reduces</i>] 50* [-inFormat <i>input format class</i>] 51* [-outFormat <i>output format class</i>] 52* [-outKey <i>output key class</i>] 53* [-outValue <i>output value class</i>] 54* [-totalOrder <i>pcnt</i> <i>num samples</i> <i>max splits</i>] 55* <i>in-dir</i> <i>out-dir</i> 56*/ 57public class Sort <K ,V >extends Configured implements Tool { 58public static final String REDUCES_PER_HOST = 59"mapreduce.sort.reducesperhost" ; 60private Job job =null ; 61 62static int printUsage () { 63System .out .println ("sort [-r <reduces>] " + 64"[-inFormat <input format class>] " + 65"[-outFormat <output format class>] " + 66"[-outKey <output key class>] " + 67"[-outValue <output value class>] " + 68"[-totalOrder <pcnt> <num samples> <max splits>] " + 69"<input> <output>" ); 70ToolRunner .printGenericCommandUsage (System .out ); 71return 2 ; 72 } 73 74/** 75* The main driver for sort program. 76* Invoke this method to submit the map/reduce job. 77* @throws IOException When there is communication problems with the 78* job tracker. 79*/ 80public int run (String []args )throws Exception { 81 82Configuration conf =getConf (); 83JobClient client =new JobClient (conf ); 84ClusterStatus cluster =client .getClusterStatus (); 85int num_reduces = (int ) (cluster .getMaxReduceTasks () *0.9 ); 86String sort_reduces =conf .get (REDUCES_PER_HOST ); 87if (sort_reduces !=null ) { 88num_reduces =cluster .getTaskTrackers () * 89Integer .parseInt (sort_reduces ); 90 } 91Class <?extends InputFormat >inputFormatClass = 92SequenceFileInputFormat .class ; 93Class <?extends OutputFormat >outputFormatClass = 94SequenceFileOutputFormat .class ; 95Class <?extends WritableComparable >outputKeyClass =BytesWritable .class ; 96Class <?extends Writable >outputValueClass =BytesWritable .class ; 97List <String >otherArgs =new ArrayList <String >(); 98InputSampler .Sampler <K ,V >sampler =null ; 99for (int i =0 ;i <args .length ; ++i ) { 100try { 101if ("-r" .equals (args [i ])) { 102num_reduces =Integer .parseInt (args [++i ]); 103 }else if ("-inFormat" .equals (args [i ])) { 104inputFormatClass = 105Class .forName (args [++i ]).asSubclass (InputFormat .class ); 106 }else if ("-outFormat" .equals (args [i ])) { 107outputFormatClass = 108Class .forName (args [++i ]).asSubclass (OutputFormat .class ); 109 }else if ("-outKey" .equals (args [i ])) { 110outputKeyClass = 111Class .forName (args [++i ]).asSubclass (WritableComparable .class ); 112 }else if ("-outValue" .equals (args [i ])) { 113outputValueClass = 114Class .forName (args [++i ]).asSubclass (Writable .class ); 115 }else if ("-totalOrder" .equals (args [i ])) { 116double pcnt =Double .parseDouble (args [++i ]); 117int numSamples =Integer .parseInt (args [++i ]); 118int maxSplits =Integer .parseInt (args [++i ]); 119if (0 >=maxSplits )maxSplits =Integer .MAX_VALUE ; 120sampler = 121new InputSampler .RandomSampler <K ,V >(pcnt ,numSamples ,maxSplits ); 122 }else { 123otherArgs .add (args [i ]); 124 } 125 }catch (NumberFormatException except ) { 126System .out .println ("ERROR: Integer expected instead of " +args [i ]); 127return printUsage (); 128 }catch (ArrayIndexOutOfBoundsException except ) { 129System .out .println ("ERROR: Required parameter missing from " + 130args [i -1 ]); 131return printUsage ();// exits 132 } 133 } 134// Set user-supplied (possibly default) job configs 135job =Job .getInstance (conf ); 136job .setJobName ("sorter" ); 137job .setJarByClass (Sort .class ); 138 139job .setMapperClass (Mapper .class ); 140job .setReducerClass (Reducer .class ); 141 142job .setNumReduceTasks (num_reduces ); 143 144job .setInputFormatClass (inputFormatClass ); 145job .setOutputFormatClass (outputFormatClass ); 146 147job .setOutputKeyClass (outputKeyClass ); 148job .setOutputValueClass (outputValueClass ); 149 150// Make sure there are exactly 2 parameters left. 151if (otherArgs .size () !=2 ) { 152System .out .println ("ERROR: Wrong number of parameters: " + 153otherArgs .size () +" instead of 2." ); 154return printUsage (); 155 } 156FileInputFormat .setInputPaths (job ,otherArgs .get (0 )); 157FileOutputFormat .setOutputPath (job ,new Path (otherArgs .get (1 ))); 158 159if (sampler !=null ) { 160System .out .println ("Sampling input to effect total-order sort..." ); 161job .setPartitionerClass (TotalOrderPartitioner .class ); 162Path inputDir =FileInputFormat .getInputPaths (job )[0 ]; 163inputDir =inputDir .makeQualified (inputDir .getFileSystem (conf )); 164Path partitionFile =new Path (inputDir ,"_sortPartitioning" ); 165TotalOrderPartitioner .setPartitionFile (conf ,partitionFile ); 166InputSampler .<K ,V >writePartitionFile (job ,sampler ); 167URI partitionUri =new URI (partitionFile .toString () + 168"#" +"_sortPartitioning" ); 169DistributedCache .addCacheFile (partitionUri ,conf ); 170 } 171 172System .out .println ("Running on " + 173cluster .getTaskTrackers () + 174" nodes to sort from " + 175FileInputFormat .getInputPaths (job )[0 ] +" into " + 176FileOutputFormat .getOutputPath (job ) + 177" with " +num_reduces +" reduces." ); 178Date startTime =new Date (); 179System .out .println ("Job started: " +startTime ); 180int ret =job .waitForCompletion (true ) ?0 :1 ; 181Date end_time =new Date (); 182System .out .println ("Job ended: " +end_time ); 183System .out .println ("The job took " + 184 (end_time .getTime () -startTime .getTime ()) /1000 +" seconds." ); 185return ret ; 186 } 187 188 189 190public static void main (String []args )throws Exception { 191int res =ToolRunner .run (new Configuration (),new Sort (),args ); 192System .exit (res ); 193 } 194 195/** 196* Get the last job that was run using this instance. 197* @return the results of the last job that was run 198*/ 199public Job getResult () { 200return job ; 201 } 202}