+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18 19package org .apache .hadoop .examples ; 20 21import java .io .IOException ; 22import java .util .ArrayList ; 23import java .util .Date ; 24import java .util .List ; 25 26import org .apache .hadoop .conf .Configuration ; 27import org .apache .hadoop .conf .Configured ; 28import org .apache .hadoop .fs .Path ; 29import org .apache .hadoop .io .BytesWritable ; 30import org .apache .hadoop .io .Writable ; 31import org .apache .hadoop .io .WritableComparable ; 32import org .apache .hadoop .mapred .ClusterStatus ; 33import org .apache .hadoop .mapred .JobClient ; 34import org .apache .hadoop .mapreduce .InputFormat ; 35import org .apache .hadoop .mapreduce .Job ; 36import org .apache .hadoop .mapreduce .Mapper ; 37import org .apache .hadoop .mapreduce .OutputFormat ; 38import org .apache .hadoop .mapreduce .Reducer ; 39import org .apache .hadoop .mapreduce .lib .input .SequenceFileInputFormat ; 40import org .apache .hadoop .mapreduce .lib .join .CompositeInputFormat ; 41import org .apache .hadoop .mapreduce .lib .join .TupleWritable ; 42import org .apache .hadoop .mapreduce .lib .output .FileOutputFormat ; 43import org .apache .hadoop .mapreduce .lib .output .SequenceFileOutputFormat ; 44import org .apache .hadoop .util .Tool ; 45import org .apache .hadoop .util .ToolRunner ; 46 47/** 48* Given a set of sorted datasets keyed with the same class and yielding 49* equal partitions, it is possible to effect a join of those datasets 50* prior to the map. The example facilitates the same. 51* 52* To run: bin/hadoop jar build/hadoop-examples.jar join 53* [-r <i>reduces</i>] 54* [-inFormat <i>input format class</i>] 55* [-outFormat <i>output format class</i>] 56* [-outKey <i>output key class</i>] 57* [-outValue <i>output value class</i>] 58* [-joinOp <inner|outer|override>] 59* [<i>in-dir</i>]* <i>in-dir</i> <i>out-dir</i> 60*/ 61public class Join extends Configured implements Tool { 62public static final String REDUCES_PER_HOST ="mapreduce.join.reduces_per_host" ; 63static int printUsage () { 64System .out .println ("join [-r <reduces>] " + 65"[-inFormat <input format class>] " + 66"[-outFormat <output format class>] " + 67"[-outKey <output key class>] " + 68"[-outValue <output value class>] " + 69"[-joinOp <inner|outer|override>] " + 70"[input]* <input> <output>" ); 71ToolRunner .printGenericCommandUsage (System .out ); 72return 2 ; 73 } 74 75/** 76* The main driver for sort program. 77* Invoke this method to submit the map/reduce job. 78* @throws IOException When there is communication problems with the 79* job tracker. 80*/ 81@ SuppressWarnings ("unchecked" ) 82public int run (String []args )throws Exception { 83Configuration conf =getConf (); 84JobClient client =new JobClient (conf ); 85ClusterStatus cluster =client .getClusterStatus (); 86int num_reduces = (int ) (cluster .getMaxReduceTasks () *0.9 ); 87String join_reduces =conf .get (REDUCES_PER_HOST ); 88if (join_reduces !=null ) { 89num_reduces =cluster .getTaskTrackers () * 90Integer .parseInt (join_reduces ); 91 } 92Job job =Job .getInstance (conf ); 93job .setJobName ("join" ); 94job .setJarByClass (Sort .class ); 95 96job .setMapperClass (Mapper .class ); 97job .setReducerClass (Reducer .class ); 98 99Class <?extends InputFormat >inputFormatClass = 100SequenceFileInputFormat .class ; 101Class <?extends OutputFormat >outputFormatClass = 102SequenceFileOutputFormat .class ; 103Class <?extends WritableComparable >outputKeyClass =BytesWritable .class ; 104Class <?extends Writable >outputValueClass =TupleWritable .class ; 105String op ="inner" ; 106List <String >otherArgs =new ArrayList <String >(); 107for (int i =0 ;i <args .length ; ++i ) { 108try { 109if ("-r" .equals (args [i ])) { 110num_reduces =Integer .parseInt (args [++i ]); 111 }else if ("-inFormat" .equals (args [i ])) { 112inputFormatClass = 113Class .forName (args [++i ]).asSubclass (InputFormat .class ); 114 }else if ("-outFormat" .equals (args [i ])) { 115outputFormatClass = 116Class .forName (args [++i ]).asSubclass (OutputFormat .class ); 117 }else if ("-outKey" .equals (args [i ])) { 118outputKeyClass = 119Class .forName (args [++i ]).asSubclass (WritableComparable .class ); 120 }else if ("-outValue" .equals (args [i ])) { 121outputValueClass = 122Class .forName (args [++i ]).asSubclass (Writable .class ); 123 }else if ("-joinOp" .equals (args [i ])) { 124op =args [++i ]; 125 }else { 126otherArgs .add (args [i ]); 127 } 128 }catch (NumberFormatException except ) { 129System .out .println ("ERROR: Integer expected instead of " +args [i ]); 130return printUsage (); 131 }catch (ArrayIndexOutOfBoundsException except ) { 132System .out .println ("ERROR: Required parameter missing from " + 133args [i -1 ]); 134return printUsage ();// exits 135 } 136 } 137 138// Set user-supplied (possibly default) job configs 139job .setNumReduceTasks (num_reduces ); 140 141if (otherArgs .size () <2 ) { 142System .out .println ("ERROR: Wrong number of parameters: " ); 143return printUsage (); 144 } 145 146FileOutputFormat .setOutputPath (job , 147new Path (otherArgs .remove (otherArgs .size () -1 ))); 148List <Path >plist =new ArrayList <Path >(otherArgs .size ()); 149for (String s :otherArgs ) { 150plist .add (new Path (s )); 151 } 152 153job .setInputFormatClass (CompositeInputFormat .class ); 154job .getConfiguration ().set (CompositeInputFormat .JOIN_EXPR , 155CompositeInputFormat .compose (op ,inputFormatClass , 156plist .toArray (new Path [0 ]))); 157job .setOutputFormatClass (outputFormatClass ); 158 159job .setOutputKeyClass (outputKeyClass ); 160job .setOutputValueClass (outputValueClass ); 161 162Date startTime =new Date (); 163System .out .println ("Job started: " +startTime ); 164int ret =job .waitForCompletion (true ) ?0 :1 ; 165Date end_time =new Date (); 166System .out .println ("Job ended: " +end_time ); 167System .out .println ("The job took " + 168 (end_time .getTime () -startTime .getTime ()) /1000 +" seconds." ); 169return ret ; 170 } 171 172public static void main (String []args )throws Exception { 173int res =ToolRunner .run (new Configuration (),new Join (),args ); 174System .exit (res ); 175 } 176 177}