+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18 19package org .apache .hadoop .examples ; 20 21import java .io .DataInput ; 22import java .io .DataOutput ; 23import java .io .IOException ; 24import java .util .StringTokenizer ; 25 26import org .apache .hadoop .conf .Configuration ; 27import org .apache .hadoop .fs .Path ; 28import org .apache .hadoop .io .IntWritable ; 29import org .apache .hadoop .io .LongWritable ; 30import org .apache .hadoop .io .RawComparator ; 31import org .apache .hadoop .io .Text ; 32import org .apache .hadoop .io .WritableComparable ; 33import org .apache .hadoop .io .WritableComparator ; 34import org .apache .hadoop .mapreduce .lib .input .FileInputFormat ; 35import org .apache .hadoop .mapreduce .lib .output .FileOutputFormat ; 36import org .apache .hadoop .mapreduce .Job ; 37import org .apache .hadoop .mapreduce .Mapper ; 38import org .apache .hadoop .mapreduce .Partitioner ; 39import org .apache .hadoop .mapreduce .Reducer ; 40import org .apache .hadoop .util .GenericOptionsParser ; 41 42/** 43* This is an example Hadoop Map/Reduce application. 44* It reads the text input files that must contain two integers per a line. 45* The output is sorted by the first and second number and grouped on the 46* first number. 47* 48* To run: bin/hadoop jar build/hadoop-examples.jar secondarysort 49* <i>in-dir</i> <i>out-dir</i> 50*/ 51public class SecondarySort { 52 53/** 54* Define a pair of integers that are writable. 55* They are serialized in a byte comparable format. 56*/ 57public static class IntPair 58implements WritableComparable <IntPair > { 59private int first =0 ; 60private int second =0 ; 61 62/** 63* Set the left and right values. 64*/ 65public void set (int left ,int right ) { 66first =left ; 67second =right ; 68 } 69public int getFirst () { 70return first ; 71 } 72public int getSecond () { 73return second ; 74 } 75/** 76* Read the two integers. 77* Encoded as: MIN_VALUE -> 0, 0 -> -MIN_VALUE, MAX_VALUE-> -1 78*/ 79@ Override 80public void readFields (DataInput in )throws IOException { 81first =in .readInt () +Integer .MIN_VALUE ; 82second =in .readInt () +Integer .MIN_VALUE ; 83 } 84@ Override 85public void write (DataOutput out )throws IOException { 86out .writeInt (first -Integer .MIN_VALUE ); 87out .writeInt (second -Integer .MIN_VALUE ); 88 } 89@ Override 90public int hashCode () { 91return first *157 +second ; 92 } 93@ Override 94public boolean equals (Object right ) { 95if (right instanceof IntPair ) { 96IntPair r = (IntPair )right ; 97return r .first ==first &&r .second ==second ; 98 }else { 99return false ; 100 } 101 } 102/** A Comparator that compares serialized IntPair. */ 103public static class Comparator extends WritableComparator { 104public Comparator () { 105super (IntPair .class ); 106 } 107 108public int compare (byte []b1 ,int s1 ,int l1 , 109byte []b2 ,int s2 ,int l2 ) { 110return compareBytes (b1 ,s1 ,l1 ,b2 ,s2 ,l2 ); 111 } 112 } 113 114static {// register this comparator 115WritableComparator .define (IntPair .class ,new Comparator ()); 116 } 117 118@ Override 119public int compareTo (IntPair o ) { 120if (first !=o .first ) { 121return first <o .first ? -1 :1 ; 122 }else if (second !=o .second ) { 123return second <o .second ? -1 :1 ; 124 }else { 125return 0 ; 126 } 127 } 128 } 129 130/** 131* Partition based on the first part of the pair. 132*/ 133public static class FirstPartitioner extends Partitioner <IntPair ,IntWritable >{ 134@ Override 135public int getPartition (IntPair key ,IntWritable value , 136int numPartitions ) { 137return Math .abs (key .getFirst () *127 ) %numPartitions ; 138 } 139 } 140 141/** 142* Compare only the first part of the pair, so that reduce is called once 143* for each value of the first part. 144*/ 145public static class FirstGroupingComparator 146implements RawComparator <IntPair > { 147@ Override 148public int compare (byte []b1 ,int s1 ,int l1 ,byte []b2 ,int s2 ,int l2 ) { 149return WritableComparator .compareBytes (b1 ,s1 ,Integer .SIZE /8 , 150b2 ,s2 ,Integer .SIZE /8 ); 151 } 152 153@ Override 154public int compare (IntPair o1 ,IntPair o2 ) { 155int l =o1 .getFirst (); 156int r =o2 .getFirst (); 157return l ==r ?0 : (l <r ? -1 :1 ); 158 } 159 } 160 161/** 162* Read two integers from each line and generate a key, value pair 163* as ((left, right), right). 164*/ 165public static class MapClass 166extends Mapper <LongWritable ,Text ,IntPair ,IntWritable > { 167 168private final IntPair key =new IntPair (); 169private final IntWritable value =new IntWritable (); 170 171@ Override 172public void map (LongWritable inKey ,Text inValue , 173Context context )throws IOException ,InterruptedException { 174StringTokenizer itr =new StringTokenizer (inValue .toString ()); 175int left =0 ; 176int right =0 ; 177if (itr .hasMoreTokens ()) { 178left =Integer .parseInt (itr .nextToken ()); 179if (itr .hasMoreTokens ()) { 180right =Integer .parseInt (itr .nextToken ()); 181 } 182key .set (left ,right ); 183value .set (right ); 184context .write (key ,value ); 185 } 186 } 187 } 188 189/** 190* A reducer class that just emits the sum of the input values. 191*/ 192public static class Reduce 193extends Reducer <IntPair ,IntWritable ,Text ,IntWritable > { 194private static final Text SEPARATOR = 195new Text ("------------------------------------------------" ); 196private final Text first =new Text (); 197 198@ Override 199public void reduce (IntPair key ,Iterable <IntWritable >values , 200Context context 201 )throws IOException ,InterruptedException { 202context .write (SEPARATOR ,null ); 203first .set (Integer .toString (key .getFirst ())); 204for (IntWritable value :values ) { 205context .write (first ,value ); 206 } 207 } 208 } 209 210public static void main (String []args )throws Exception { 211Configuration conf =new Configuration (); 212String []otherArgs =new GenericOptionsParser (conf ,args ).getRemainingArgs (); 213if (otherArgs .length !=2 ) { 214System .err .println ("Usage: secondarysort <in> <out>" ); 215System .exit (2 ); 216 } 217Job job =Job .getInstance (conf ,"secondary sort" ); 218job .setJarByClass (SecondarySort .class ); 219job .setMapperClass (MapClass .class ); 220job .setReducerClass (Reduce .class ); 221 222// group and partition by the first int in the pair 223job .setPartitionerClass (FirstPartitioner .class ); 224job .setGroupingComparatorClass (FirstGroupingComparator .class ); 225 226// the map output is IntPair, IntWritable 227job .setMapOutputKeyClass (IntPair .class ); 228job .setMapOutputValueClass (IntWritable .class ); 229 230// the reduce output is Text, IntWritable 231job .setOutputKeyClass (Text .class ); 232job .setOutputValueClass (IntWritable .class ); 233 234FileInputFormat .addInputPath (job ,new Path (otherArgs [0 ])); 235FileOutputFormat .setOutputPath (job ,new Path (otherArgs [1 ])); 236System .exit (job .waitForCompletion (true ) ?0 :1 ); 237 } 238 239}