+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
bde7f97
master
1import java .io .IOException ; 2import java .util .*; 3import java .util .stream .*; 4import org .apache .hadoop .conf .*; 5import org .apache .hadoop .fs .Path ; 6import org .apache .hadoop .io .*; 7import org .apache .hadoop .mapreduce .*; 8import org .apache .hadoop .mapreduce .lib .input .FileInputFormat ; 9import org .apache .hadoop .mapreduce .lib .input .TextInputFormat ; 10import org .apache .hadoop .mapreduce .lib .output .FileOutputFormat ; 11import org .apache .hadoop .mapreduce .lib .output .TextOutputFormat ; 12import org .apache .hadoop .util .Tool ; 13import org .apache .hadoop .util .ToolRunner ; 14 15public class TextAnalyzer extends Configured implements Tool { 16public static class TextMapper 17extends Mapper <LongWritable ,Text ,Text ,LongWritable > { 18public void map (LongWritable lineNum ,Text rawLine ,Context context ) 19throws IOException ,InterruptedException { 20Set <String >line =Stream 21 .of (rawLine .toString () 22 .toLowerCase () 23 .replaceAll ("[^a-z0-9]" ," " ) 24 .trim () 25 .split ("\\s+" )) 26 .collect (Collectors .toSet ()); 27for (String a :line ) { 28for (String b :line ) { 29if (!a .equals (b )) { 30context .write (new Text (a +" " +b ), 31new LongWritable (1 )); 32 } 33 } 34 } 35 } 36 } 37 38// public static class TextCombiner extends Reducer<?, ?, ?, ?> { 39// public void reduce(Text key, Iterable<Tuple> tuples, Context context) 40// throws IOException, InterruptedException { 41// // Implementation of you combiner function 42// } 43// } 44 45public static class TextReducer 46extends Reducer <Text ,LongWritable ,Text ,Text > { 47public void reduce (Text key ,Iterable <LongWritable >counts , 48Context context ) 49throws IOException ,InterruptedException { 50context .write ( 51key ,new Text (StreamSupport .stream (counts .spliterator (),false ) 52 .map (LongWritable ::get ) 53 .reduce (0l ,Long ::sum ) 54 .toString ())); 55 } 56 } 57 58public int run (String []args )throws Exception { 59Configuration conf =this .getConf (); 60 61Job job =new Job (conf ,"jbr2558_jkj858" ); 62job .setJarByClass (TextAnalyzer .class ); 63 64job .setMapperClass (TextMapper .class ); 65// job.setCombinerClass(TextCombiner.class); 66job .setReducerClass (TextReducer .class ); 67 68// Specify key / value types (Don't change them for the purpose of this 69// assignment) 70job .setOutputKeyClass (Text .class ); 71job .setOutputValueClass (Text .class ); 72// If your mapper and combiner's output types are different from 73// Text.class, then uncomment the following lines to specify the data 74// types. 75// job.setMapOutputKeyClass(String.class); 76job .setMapOutputValueClass (LongWritable .class ); 77 78// Input 79FileInputFormat .addInputPath (job ,new Path (args [0 ])); 80job .setInputFormatClass (TextInputFormat .class ); 81 82// Output 83FileOutputFormat .setOutputPath (job ,new Path (args [1 ])); 84job .setOutputFormatClass (TextOutputFormat .class ); 85 86// Execute job and return status 87return job .waitForCompletion (true ) ?0 :1 ; 88 } 89 90// Do not modify the main method 91public static void main (String []args )throws Exception { 92int res =ToolRunner .run (new Configuration (),new TextAnalyzer (),args ); 93System .exit (res ); 94 } 95// public static class Pair<A, B> { 96// public final A a; 97// public final B b; 98// public Pair(A a, B b) { 99// this.a = a; 100// this.b = b; 101// } 102// public String toString() { return "" + a + " " + b; } 103// public int hashCode() { return Objects.hash(a, b); } 104// } 105}