+/EE360P

distributed systems

git clone https://git.julia.blue/+/EE360P

Joseph RyanI love javabde7f97

master
4.0 KiB105 linesraw
1import java.io.IOException;
2import java.util.*;
3import java.util.stream.*;
4import org.apache.hadoop.conf.*;
5import org.apache.hadoop.fs.Path;
6import org.apache.hadoop.io.*;
7import org.apache.hadoop.mapreduce.*;
8import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
9import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
10import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
11import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
12import org.apache.hadoop.util.Tool;
13import org.apache.hadoop.util.ToolRunner;
14
15public class TextAnalyzer extends Configured implements Tool {
16    public static class TextMapper
17        extends Mapper<LongWritable, Text, Text, LongWritable> {
18        public void map(LongWritable lineNum, Text rawLine, Context context)
19            throws IOException, InterruptedException {
20            Set<String> line = Stream
21                                   .of(rawLine.toString()
22                                           .toLowerCase()
23                                           .replaceAll("[^a-z0-9]", " ")
24                                           .trim()
25                                           .split("\\s+"))
26                                   .collect(Collectors.toSet());
27            for (String a : line) {
28                for (String b : line) {
29                    if (!a.equals(b)) {
30                        context.write(new Text(a + " " + b),
31                                      new LongWritable(1));
32                    }
33                }
34            }
35        }
36    }
37
38    // public static class TextCombiner extends Reducer<?, ?, ?, ?> {
39    //     public void reduce(Text key, Iterable<Tuple> tuples, Context context)
40    //         throws IOException, InterruptedException {
41    //         // Implementation of you combiner function
42    //     }
43    // }
44
45    public static class TextReducer
46        extends Reducer<Text, LongWritable, Text, Text> {
47        public void reduce(Text key, Iterable<LongWritable> counts,
48                           Context context)
49            throws IOException, InterruptedException {
50            context.write(
51                key, new Text(StreamSupport.stream(counts.spliterator(), false)
52                                  .map(LongWritable::get)
53                                  .reduce(0l, Long::sum)
54                                  .toString()));
55        }
56    }
57
58    public int run(String[] args) throws Exception {
59        Configuration conf = this.getConf();
60
61        Job job = new Job(conf, "jbr2558_jkj858");
62        job.setJarByClass(TextAnalyzer.class);
63
64        job.setMapperClass(TextMapper.class);
65        // job.setCombinerClass(TextCombiner.class);
66        job.setReducerClass(TextReducer.class);
67
68        // Specify key / value types (Don't change them for the purpose of this
69        // assignment)
70        job.setOutputKeyClass(Text.class);
71        job.setOutputValueClass(Text.class);
72        //   If your mapper and combiner's  output types are different from
73        //   Text.class, then uncomment the following lines to specify the data
74        //   types.
75        // job.setMapOutputKeyClass(String.class);
76        job.setMapOutputValueClass(LongWritable.class);
77
78        // Input
79        FileInputFormat.addInputPath(job, new Path(args[0]));
80        job.setInputFormatClass(TextInputFormat.class);
81
82        // Output
83        FileOutputFormat.setOutputPath(job, new Path(args[1]));
84        job.setOutputFormatClass(TextOutputFormat.class);
85
86        // Execute job and return status
87        return job.waitForCompletion(true) ? 0 : 1;
88    }
89
90    // Do not modify the main method
91    public static void main(String[] args) throws Exception {
92        int res = ToolRunner.run(new Configuration(), new TextAnalyzer(), args);
93        System.exit(res);
94    }
95    // public static class Pair<A, B> {
96    //     public final A a;
97    //     public final B b;
98    //     public Pair(A a, B b) {
99    //         this.a = a;
100    //         this.b = b;
101    //     }
102    //     public String toString() { return "" + a + " " + b; }
103    //     public int hashCode() { return Objects.hash(a, b); }
104    // }
105}