+/EE360P

distributed systems

git clone https://git.julia.blue/+/EE360P

vijaygarg1init07ca65d

master
3.7 KiB107 linesraw
1/**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements.  See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership.  The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License.  You may obtain a copy of the License at
9 *
10 *     http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18package org.apache.hadoop.examples;
19
20import java.util.Random;
21
22import org.apache.hadoop.conf.Configuration;
23import org.apache.hadoop.conf.Configured;
24import org.apache.hadoop.fs.FileSystem;
25import org.apache.hadoop.fs.Path;
26import org.apache.hadoop.io.LongWritable;
27import org.apache.hadoop.io.Text;
28import org.apache.hadoop.mapreduce.*;
29import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
30import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
31import org.apache.hadoop.mapreduce.lib.map.InverseMapper;
32import org.apache.hadoop.mapreduce.lib.map.RegexMapper;
33import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
34import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
35import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer;
36import org.apache.hadoop.util.Tool;
37import org.apache.hadoop.util.ToolRunner;
38
39/* Extracts matching regexs from input files and counts them. */
40public class Grep extends Configured implements Tool {
41  private Grep() {}                               // singleton
42
43  public int run(String[] args) throws Exception {
44    if (args.length < 3) {
45      System.out.println("Grep <inDir> <outDir> <regex> [<group>]");
46      ToolRunner.printGenericCommandUsage(System.out);
47      return 2;
48    }
49
50    Path tempDir =
51      new Path("grep-temp-"+
52          Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
53
54    Configuration conf = getConf();
55    conf.set(RegexMapper.PATTERN, args[2]);
56    if (args.length == 4)
57      conf.set(RegexMapper.GROUP, args[3]);
58
59    Job grepJob = Job.getInstance(conf);
60    
61    try {
62      
63      grepJob.setJobName("grep-search");
64      grepJob.setJarByClass(Grep.class);
65
66      FileInputFormat.setInputPaths(grepJob, args[0]);
67
68      grepJob.setMapperClass(RegexMapper.class);
69
70      grepJob.setCombinerClass(LongSumReducer.class);
71      grepJob.setReducerClass(LongSumReducer.class);
72
73      FileOutputFormat.setOutputPath(grepJob, tempDir);
74      grepJob.setOutputFormatClass(SequenceFileOutputFormat.class);
75      grepJob.setOutputKeyClass(Text.class);
76      grepJob.setOutputValueClass(LongWritable.class);
77
78      grepJob.waitForCompletion(true);
79
80      Job sortJob = Job.getInstance(conf);
81      sortJob.setJobName("grep-sort");
82      sortJob.setJarByClass(Grep.class);
83
84      FileInputFormat.setInputPaths(sortJob, tempDir);
85      sortJob.setInputFormatClass(SequenceFileInputFormat.class);
86
87      sortJob.setMapperClass(InverseMapper.class);
88
89      sortJob.setNumReduceTasks(1);                 // write a single file
90      FileOutputFormat.setOutputPath(sortJob, new Path(args[1]));
91      sortJob.setSortComparatorClass(          // sort by decreasing freq
92        LongWritable.DecreasingComparator.class);
93
94      sortJob.waitForCompletion(true);
95    }
96    finally {
97      FileSystem.get(conf).delete(tempDir, true);
98    }
99    return 0;
100  }
101
102  public static void main(String[] args) throws Exception {
103    int res = ToolRunner.run(new Configuration(), new Grep(), args);
104    System.exit(res);
105  }
106
107}