+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18package org .apache .hadoop .examples ; 19 20import java .util .Random ; 21 22import org .apache .hadoop .conf .Configuration ; 23import org .apache .hadoop .conf .Configured ; 24import org .apache .hadoop .fs .FileSystem ; 25import org .apache .hadoop .fs .Path ; 26import org .apache .hadoop .io .LongWritable ; 27import org .apache .hadoop .io .Text ; 28import org .apache .hadoop .mapreduce .*; 29import org .apache .hadoop .mapreduce .lib .input .FileInputFormat ; 30import org .apache .hadoop .mapreduce .lib .input .SequenceFileInputFormat ; 31import org .apache .hadoop .mapreduce .lib .map .InverseMapper ; 32import org .apache .hadoop .mapreduce .lib .map .RegexMapper ; 33import org .apache .hadoop .mapreduce .lib .output .FileOutputFormat ; 34import org .apache .hadoop .mapreduce .lib .output .SequenceFileOutputFormat ; 35import org .apache .hadoop .mapreduce .lib .reduce .LongSumReducer ; 36import org .apache .hadoop .util .Tool ; 37import org .apache .hadoop .util .ToolRunner ; 38 39/* Extracts matching regexs from input files and counts them. */ 40public class Grep extends Configured implements Tool { 41private Grep () {}// singleton 42 43public int run (String []args )throws Exception { 44if (args .length <3 ) { 45System .out .println ("Grep <inDir> <outDir> <regex> [<group>]" ); 46ToolRunner .printGenericCommandUsage (System .out ); 47return 2 ; 48 } 49 50Path tempDir = 51new Path ("grep-temp-" + 52Integer .toString (new Random ().nextInt (Integer .MAX_VALUE ))); 53 54Configuration conf =getConf (); 55conf .set (RegexMapper .PATTERN ,args [2 ]); 56if (args .length ==4 ) 57conf .set (RegexMapper .GROUP ,args [3 ]); 58 59Job grepJob =Job .getInstance (conf ); 60 61try { 62 63grepJob .setJobName ("grep-search" ); 64grepJob .setJarByClass (Grep .class ); 65 66FileInputFormat .setInputPaths (grepJob ,args [0 ]); 67 68grepJob .setMapperClass (RegexMapper .class ); 69 70grepJob .setCombinerClass (LongSumReducer .class ); 71grepJob .setReducerClass (LongSumReducer .class ); 72 73FileOutputFormat .setOutputPath (grepJob ,tempDir ); 74grepJob .setOutputFormatClass (SequenceFileOutputFormat .class ); 75grepJob .setOutputKeyClass (Text .class ); 76grepJob .setOutputValueClass (LongWritable .class ); 77 78grepJob .waitForCompletion (true ); 79 80Job sortJob =Job .getInstance (conf ); 81sortJob .setJobName ("grep-sort" ); 82sortJob .setJarByClass (Grep .class ); 83 84FileInputFormat .setInputPaths (sortJob ,tempDir ); 85sortJob .setInputFormatClass (SequenceFileInputFormat .class ); 86 87sortJob .setMapperClass (InverseMapper .class ); 88 89sortJob .setNumReduceTasks (1 );// write a single file 90FileOutputFormat .setOutputPath (sortJob ,new Path (args [1 ])); 91sortJob .setSortComparatorClass (// sort by decreasing freq 92LongWritable .DecreasingComparator .class ); 93 94sortJob .waitForCompletion (true ); 95 } 96finally { 97FileSystem .get (conf ).delete (tempDir ,true ); 98 } 99return 0 ; 100 } 101 102public static void main (String []args )throws Exception { 103int res =ToolRunner .run (new Configuration (),new Grep (),args ); 104System .exit (res ); 105 } 106 107}