+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18 19package org .apache .hadoop .examples ; 20 21import java .io .IOException ; 22import java .util .ArrayList ; 23import java .util .StringTokenizer ; 24import java .util .Map .Entry ; 25 26import org .apache .hadoop .io .Text ; 27import org .apache .hadoop .mapreduce .Job ; 28import org .apache .hadoop .mapreduce .lib .aggregate .ValueAggregatorBaseDescriptor ; 29import org .apache .hadoop .mapreduce .lib .aggregate .ValueAggregatorJob ; 30 31/** 32* This is an example Aggregated Hadoop Map/Reduce application. It reads the 33* text input files, breaks each line into words and counts them. The output is 34* a locally sorted list of words and the count of how often they occurred. 35* 36* To run: bin/hadoop jar hadoop-*-examples.jar aggregatewordcount 37* <i>in-dir</i> <i>out-dir</i> <i>numOfReducers</i> textinputformat 38* 39*/ 40public class AggregateWordCount { 41 42public static class WordCountPlugInClass extends 43ValueAggregatorBaseDescriptor { 44@ Override 45public ArrayList <Entry <Text ,Text >>generateKeyValPairs (Object key , 46Object val ) { 47String countType =LONG_VALUE_SUM ; 48ArrayList <Entry <Text ,Text >>retv =new ArrayList <Entry <Text ,Text >>(); 49String line =val .toString (); 50StringTokenizer itr =new StringTokenizer (line ); 51while (itr .hasMoreTokens ()) { 52Entry <Text ,Text >e =generateEntry (countType ,itr .nextToken (),ONE ); 53if (e !=null ) { 54retv .add (e ); 55 } 56 } 57return retv ; 58 } 59 } 60 61/** 62* The main driver for word count map/reduce program. Invoke this method to 63* submit the map/reduce job. 64* 65* @throws IOException 66* When there is communication problems with the job tracker. 67*/ 68@ SuppressWarnings ("unchecked" ) 69public static void main (String []args ) 70throws IOException ,InterruptedException ,ClassNotFoundException { 71Job job =ValueAggregatorJob .createValueAggregatorJob (args 72 ,new Class [] {WordCountPlugInClass .class }); 73job .setJarByClass (AggregateWordCount .class ); 74int ret =job .waitForCompletion (true ) ?0 :1 ; 75System .exit (ret ); 76 } 77 78}