+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18 19package org .apache .hadoop .examples ; 20 21import java .io .IOException ; 22import java .util .ArrayList ; 23import java .util .Map .Entry ; 24 25import org .apache .hadoop .io .Text ; 26import org .apache .hadoop .mapreduce .Job ; 27import org .apache .hadoop .mapreduce .lib .aggregate .ValueAggregatorBaseDescriptor ; 28import org .apache .hadoop .mapreduce .lib .aggregate .ValueAggregatorJob ; 29 30/** 31* This is an example Aggregated Hadoop Map/Reduce application. Computes the 32* histogram of the words in the input texts. 33* 34* To run: bin/hadoop jar hadoop-*-examples.jar aggregatewordhist <i>in-dir</i> 35* <i>out-dir</i> <i>numOfReducers</i> textinputformat 36* 37*/ 38public class AggregateWordHistogram { 39 40public static class AggregateWordHistogramPlugin 41extends ValueAggregatorBaseDescriptor { 42 43/** 44* Parse the given value, generate an aggregation-id/value pair per word. 45* The ID is of type VALUE_HISTOGRAM, with WORD_HISTOGRAM as the real id. 46* The value is WORD\t1. 47* 48* @return a list of the generated pairs. 49*/ 50@ Override 51public ArrayList <Entry <Text ,Text >>generateKeyValPairs (Object key , 52Object val ) { 53String words [] =val .toString ().split (" |\t" ); 54ArrayList <Entry <Text ,Text >>retv =new ArrayList <Entry <Text ,Text >>(); 55for (int i =0 ;i <words .length ;i ++) { 56Text valCount =new Text (words [i ] +"\t" +"1" ); 57Entry <Text ,Text >en =generateEntry (VALUE_HISTOGRAM ,"WORD_HISTOGRAM" , 58valCount ); 59retv .add (en ); 60 } 61return retv ; 62 } 63 64 } 65 66/** 67* The main driver for word count map/reduce program. Invoke this method to 68* submit the map/reduce job. 69* 70* @throws IOException 71* When there is communication problems with the job tracker. 72*/ 73@ SuppressWarnings ("unchecked" ) 74public static void main (String []args ) 75throws IOException ,InterruptedException ,ClassNotFoundException { 76Job job =ValueAggregatorJob .createValueAggregatorJob (args 77 ,new Class [] {AggregateWordHistogramPlugin .class }); 78job .setJarByClass (AggregateWordCount .class ); 79int ret =job .waitForCompletion (true ) ?0 :1 ; 80System .exit (ret ); 81 } 82}