+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18package org .apache .hadoop .examples .terasort ; 19 20import java .io .IOException ; 21import java .util .zip .Checksum ; 22 23import org .apache .hadoop .conf .Configuration ; 24import org .apache .hadoop .conf .Configured ; 25import org .apache .hadoop .fs .Path ; 26import org .apache .hadoop .io .NullWritable ; 27import org .apache .hadoop .io .Text ; 28import org .apache .hadoop .mapreduce .Cluster ; 29import org .apache .hadoop .mapreduce .Job ; 30import org .apache .hadoop .mapreduce .Mapper ; 31import org .apache .hadoop .mapreduce .Reducer ; 32import org .apache .hadoop .mapreduce .lib .output .FileOutputFormat ; 33import org .apache .hadoop .util .PureJavaCrc32 ; 34import org .apache .hadoop .util .Tool ; 35import org .apache .hadoop .util .ToolRunner ; 36 37public class TeraChecksum extends Configured implements Tool { 38static class ChecksumMapper 39extends Mapper <Text ,Text ,NullWritable ,Unsigned16 > { 40private Unsigned16 checksum =new Unsigned16 (); 41private Unsigned16 sum =new Unsigned16 (); 42private Checksum crc32 =new PureJavaCrc32 (); 43 44public void map (Text key ,Text value , 45Context context )throws IOException { 46crc32 .reset (); 47crc32 .update (key .getBytes (),0 ,key .getLength ()); 48crc32 .update (value .getBytes (),0 ,value .getLength ()); 49checksum .set (crc32 .getValue ()); 50sum .add (checksum ); 51 } 52 53public void cleanup (Context context ) 54throws IOException ,InterruptedException { 55context .write (NullWritable .get (),sum ); 56 } 57 } 58 59static class ChecksumReducer 60extends Reducer <NullWritable ,Unsigned16 ,NullWritable ,Unsigned16 > { 61 62public void reduce (NullWritable key ,Iterable <Unsigned16 >values , 63Context context )throws IOException ,InterruptedException { 64Unsigned16 sum =new Unsigned16 (); 65for (Unsigned16 val :values ) { 66sum .add (val ); 67 } 68context .write (key ,sum ); 69 } 70 } 71 72private static void usage ()throws IOException { 73System .err .println ("terasum <out-dir> <report-dir>" ); 74 } 75 76public int run (String []args )throws Exception { 77Job job =Job .getInstance (getConf ()); 78if (args .length !=2 ) { 79usage (); 80return 2 ; 81 } 82TeraInputFormat .setInputPaths (job ,new Path (args [0 ])); 83FileOutputFormat .setOutputPath (job ,new Path (args [1 ])); 84job .setJobName ("TeraSum" ); 85job .setJarByClass (TeraChecksum .class ); 86job .setMapperClass (ChecksumMapper .class ); 87job .setReducerClass (ChecksumReducer .class ); 88job .setOutputKeyClass (NullWritable .class ); 89job .setOutputValueClass (Unsigned16 .class ); 90// force a single reducer 91job .setNumReduceTasks (1 ); 92job .setInputFormatClass (TeraInputFormat .class ); 93return job .waitForCompletion (true ) ?0 :1 ; 94 } 95 96/** 97* @param args 98*/ 99public static void main (String []args )throws Exception { 100int res =ToolRunner .run (new Configuration (),new TeraChecksum (),args ); 101System .exit (res ); 102 } 103 104}