+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18 19package org .apache .hadoop .examples .terasort ; 20 21import java .io .IOException ; 22 23import org .apache .hadoop .conf .Configuration ; 24import org .apache .hadoop .fs .FSDataOutputStream ; 25import org .apache .hadoop .fs .FileStatus ; 26import org .apache .hadoop .fs .FileSystem ; 27import org .apache .hadoop .fs .Path ; 28import org .apache .hadoop .io .Text ; 29import org .apache .hadoop .mapred .FileAlreadyExistsException ; 30import org .apache .hadoop .mapred .InvalidJobConfException ; 31import org .apache .hadoop .mapreduce .JobContext ; 32import org .apache .hadoop .mapreduce .OutputCommitter ; 33import org .apache .hadoop .mapreduce .RecordWriter ; 34import org .apache .hadoop .mapreduce .TaskAttemptContext ; 35import org .apache .hadoop .mapreduce .lib .output .FileOutputCommitter ; 36import org .apache .hadoop .mapreduce .lib .output .FileOutputFormat ; 37import org .apache .hadoop .mapreduce .security .TokenCache ; 38 39/** 40* An output format that writes the key and value appended together. 41*/ 42public class TeraOutputFormat extends FileOutputFormat <Text ,Text > { 43static final String FINAL_SYNC_ATTRIBUTE ="mapreduce.terasort.final.sync" ; 44private OutputCommitter committer =null ; 45 46/** 47* Set the requirement for a final sync before the stream is closed. 48*/ 49static void setFinalSync (JobContext job ,boolean newValue ) { 50job .getConfiguration ().setBoolean (FINAL_SYNC_ATTRIBUTE ,newValue ); 51 } 52 53/** 54* Does the user want a final sync at close? 55*/ 56public static boolean getFinalSync (JobContext job ) { 57return job .getConfiguration ().getBoolean (FINAL_SYNC_ATTRIBUTE ,false ); 58 } 59 60static class TeraRecordWriter extends RecordWriter <Text ,Text > { 61private boolean finalSync =false ; 62private FSDataOutputStream out ; 63 64public TeraRecordWriter (FSDataOutputStream out , 65JobContext job ) { 66finalSync =getFinalSync (job ); 67this .out =out ; 68 } 69 70public synchronized void write (Text key , 71Text value )throws IOException { 72out .write (key .getBytes (),0 ,key .getLength ()); 73out .write (value .getBytes (),0 ,value .getLength ()); 74 } 75 76public void close (TaskAttemptContext context )throws IOException { 77if (finalSync ) { 78out .sync (); 79 } 80out .close (); 81 } 82 } 83 84@ Override 85public void checkOutputSpecs (JobContext job 86 )throws InvalidJobConfException ,IOException { 87// Ensure that the output directory is set 88Path outDir =getOutputPath (job ); 89if (outDir ==null ) { 90throw new InvalidJobConfException ("Output directory not set in JobConf." ); 91 } 92 93final Configuration jobConf =job .getConfiguration (); 94 95// get delegation token for outDir's file system 96TokenCache .obtainTokensForNamenodes (job .getCredentials (), 97new Path [] {outDir },jobConf ); 98 99final FileSystem fs =outDir .getFileSystem (jobConf ); 100 101if (fs .exists (outDir )) { 102// existing output dir is considered empty iff its only content is the 103// partition file. 104// 105final FileStatus []outDirKids =fs .listStatus (outDir ); 106boolean empty =false ; 107if (outDirKids !=null &&outDirKids .length ==1 ) { 108final FileStatus st =outDirKids [0 ]; 109final String fname =st .getPath ().getName (); 110empty = 111 !st .isDirectory () &&TeraInputFormat .PARTITION_FILENAME .equals (fname ); 112 } 113if (TeraSort .getUseSimplePartitioner (job ) || !empty ) { 114throw new FileAlreadyExistsException ("Output directory " +outDir 115 +" already exists" ); 116 } 117 } 118 } 119 120public RecordWriter <Text ,Text >getRecordWriter (TaskAttemptContext job 121 )throws IOException { 122Path file =getDefaultWorkFile (job ,"" ); 123FileSystem fs =file .getFileSystem (job .getConfiguration ()); 124FSDataOutputStream fileOut =fs .create (file ); 125return new TeraRecordWriter (fileOut ,job ); 126 } 127 128public OutputCommitter getOutputCommitter (TaskAttemptContext context ) 129throws IOException { 130if (committer ==null ) { 131Path output =getOutputPath (context ); 132committer =new FileOutputCommitter (output ,context ); 133 } 134return committer ; 135 } 136 137}