+/EE360P

distributed systems

git clone https://git.julia.blue/+/EE360P

vijaygarg1init07ca65d

master
4.8 KiB137 linesraw
1/**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements.  See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership.  The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License.  You may obtain a copy of the License at
9 *
10 *     http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19package org.apache.hadoop.examples.terasort;
20
21import java.io.IOException;
22
23import org.apache.hadoop.conf.Configuration;
24import org.apache.hadoop.fs.FSDataOutputStream;
25import org.apache.hadoop.fs.FileStatus;
26import org.apache.hadoop.fs.FileSystem;
27import org.apache.hadoop.fs.Path;
28import org.apache.hadoop.io.Text;
29import org.apache.hadoop.mapred.FileAlreadyExistsException;
30import org.apache.hadoop.mapred.InvalidJobConfException;
31import org.apache.hadoop.mapreduce.JobContext;
32import org.apache.hadoop.mapreduce.OutputCommitter;
33import org.apache.hadoop.mapreduce.RecordWriter;
34import org.apache.hadoop.mapreduce.TaskAttemptContext;
35import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
36import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
37import org.apache.hadoop.mapreduce.security.TokenCache;
38
39/**
40 * An output format that writes the key and value appended together.
41 */
42public class TeraOutputFormat extends FileOutputFormat<Text,Text> {
43  static final String FINAL_SYNC_ATTRIBUTE = "mapreduce.terasort.final.sync";
44  private OutputCommitter committer = null;
45
46  /**
47   * Set the requirement for a final sync before the stream is closed.
48   */
49  static void setFinalSync(JobContext job, boolean newValue) {
50    job.getConfiguration().setBoolean(FINAL_SYNC_ATTRIBUTE, newValue);
51  }
52
53  /**
54   * Does the user want a final sync at close?
55   */
56  public static boolean getFinalSync(JobContext job) {
57    return job.getConfiguration().getBoolean(FINAL_SYNC_ATTRIBUTE, false);
58  }
59
60  static class TeraRecordWriter extends RecordWriter<Text,Text> {
61    private boolean finalSync = false;
62    private FSDataOutputStream out;
63
64    public TeraRecordWriter(FSDataOutputStream out,
65                            JobContext job) {
66      finalSync = getFinalSync(job);
67      this.out = out;
68    }
69
70    public synchronized void write(Text key, 
71                                   Text value) throws IOException {
72      out.write(key.getBytes(), 0, key.getLength());
73      out.write(value.getBytes(), 0, value.getLength());
74    }
75    
76    public void close(TaskAttemptContext context) throws IOException {
77      if (finalSync) {
78        out.sync();
79      }
80      out.close();
81    }
82  }
83
84  @Override
85  public void checkOutputSpecs(JobContext job
86                              ) throws InvalidJobConfException, IOException {
87    // Ensure that the output directory is set
88    Path outDir = getOutputPath(job);
89    if (outDir == null) {
90      throw new InvalidJobConfException("Output directory not set in JobConf.");
91    }
92
93    final Configuration jobConf = job.getConfiguration();
94
95    // get delegation token for outDir's file system
96    TokenCache.obtainTokensForNamenodes(job.getCredentials(),
97        new Path[] { outDir }, jobConf);
98
99    final FileSystem fs = outDir.getFileSystem(jobConf);
100
101    if (fs.exists(outDir)) {
102      // existing output dir is considered empty iff its only content is the
103      // partition file.
104      //
105      final FileStatus[] outDirKids = fs.listStatus(outDir);
106      boolean empty = false;
107      if (outDirKids != null && outDirKids.length == 1) {
108        final FileStatus st = outDirKids[0];
109        final String fname = st.getPath().getName();
110        empty =
111          !st.isDirectory() && TeraInputFormat.PARTITION_FILENAME.equals(fname);
112      }
113      if (TeraSort.getUseSimplePartitioner(job) || !empty) {
114        throw new FileAlreadyExistsException("Output directory " + outDir
115            + " already exists");
116      }
117    }
118  }
119
120  public RecordWriter<Text,Text> getRecordWriter(TaskAttemptContext job
121                                                 ) throws IOException {
122    Path file = getDefaultWorkFile(job, "");
123    FileSystem fs = file.getFileSystem(job.getConfiguration());
124     FSDataOutputStream fileOut = fs.create(file);
125    return new TeraRecordWriter(fileOut, job);
126  }
127  
128  public OutputCommitter getOutputCommitter(TaskAttemptContext context) 
129      throws IOException {
130    if (committer == null) {
131      Path output = getOutputPath(context);
132      committer = new FileOutputCommitter(output, context);
133    }
134    return committer;
135  }
136
137}