+/EE360P

distributed systems

git clone https://git.julia.blue/+/EE360P

vijaygarg1init07ca65d

master
11.1 KiB333 linesraw
1/**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements.  See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership.  The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License.  You may obtain a copy of the License at
9 *
10 *     http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18package org.apache.hadoop.examples.pi;
19
20import java.io.BufferedReader;
21import java.io.File;
22import java.io.FileOutputStream;
23import java.io.IOException;
24import java.io.InputStreamReader;
25import java.io.OutputStreamWriter;
26import java.io.PrintStream;
27import java.io.PrintWriter;
28import java.text.SimpleDateFormat;
29import java.util.ArrayList;
30import java.util.Arrays;
31import java.util.Collection;
32import java.util.Collections;
33import java.util.Date;
34import java.util.List;
35import java.util.concurrent.Callable;
36import java.util.concurrent.ExecutionException;
37import java.util.concurrent.ExecutorService;
38import java.util.concurrent.Executors;
39import java.util.concurrent.Future;
40import java.util.concurrent.Semaphore;
41
42import org.apache.hadoop.examples.pi.DistSum.Machine;
43import org.apache.hadoop.fs.FileStatus;
44import org.apache.hadoop.fs.FileSystem;
45import org.apache.hadoop.fs.Path;
46import org.apache.hadoop.fs.permission.FsPermission;
47import org.apache.hadoop.mapreduce.Job;
48import org.apache.hadoop.util.ToolRunner;
49
50import com.google.common.base.Charsets;
51
52/** Utility methods */
53public class Util {
54  /** Output stream */
55  public static final PrintStream out = System.out; 
56  /** Error stream */
57  public static final PrintStream err = System.out; 
58
59  /** Timer */
60  public static class Timer {
61    private final boolean isAccumulative;
62    private final long start = System.currentTimeMillis();
63    private long previous = start;
64  
65    /** Timer constructor
66     * @param isAccumulative  Is accumulating the time duration?
67     */
68    public Timer(boolean isAccumulative) {
69      this.isAccumulative = isAccumulative;
70      final StackTraceElement[] stack = Thread.currentThread().getStackTrace();
71      final StackTraceElement e = stack[stack.length - 1];
72      out.println(e + " started at " + new Date(start));
73    }
74  
75    /** Same as tick(null). */
76    public long tick() {return tick(null);}
77
78    /**
79     * Tick
80     * @param s Output message.  No output if it is null.
81     * @return delta
82     */
83    public synchronized long tick(String s) {
84      final long t = System.currentTimeMillis();
85      final long delta = t - (isAccumulative? start: previous);
86      if (s != null) {
87        out.format("%15dms (=%-15s: %s%n", delta, millis2String(delta) + ")", s);
88        out.flush();
89      }
90      previous = t;
91      return delta;
92    }
93  }
94    
95  /** Covert milliseconds to a String. */
96  public static String millis2String(long n) {
97    if (n < 0)
98      return "-" + millis2String(-n);
99    else if (n < 1000)
100      return n + "ms";
101
102    final StringBuilder b = new StringBuilder();
103    final int millis = (int)(n % 1000L);
104    if (millis != 0)
105      b.append(String.format(".%03d", millis)); 
106    if ((n /= 1000) < 60)
107      return b.insert(0, n).append("s").toString();
108
109    b.insert(0, String.format(":%02d", (int)(n % 60L)));
110    if ((n /= 60) < 60)
111      return b.insert(0, n).toString();
112
113    b.insert(0, String.format(":%02d", (int)(n % 60L)));
114    if ((n /= 60) < 24)
115      return b.insert(0, n).toString();
116
117    b.insert(0, n % 24L);
118    final int days = (int)((n /= 24) % 365L);
119    b.insert(0, days == 1? " day ": " days ").insert(0, days);
120    if ((n /= 365L) > 0)
121      b.insert(0, n == 1? " year ": " years ").insert(0, n);
122
123    return b.toString();
124  }
125
126  /** Covert a String to a long.  
127   * This support comma separated number format.
128   */
129  public static long string2long(String s) {
130    return Long.parseLong(s.trim().replace(",", ""));
131  }
132
133  /** Covert a long to a String in comma separated number format. */  
134  public static String long2string(long n) {
135    if (n < 0)
136      return "-" + long2string(-n);
137    
138    final StringBuilder b = new StringBuilder();
139    for(; n >= 1000; n = n/1000)
140      b.insert(0, String.format(",%03d", n % 1000));
141    return n + b.toString();    
142  }
143
144  /** Parse a variable. */  
145  public static long parseLongVariable(final String name, final String s) {
146    return string2long(parseStringVariable(name, s));
147  }
148
149  /** Parse a variable. */  
150  public static String parseStringVariable(final String name, final String s) {
151    if (!s.startsWith(name + '='))
152      throw new IllegalArgumentException("!s.startsWith(name + '='), name="
153          + name + ", s=" + s);
154    return s.substring(name.length() + 1);
155  }
156
157  /** Execute the callables by a number of threads */
158  public static <T, E extends Callable<T>> void execute(int nThreads, List<E> callables
159      ) throws InterruptedException, ExecutionException {
160    final ExecutorService executor = Executors.newFixedThreadPool(nThreads); 
161    final List<Future<T>> futures = executor.invokeAll(callables);
162    for(Future<T> f : futures)
163      f.get();
164  }
165
166  /** Print usage messages */
167  public static int printUsage(String[] args, String usage) {
168    err.println("args = " + Arrays.asList(args));
169    err.println();
170    err.println("Usage: java " + usage);
171    err.println();
172    ToolRunner.printGenericCommandUsage(err);
173    return -1;
174  }
175
176  /** Combine a list of items. */
177  public static <T extends Combinable<T>> List<T> combine(Collection<T> items) {
178    final List<T> sorted = new ArrayList<T>(items);
179    if (sorted.size() <= 1)
180      return sorted;
181
182    Collections.sort(sorted);
183    final List<T> combined = new ArrayList<T>(items.size());
184    T prev = sorted.get(0);
185    for(int i = 1; i < sorted.size(); i++) {
186      final T curr = sorted.get(i);
187      final T c = curr.combine(prev);
188
189      if (c != null)
190        prev = c;
191      else {
192        combined.add(prev);
193        prev = curr;
194      }
195    }
196    combined.add(prev);
197    return combined;
198  }
199
200  /** Check local directory. */
201  public static void checkDirectory(File dir) {
202    if (!dir.exists())
203      if (!dir.mkdirs())
204        throw new IllegalArgumentException("!dir.mkdirs(), dir=" + dir);
205    if (!dir.isDirectory())
206      throw new IllegalArgumentException("dir (=" + dir + ") is not a directory.");
207  }
208
209  /** Create a writer of a local file. */
210  public static PrintWriter createWriter(File dir, String prefix) throws IOException {
211    checkDirectory(dir);
212    
213    SimpleDateFormat dateFormat = new SimpleDateFormat("-yyyyMMdd-HHmmssSSS");
214    for(;;) {
215      final File f = new File(dir,
216          prefix + dateFormat.format(new Date(System.currentTimeMillis())) + ".txt");
217      if (!f.exists())
218        return new PrintWriter(new OutputStreamWriter(new FileOutputStream(f), Charsets.UTF_8));
219
220      try {Thread.sleep(10);} catch (InterruptedException e) {}
221    }
222  }
223
224  /** Print a "bits skipped" message. */
225  public static void printBitSkipped(final long b) {
226    out.println();
227    out.println("b = " + long2string(b)
228        + " (" + (b < 2? "bit": "bits") + " skipped)");
229  }
230
231  /** Convert a pi value to a String. */
232  public static String pi2string(final double pi, final long terms) {
233    final long value = (long)(pi * (1L << DOUBLE_PRECISION));
234    final int acc_bit = accuracy(terms, false);
235    final int acc_hex = acc_bit/4;
236    final int shift = DOUBLE_PRECISION - acc_bit;
237    return String.format("%0" + acc_hex + "X %0" + (13-acc_hex) + "X (%d hex digits)",
238        value >> shift, value & ((1 << shift) - 1), acc_hex);
239  }
240
241  static final int DOUBLE_PRECISION = 52; //mantissa size
242  static final int MACHEPS_EXPONENT = DOUBLE_PRECISION + 1;
243  /** Estimate accuracy. */
244  public static int accuracy(final long terms, boolean print) {
245    final double error = terms <= 0? 2: (Math.log(terms) / Math.log(2)) / 2;
246    final int bits = MACHEPS_EXPONENT - (int)Math.ceil(error);
247    if (print)
248      out.println("accuracy: bits=" + bits + ", terms=" + long2string(terms) + ", error exponent=" + error);
249    return bits - bits%4;
250  }
251
252  private static final String JOB_SEPARATION_PROPERTY = "pi.job.separation.seconds";
253  private static final Semaphore JOB_SEMAPHORE = new Semaphore(1);
254
255  /** Run a job. */
256  static void runJob(String name, Job job, Machine machine, String startmessage, Util.Timer timer) {
257    JOB_SEMAPHORE.acquireUninterruptibly();
258    Long starttime = null;
259    try {
260      try {
261        starttime = timer.tick("starting " + name + " ...\n  " + startmessage);
262
263        //initialize and submit a job
264        machine.init(job);
265        job.submit();
266  
267        // Separate jobs
268        final long sleeptime = 1000L * job.getConfiguration().getInt(JOB_SEPARATION_PROPERTY, 10);
269        if (sleeptime > 0) {
270          Util.out.println(name + "> sleep(" + Util.millis2String(sleeptime) + ")");
271          Thread.sleep(sleeptime);
272        }
273      } finally {
274        JOB_SEMAPHORE.release();
275      }
276  
277      if (!job.waitForCompletion(false))
278        throw new RuntimeException(name + " failed.");
279    } catch(Exception e) {
280      throw e instanceof RuntimeException? (RuntimeException)e: new RuntimeException(e);
281    } finally {
282      if (starttime != null)
283        timer.tick(name + "> timetaken=" + Util.millis2String(timer.tick() - starttime));
284    }
285  }
286
287  /** Read job outputs */
288  static List<TaskResult> readJobOutputs(FileSystem fs, Path outdir) throws IOException {
289    final List<TaskResult> results = new ArrayList<TaskResult>();
290    for(FileStatus status : fs.listStatus(outdir)) {
291      if (status.getPath().getName().startsWith("part-")) {
292        final BufferedReader in = new BufferedReader(
293            new InputStreamReader(fs.open(status.getPath()), Charsets.UTF_8));
294        try {
295          for(String line; (line = in.readLine()) != null; )
296            results.add(TaskResult.valueOf(line));
297        }
298        finally {
299          in.close();
300        }
301      }
302    }
303    if (results.isEmpty())
304      throw new IOException("Output not found");
305    return results;
306  }
307  
308  /** Write results */
309  static void writeResults(String name, List<TaskResult> results, FileSystem fs, String dir) throws IOException {
310    final Path outfile = new Path(dir, name + ".txt");
311    Util.out.println(name + "> writing results to " + outfile);
312    final PrintWriter out = new PrintWriter(new OutputStreamWriter(fs.create(outfile), Charsets.UTF_8), true);
313    try {
314      for(TaskResult r : results)
315        out.println(r);
316    }
317    finally {
318      out.close();
319    }
320  }
321
322  /** Create a directory. */
323  static boolean createNonexistingDirectory(FileSystem fs, Path dir) throws IOException {
324    if (fs.exists(dir)) {
325      Util.err.println("dir (= " + dir + ") already exists.");
326      return false;
327    } else if (!fs.mkdirs(dir)) {
328      throw new IOException("Cannot create working directory " + dir);
329    }
330    fs.setPermission(dir, new FsPermission((short)0777));
331    return true;
332  }
333}