+/EE360P

distributed systems

git clone https://git.julia.blue/+/EE360P

vijaygarg1init07ca65d

master
6.4 KiB192 linesraw
1/**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements.  See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership.  The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License.  You may obtain a copy of the License at
9 *
10 *     http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18package org.apache.hadoop.examples.pi;
19
20import java.io.BufferedReader;
21import java.io.File;
22import java.io.FileInputStream;
23import java.io.FileOutputStream;
24import java.io.IOException;
25import java.io.InputStreamReader;
26import java.io.OutputStreamWriter;
27import java.io.PrintWriter;
28import java.util.ArrayList;
29import java.util.Collections;
30import java.util.List;
31import java.util.Map;
32import java.util.TreeMap;
33
34import org.apache.hadoop.examples.pi.math.Bellard;
35import org.apache.hadoop.examples.pi.math.Bellard.Parameter;
36
37import com.google.common.base.Charsets;
38
39/** A class for parsing outputs */
40public final class Parser {
41  static final String VERBOSE_PROPERTY = "pi.parser.verbose";
42
43  final boolean isVerbose;
44  
45  public Parser(boolean isVerbose) {
46    this.isVerbose = isVerbose;
47  }
48  
49  private void println(String s) {
50    if (isVerbose)
51      Util.out.println(s);
52  }
53
54  /** Parse a line */
55  private static void parseLine(final String line, Map<Parameter, List<TaskResult>> m) {
56//      LOG.info("line = " + line);
57    final Map.Entry<String, TaskResult> e = DistSum.string2TaskResult(line);
58    if (e != null) {
59      final List<TaskResult> sums = m.get(Parameter.get(e.getKey()));
60      if (sums == null)
61        throw new IllegalArgumentException("sums == null, line=" + line + ", e=" + e);
62      sums.add(e.getValue());
63    }
64  }
65
66  /** Parse a file or a directory tree */
67  private void parse(File f, Map<Parameter, List<TaskResult>> sums) throws IOException {
68    if (f.isDirectory()) {
69      println("Process directory " + f);
70      for(File child : f.listFiles())
71        parse(child, sums);
72    } else if (f.getName().endsWith(".txt")) {
73      println("Parse file " + f);
74      final Map<Parameter, List<TaskResult>> m = new TreeMap<Parameter, List<TaskResult>>();    
75      for(Parameter p : Parameter.values())
76        m.put(p, new ArrayList<TaskResult>());
77
78      final BufferedReader in = new BufferedReader(
79          new InputStreamReader(new FileInputStream(f), Charsets.UTF_8)); 
80      try {
81        for(String line; (line = in.readLine()) != null; )
82          try {
83            parseLine(line, m);
84          } catch(RuntimeException e) {
85            Util.err.println("line = " + line);
86            throw e;
87          }
88      } finally {
89        in.close();
90      }
91
92      for(Parameter p : Parameter.values()) {
93        final List<TaskResult> combined = Util.combine(m.get(p));
94        if (!combined.isEmpty()) {
95          println(p + " (size=" + combined.size() + "):");
96          for(TaskResult r : combined)
97            println("  " + r);
98        }
99        sums.get(p).addAll(m.get(p));
100      }
101    }
102  }
103
104  /** Parse a path */
105  private Map<Parameter, List<TaskResult>> parse(String f) throws IOException {
106    final Map<Parameter, List<TaskResult>> m = new TreeMap<Parameter, List<TaskResult>>();
107    for(Parameter p : Parameter.values())
108      m.put(p, new ArrayList<TaskResult>());
109    parse(new File(f), m);
110
111    //LOG.info("m=" + m.toString().replace(", ", ",\n  "));
112    for(Parameter p : Parameter.values())
113      m.put(p, m.get(p));
114    return m;
115  }
116
117  /** Parse input and re-write results. */
118  Map<Parameter, List<TaskResult>> parse(String inputpath, String outputdir
119      ) throws IOException {
120    //parse input
121    Util.out.print("\nParsing " + inputpath + " ... ");
122    Util.out.flush();
123    final Map<Parameter, List<TaskResult>> parsed = parse(inputpath);
124    Util.out.println("DONE");
125
126    //re-write the results
127    if (outputdir != null) {
128      Util.out.print("\nWriting to " + outputdir + " ...");
129      Util.out.flush();
130      for(Parameter p : Parameter.values()) {
131        final List<TaskResult> results = parsed.get(p);
132        Collections.sort(results);
133
134        final PrintWriter out = new PrintWriter(
135            new OutputStreamWriter(new FileOutputStream(
136                new File(outputdir, p + ".txt")), Charsets.UTF_8), true);
137        try {
138          for(int i = 0; i < results.size(); i++)
139            out.println(DistSum.taskResult2string(p + "." + i, results.get(i)));
140        }
141        finally {
142          out.close();
143        }
144      }
145      Util.out.println("DONE");
146    }
147    return parsed;
148  }
149
150  /** Combine results */
151  static <T extends Combinable<T>> Map<Parameter, T> combine(Map<Parameter, List<T>> m) {
152    final Map<Parameter, T> combined = new TreeMap<Parameter, T>();
153    for(Parameter p : Parameter.values()) {
154      //note: results would never be null due to the design of Util.combine
155      final List<T> results = Util.combine(m.get(p));
156      Util.out.format("%-6s => ", p); 
157      if (results.size() != 1)
158        Util.out.println(results.toString().replace(", ", ",\n           "));
159      else {
160        final T r = results.get(0);
161        combined.put(p, r); 
162        Util.out.println(r);
163      }
164    }
165    return combined;
166  }
167
168  /** main */
169  public static void main(String[] args) throws IOException {
170    if (args.length < 2 || args.length > 3)
171      Util.printUsage(args, Parser.class.getName()
172          + " <b> <inputpath> [<outputdir>]");
173
174    int i = 0;
175    final long b = Util.string2long(args[i++]);
176    final String inputpath = args[i++];
177    final String outputdir = args.length >= 3? args[i++]: null;
178
179    //read input
180    final Map<Parameter, List<TaskResult>> parsed = new Parser(true).parse(inputpath, outputdir);
181    final Map<Parameter, TaskResult> combined = combine(parsed);
182    long duration = 0;
183    for(TaskResult r : combined.values())
184      duration += r.getDuration();
185
186    //print pi
187    final double pi = Bellard.computePi(b, combined);
188    Util.printBitSkipped(b);
189    Util.out.println(Util.pi2string(pi, Bellard.bit2terms(b)));
190    Util.out.println("cpu time = " + Util.millis2String(duration));
191  }
192}