+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18package org .apache .hadoop .examples .pi ; 19 20import java .io .BufferedReader ; 21import java .io .File ; 22import java .io .FileInputStream ; 23import java .io .FileOutputStream ; 24import java .io .IOException ; 25import java .io .InputStreamReader ; 26import java .io .OutputStreamWriter ; 27import java .io .PrintWriter ; 28import java .util .ArrayList ; 29import java .util .Collections ; 30import java .util .List ; 31import java .util .Map ; 32import java .util .TreeMap ; 33 34import org .apache .hadoop .examples .pi .math .Bellard ; 35import org .apache .hadoop .examples .pi .math .Bellard .Parameter ; 36 37import com .common .base .Charsets ; 38 39/** A class for parsing outputs */ 40public final class Parser { 41static final String VERBOSE_PROPERTY ="pi.parser.verbose" ; 42 43final boolean isVerbose ; 44 45public Parser (boolean isVerbose ) { 46this .isVerbose =isVerbose ; 47 } 48 49private void println (String s ) { 50if (isVerbose ) 51Util .out .println (s ); 52 } 53 54/** Parse a line */ 55private static void parseLine (final String line ,Map <Parameter ,List <TaskResult >>m ) { 56// LOG.info("line = " + line); 57final Map .Entry <String ,TaskResult >e =DistSum .string2TaskResult (line ); 58if (e !=null ) { 59final List <TaskResult >sums =m .get (Parameter .get (e .getKey ())); 60if (sums ==null ) 61throw new IllegalArgumentException ("sums == null, line=" +line +", e=" +e ); 62sums .add (e .getValue ()); 63 } 64 } 65 66/** Parse a file or a directory tree */ 67private void parse (File f ,Map <Parameter ,List <TaskResult >>sums )throws IOException { 68if (f .isDirectory ()) { 69println ("Process directory " +f ); 70for (File child :f .listFiles ()) 71parse (child ,sums ); 72 }else if (f .getName ().endsWith (".txt" )) { 73println ("Parse file " +f ); 74final Map <Parameter ,List <TaskResult >>m =new TreeMap <Parameter ,List <TaskResult >>(); 75for (Parameter p :Parameter .values ()) 76m .put (p ,new ArrayList <TaskResult >()); 77 78final BufferedReader in =new BufferedReader ( 79new InputStreamReader (new FileInputStream (f ),Charsets .UTF_8 )); 80try { 81for (String line ; (line =in .readLine ()) !=null ; ) 82try { 83parseLine (line ,m ); 84 }catch (RuntimeException e ) { 85Util .err .println ("line = " +line ); 86throw e ; 87 } 88 }finally { 89in .close (); 90 } 91 92for (Parameter p :Parameter .values ()) { 93final List <TaskResult >combined =Util .combine (m .get (p )); 94if (!combined .isEmpty ()) { 95println (p +" (size=" +combined .size () +"):" ); 96for (TaskResult r :combined ) 97println (" " +r ); 98 } 99sums .get (p ).addAll (m .get (p )); 100 } 101 } 102 } 103 104/** Parse a path */ 105private Map <Parameter ,List <TaskResult >>parse (String f )throws IOException { 106final Map <Parameter ,List <TaskResult >>m =new TreeMap <Parameter ,List <TaskResult >>(); 107for (Parameter p :Parameter .values ()) 108m .put (p ,new ArrayList <TaskResult >()); 109parse (new File (f ),m ); 110 111//LOG.info("m=" + m.toString().replace(", ", ",\n ")); 112for (Parameter p :Parameter .values ()) 113m .put (p ,m .get (p )); 114return m ; 115 } 116 117/** Parse input and re-write results. */ 118Map <Parameter ,List <TaskResult >>parse (String inputpath ,String outputdir 119 )throws IOException { 120//parse input 121Util .out ."\nParsing " +inputpath +" ... " ); 122Util .out .flush (); 123final Map <Parameter ,List <TaskResult >>parsed =parse (inputpath ); 124Util .out .println ("DONE" ); 125 126//re-write the results 127if (outputdir !=null ) { 128Util .out ."\nWriting to " +outputdir +" ..." ); 129Util .out .flush (); 130for (Parameter p :Parameter .values ()) { 131final List <TaskResult >results =parsed .get (p ); 132Collections .sort (results ); 133 134final PrintWriter out =new PrintWriter ( 135new OutputStreamWriter (new FileOutputStream ( 136new File (outputdir ,p +".txt" )),Charsets .UTF_8 ),true ); 137try { 138for (int i =0 ;i <results .size ();i ++) 139out .println (DistSum .taskResult2string (p +"." +i ,results .get (i ))); 140 } 141finally { 142out .close (); 143 } 144 } 145Util .out .println ("DONE" ); 146 } 147return parsed ; 148 } 149 150/** Combine results */ 151static <T extends Combinable <T >>Map <Parameter ,T >combine (Map <Parameter ,List <T >>m ) { 152final Map <Parameter ,T >combined =new TreeMap <Parameter ,T >(); 153for (Parameter p :Parameter .values ()) { 154//note: results would never be null due to the design of Util.combine 155final List <T >results =Util .combine (m .get (p )); 156Util .out .format ("%-6s => " ,p ); 157if (results .size () !=1 ) 158Util .out .println (results .toString ().replace (", " ,",\n " )); 159else { 160final T r =results .get (0 ); 161combined .put (p ,r ); 162Util .out .println (r ); 163 } 164 } 165return combined ; 166 } 167 168/** main */ 169public static void main (String []args )throws IOException { 170if (args .length <2 ||args .length >3 ) 171Util .printUsage (args ,Parser .class .getName () 172 +" <b> <inputpath> [<outputdir>]" ); 173 174int i =0 ; 175final long b =Util .string2long (args [i ++]); 176final String inputpath =args [i ++]; 177final String outputdir =args .length >=3 ?args [i ++]:null ; 178 179//read input 180final Map <Parameter ,List <TaskResult >>parsed =new Parser (true ).parse (inputpath ,outputdir ); 181final Map <Parameter ,TaskResult >combined =combine (parsed ); 182long duration =0 ; 183for (TaskResult r :combined .values ()) 184duration +=r .getDuration (); 185 186//print pi 187final double pi =Bellard .computePi (b ,combined ); 188Util .printBitSkipped (b ); 189Util .out .println (Util .pi2string (pi ,Bellard .bit2terms (b ))); 190Util .out .println ("cpu time = " +Util .millis2String (duration )); 191 } 192}