+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18package org .apache .hadoop .examples .pi ; 19 20import java .io .BufferedReader ; 21import java .io .File ; 22import java .io .FileOutputStream ; 23import java .io .IOException ; 24import java .io .InputStreamReader ; 25import java .io .OutputStreamWriter ; 26import java .io .PrintStream ; 27import java .io .PrintWriter ; 28import java .text .SimpleDateFormat ; 29import java .util .ArrayList ; 30import java .util .Arrays ; 31import java .util .Collection ; 32import java .util .Collections ; 33import java .util .Date ; 34import java .util .List ; 35import java .util .concurrent .Callable ; 36import java .util .concurrent .ExecutionException ; 37import java .util .concurrent .ExecutorService ; 38import java .util .concurrent .Executors ; 39import java .util .concurrent .Future ; 40import java .util .concurrent .Semaphore ; 41 42import org .apache .hadoop .examples .pi .DistSum .Machine ; 43import org .apache .hadoop .fs .FileStatus ; 44import org .apache .hadoop .fs .FileSystem ; 45import org .apache .hadoop .fs .Path ; 46import org .apache .hadoop .fs .permission .FsPermission ; 47import org .apache .hadoop .mapreduce .Job ; 48import org .apache .hadoop .util .ToolRunner ; 49 50import com .common .base .Charsets ; 51 52/** Utility methods */ 53public class Util { 54/** Output stream */ 55public static final PrintStream out =System .out ; 56/** Error stream */ 57public static final PrintStream err =System .out ; 58 59/** Timer */ 60public static class Timer { 61private final boolean isAccumulative ; 62private final long start =System .currentTimeMillis (); 63private long previous =start ; 64 65/** Timer constructor 66* @param isAccumulative Is accumulating the time duration? 67*/ 68public Timer (boolean isAccumulative ) { 69this .isAccumulative =isAccumulative ; 70final StackTraceElement []stack =Thread .currentThread ().getStackTrace (); 71final StackTraceElement e =stack [stack .length -1 ]; 72out .println (e +" started at " +new Date (start )); 73 } 74 75/** Same as tick(null). */ 76public long tick () {return tick (null );} 77 78/** 79* Tick 80* @param s Output message. No output if it is null. 81* @return delta 82*/ 83public synchronized long tick (String s ) { 84final long t =System .currentTimeMillis (); 85final long delta =t - (isAccumulative ?start :previous ); 86if (s !=null ) { 87out .format ("%15dms (=%-15s: %s%n" ,delta ,millis2String (delta ) +")" ,s ); 88out .flush (); 89 } 90previous =t ; 91return delta ; 92 } 93 } 94 95/** Covert milliseconds to a String. */ 96public static String millis2String (long n ) { 97if (n <0 ) 98return "-" +millis2String (-n ); 99else if (n <1000 ) 100return n +"ms" ; 101 102final StringBuilder b =new StringBuilder (); 103final int millis = (int )(n %1000L ); 104if (millis !=0 ) 105b .append (String .format (".%03d" ,millis )); 106if ((n /=1000 ) <60 ) 107return b .insert (0 ,n ).append ("s" ).toString (); 108 109b .insert (0 ,String .format (":%02d" , (int )(n %60L ))); 110if ((n /=60 ) <60 ) 111return b .insert (0 ,n ).toString (); 112 113b .insert (0 ,String .format (":%02d" , (int )(n %60L ))); 114if ((n /=60 ) <24 ) 115return b .insert (0 ,n ).toString (); 116 117b .insert (0 ,n %24L ); 118final int days = (int )((n /=24 ) %365L ); 119b .insert (0 ,days ==1 ?" day " :" days " ).insert (0 ,days ); 120if ((n /=365L ) >0 ) 121b .insert (0 ,n ==1 ?" year " :" years " ).insert (0 ,n ); 122 123return b .toString (); 124 } 125 126/** Covert a String to a long. 127* This support comma separated number format. 128*/ 129public static long string2long (String s ) { 130return Long .parseLong (s .trim ().replace ("," ,"" )); 131 } 132 133/** Covert a long to a String in comma separated number format. */ 134public static String long2string (long n ) { 135if (n <0 ) 136return "-" +long2string (-n ); 137 138final StringBuilder b =new StringBuilder (); 139for (;n >=1000 ;n =n /1000 ) 140b .insert (0 ,String .format (",%03d" ,n %1000 )); 141return n +b .toString (); 142 } 143 144/** Parse a variable. */ 145public static long parseLongVariable (final String name ,final String s ) { 146return string2long (parseStringVariable (name ,s )); 147 } 148 149/** Parse a variable. */ 150public static String parseStringVariable (final String name ,final String s ) { 151if (!s .startsWith (name +'=' )) 152throw new IllegalArgumentException ("!s.startsWith(name + '='), name=" 153 +name +", s=" +s ); 154return s .substring (name .length () +1 ); 155 } 156 157/** Execute the callables by a number of threads */ 158public static <T ,E extends Callable <T >>void execute (int nThreads ,List <E >callables 159 )throws InterruptedException ,ExecutionException { 160final ExecutorService executor =Executors .newFixedThreadPool (nThreads ); 161final List <Future <T >>futures =executor .invokeAll (callables ); 162for (Future <T >f :futures ) 163f .get (); 164 } 165 166/** Print usage messages */ 167public static int printUsage (String []args ,String usage ) { 168err .println ("args = " +Arrays .asList (args )); 169err .println (); 170err .println ("Usage: java " +usage ); 171err .println (); 172ToolRunner .printGenericCommandUsage (err ); 173return -1 ; 174 } 175 176/** Combine a list of items. */ 177public static <T extends Combinable <T >>List <T >combine (Collection <T >items ) { 178final List <T >sorted =new ArrayList <T >(items ); 179if (sorted .size () <=1 ) 180return sorted ; 181 182Collections .sort (sorted ); 183final List <T >combined =new ArrayList <T >(items .size ()); 184T prev =sorted .get (0 ); 185for (int i =1 ;i <sorted .size ();i ++) { 186final T curr =sorted .get (i ); 187final T c =curr .combine (prev ); 188 189if (c !=null ) 190prev =c ; 191else { 192combined .add (prev ); 193prev =curr ; 194 } 195 } 196combined .add (prev ); 197return combined ; 198 } 199 200/** Check local directory. */ 201public static void checkDirectory (File dir ) { 202if (!dir .exists ()) 203if (!dir .mkdirs ()) 204throw new IllegalArgumentException ("!dir.mkdirs(), dir=" +dir ); 205if (!dir .isDirectory ()) 206throw new IllegalArgumentException ("dir (=" +dir +") is not a directory." ); 207 } 208 209/** Create a writer of a local file. */ 210public static PrintWriter createWriter (File dir ,String prefix )throws IOException { 211checkDirectory (dir ); 212 213SimpleDateFormat dateFormat =new SimpleDateFormat ("-yyyyMMdd-HHmmssSSS" ); 214for (;;) { 215final File f =new File (dir , 216prefix +dateFormat .format (new Date (System .currentTimeMillis ())) +".txt" ); 217if (!f .exists ()) 218return new PrintWriter (new OutputStreamWriter (new FileOutputStream (f ),Charsets .UTF_8 )); 219 220try {Thread .sleep (10 );}catch (InterruptedException e ) {} 221 } 222 } 223 224/** Print a "bits skipped" message. */ 225public static void printBitSkipped (final long b ) { 226out .println (); 227out .println ("b = " +long2string (b ) 228 +" (" + (b <2 ?"bit" :"bits" ) +" skipped)" ); 229 } 230 231/** Convert a pi value to a String. */ 232public static String pi2string (final double pi ,final long terms ) { 233final long value = (long )(pi * (1L <<DOUBLE_PRECISION )); 234final int acc_bit =accuracy (terms ,false ); 235final int acc_hex =acc_bit /4 ; 236final int shift =DOUBLE_PRECISION -acc_bit ; 237return String .format ("%0" +acc_hex +"X %0" + (13 -acc_hex ) +"X (%d hex digits)" , 238value >>shift ,value & ((1 <<shift ) -1 ),acc_hex ); 239 } 240 241static final int DOUBLE_PRECISION =52 ;//mantissa size 242static final int MACHEPS_EXPONENT =DOUBLE_PRECISION +1 ; 243/** Estimate accuracy. */ 244public static int accuracy (final long terms ,boolean final double error =terms <=0 ?2 : (Math .log (terms ) /Math .log (2 )) /2 ; 246final int bits =MACHEPS_EXPONENT - (int )Math .ceil (error ); 247if (out .println ("accuracy: bits=" +bits +", terms=" +long2string (terms ) +", error exponent=" +error ); 249return bits -bits %4 ; 250 } 251 252private static final String JOB_SEPARATION_PROPERTY ="pi.job.separation.seconds" ; 253private static final Semaphore JOB_SEMAPHORE =new Semaphore (1 ); 254 255/** Run a job. */ 256static void runJob (String name ,Job job ,Machine machine ,String startmessage ,Util .Timer timer ) { 257JOB_SEMAPHORE .acquireUninterruptibly (); 258Long starttime =null ; 259try { 260try { 261starttime =timer .tick ("starting " +name +" ...\n " +startmessage ); 262 263//initialize and submit a job 264machine .init (job ); 265job .submit (); 266 267// Separate jobs 268final long sleeptime =1000L *job .getConfiguration ().getInt (JOB_SEPARATION_PROPERTY ,10 ); 269if (sleeptime >0 ) { 270Util .out .println (name +"> sleep(" +Util .millis2String (sleeptime ) +")" ); 271Thread .sleep (sleeptime ); 272 } 273 }finally { 274JOB_SEMAPHORE .release (); 275 } 276 277if (!job .waitForCompletion (false )) 278throw new RuntimeException (name +" failed." ); 279 }catch (Exception e ) { 280throw e instanceof RuntimeException ? (RuntimeException )e :new RuntimeException (e ); 281 }finally { 282if (starttime !=null ) 283timer .tick (name +"> timetaken=" +Util .millis2String (timer .tick () -starttime )); 284 } 285 } 286 287/** Read job outputs */ 288static List <TaskResult >readJobOutputs (FileSystem fs ,Path outdir )throws IOException { 289final List <TaskResult >results =new ArrayList <TaskResult >(); 290for (FileStatus status :fs .listStatus (outdir )) { 291if (status .getPath ().getName ().startsWith ("part-" )) { 292final BufferedReader in =new BufferedReader ( 293new InputStreamReader (fs .open (status .getPath ()),Charsets .UTF_8 )); 294try { 295for (String line ; (line =in .readLine ()) !=null ; ) 296results .add (TaskResult .valueOf (line )); 297 } 298finally { 299in .close (); 300 } 301 } 302 } 303if (results .isEmpty ()) 304throw new IOException ("Output not found" ); 305return results ; 306 } 307 308/** Write results */ 309static void writeResults (String name ,List <TaskResult >results ,FileSystem fs ,String dir )throws IOException { 310final Path outfile =new Path (dir ,name +".txt" ); 311Util .out .println (name +"> writing results to " +outfile ); 312final PrintWriter out =new PrintWriter (new OutputStreamWriter (fs .create (outfile ),Charsets .UTF_8 ),true ); 313try { 314for (TaskResult r :results ) 315out .println (r ); 316 } 317finally { 318out .close (); 319 } 320 } 321 322/** Create a directory. */ 323static boolean createNonexistingDirectory (FileSystem fs ,Path dir )throws IOException { 324if (fs .exists (dir )) { 325Util .err .println ("dir (= " +dir +") already exists." ); 326return false ; 327 }else if (!fs .mkdirs (dir )) { 328throw new IOException ("Cannot create working directory " +dir ); 329 } 330fs .setPermission (dir ,new FsPermission ((short )0777 )); 331return true ; 332 } 333}