+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18package org .apache .hadoop .examples .terasort ; 19 20import java .io .FileOutputStream ; 21import java .io .IOException ; 22import java .io .OutputStream ; 23import java .io .PrintStream ; 24import java .math .BigInteger ; 25import java .util .zip .Checksum ; 26 27import org .apache .hadoop .util .PureJavaCrc32 ; 28 29/** 30* A single process data generator for the terasort data. Based on gensort.c 31* version 1.1 (3 Mar 2009) from Chris Nyberg <chris.nyberg@ordinal.com>. 32*/ 33public class GenSort { 34 35/** 36* Generate a "binary" record suitable for all sort benchmarks *except* 37* PennySort. 38*/ 39static void generateRecord (byte []recBuf ,Unsigned16 rand , 40Unsigned16 recordNumber ) { 41/* generate the 10-byte key using the high 10 bytes of the 128-bit 42* random number 43*/ 44for (int i =0 ;i <10 ; ++i ) { 45recBuf [i ] =rand .getByte (i ); 46 } 47 48/* add 2 bytes of "break" */ 49recBuf [10 ] =0x00 ; 50recBuf [11 ] =0x11 ; 51 52/* convert the 128-bit record number to 32 bits of ascii hexadecimal 53* as the next 32 bytes of the record. 54*/ 55for (int i =0 ;i <32 ;i ++) { 56recBuf [12 +i ] = (byte )recordNumber .getHexDigit (i ); 57 } 58 59/* add 4 bytes of "break" data */ 60recBuf [44 ] = (byte )0x88 ; 61recBuf [45 ] = (byte )0x99 ; 62recBuf [46 ] = (byte )0xAA ; 63recBuf [47 ] = (byte )0xBB ; 64 65/* add 48 bytes of filler based on low 48 bits of random number */ 66for (int i =0 ;i <12 ; ++i ) { 67recBuf [48 +i *4 ] =recBuf [49 +i *4 ] =recBuf [50 +i *4 ] =recBuf [51 +i *4 ] = 68 (byte )rand .getHexDigit (20 +i ); 69 } 70 71/* add 4 bytes of "break" data */ 72recBuf [96 ] = (byte )0xCC ; 73recBuf [97 ] = (byte )0xDD ; 74recBuf [98 ] = (byte )0xEE ; 75recBuf [99 ] = (byte )0xFF ; 76 } 77 78 79private static BigInteger makeBigInteger (long x ) { 80byte []data =new byte [8 ]; 81for (int i =0 ;i <8 ; ++i ) { 82data [i ] = (byte ) (x >>> (56 -8 *i )); 83 } 84return new BigInteger (1 ,data ); 85 } 86 87private static final BigInteger NINETY_FIVE =new BigInteger ("95" ); 88 89/** 90* Generate an ascii record suitable for all sort benchmarks including 91* PennySort. 92*/ 93static void generateAsciiRecord (byte []recBuf ,Unsigned16 rand , 94Unsigned16 recordNumber ) { 95 96/* generate the 10-byte ascii key using mostly the high 64 bits. 97*/ 98long temp =rand .getHigh8 (); 99if (temp <0 ) { 100// use biginteger to avoid the negative sign problem 101BigInteger bigTemp =makeBigInteger (temp ); 102recBuf [0 ] = (byte ) (' ' + (bigTemp .mod (NINETY_FIVE ).longValue ())); 103temp =bigTemp .divide (NINETY_FIVE ).longValue (); 104 }else { 105recBuf [0 ] = (byte ) (' ' + (temp %95 )); 106temp /=95 ; 107 } 108for (int i =1 ;i <8 ; ++i ) { 109recBuf [i ] = (byte ) (' ' + (temp %95 )); 110temp /=95 ; 111 } 112temp =rand .getLow8 (); 113if (temp <0 ) { 114BigInteger bigTemp =makeBigInteger (temp ); 115recBuf [8 ] = (byte ) (' ' + (bigTemp .mod (NINETY_FIVE ).longValue ())); 116temp =bigTemp .divide (NINETY_FIVE ).longValue (); 117 }else { 118recBuf [8 ] = (byte ) (' ' + (temp %95 )); 119temp /=95 ; 120 } 121recBuf [9 ] = (byte )(' ' + (temp %95 )); 122 123/* add 2 bytes of "break" */ 124recBuf [10 ] =' ' ; 125recBuf [11 ] =' ' ; 126 127/* convert the 128-bit record number to 32 bits of ascii hexadecimal 128* as the next 32 bytes of the record. 129*/ 130for (int i =0 ;i <32 ;i ++) { 131recBuf [12 +i ] = (byte )recordNumber .getHexDigit (i ); 132 } 133 134/* add 2 bytes of "break" data */ 135recBuf [44 ] =' ' ; 136recBuf [45 ] =' ' ; 137 138/* add 52 bytes of filler based on low 48 bits of random number */ 139for (int i =0 ;i <13 ; ++i ) { 140recBuf [46 +i *4 ] =recBuf [47 +i *4 ] =recBuf [48 +i *4 ] =recBuf [49 +i *4 ] = 141 (byte )rand .getHexDigit (19 +i ); 142 } 143 144/* add 2 bytes of "break" data */ 145recBuf [98 ] ='\r' ;/* nice for Windows */ 146recBuf [99 ] ='\n' ; 147} 148 149 150private static void usage () { 151PrintStream out =System .out ; 152out .println ("usage: gensort [-a] [-c] [-bSTARTING_REC_NUM] NUM_RECS FILE_NAME" ); 153out .println ("-a Generate ascii records required for PennySort or JouleSort." ); 154out .println (" These records are also an alternative input for the other" ); 155out .println (" sort benchmarks. Without this flag, binary records will be" ); 156out .println (" generated that contain the highest density of randomness in" ); 157out .println (" the 10-byte key." ); 158out .println ("-c Calculate the sum of the crc32 checksums of each of the" ); 159out .println (" generated records and send it to standard error." ); 160out .println ("-bN Set the beginning record generated to N. By default the" ); 161out .println (" first record generated is record 0." ); 162out .println ("NUM_RECS The number of sequential records to generate." ); 163out .println ("FILE_NAME The name of the file to write the records to.\n" ); 164out .println ("Example 1 - to generate 1000000 ascii records starting at record 0 to" ); 165out .println ("the file named \"pennyinput\":" ); 166out .println (" gensort -a 1000000 pennyinput\n" ); 167out .println ("Example 2 - to generate 1000 binary records beginning with record 2000" ); 168out .println ("to the file named \"partition2\":" ); 169out .println (" gensort -b2000 1000 partition2" ); 170System .exit (1 ); 171 } 172 173 174public static void outputRecords (OutputStream out , 175boolean useAscii , 176Unsigned16 firstRecordNumber , 177Unsigned16 recordsToGenerate , 178Unsigned16 checksum 179 )throws IOException { 180byte []row =new byte [100 ]; 181Unsigned16 recordNumber =new Unsigned16 (firstRecordNumber ); 182Unsigned16 lastRecordNumber =new Unsigned16 (firstRecordNumber ); 183Checksum crc =new PureJavaCrc32 (); 184Unsigned16 tmp =new Unsigned16 (); 185lastRecordNumber .add (recordsToGenerate ); 186Unsigned16 ONE =new Unsigned16 (1 ); 187Unsigned16 rand =Random16 .skipAhead (firstRecordNumber ); 188while (!recordNumber .equals (lastRecordNumber )) { 189Random16 .nextRand (rand ); 190if (useAscii ) { 191generateAsciiRecord (row ,rand ,recordNumber ); 192 }else { 193generateRecord (row ,rand ,recordNumber ); 194 } 195if (checksum !=null ) { 196crc .reset (); 197crc .update (row ,0 ,row .length ); 198tmp .set (crc .getValue ()); 199checksum .add (tmp ); 200 } 201recordNumber .add (ONE ); 202out .write (row ); 203 } 204 } 205 206public static void main (String []args )throws Exception { 207Unsigned16 startingRecord =new Unsigned16 (); 208Unsigned16 numberOfRecords ; 209OutputStream out ; 210boolean useAscii =false ; 211Unsigned16 checksum =null ; 212 213int i ; 214for (i =0 ;i <args .length ; ++i ) { 215String arg =args [i ]; 216int argLength =arg .length (); 217if (argLength >=1 &&arg .charAt (0 ) =='-' ) { 218if (argLength <2 ) { 219usage (); 220 } 221switch (arg .charAt (1 )) { 222case 'a' : 223useAscii =true ; 224break ; 225case 'b' : 226startingRecord =Unsigned16 .fromDecimal (arg .substring (2 )); 227break ; 228case 'c' : 229checksum =new Unsigned16 (); 230break ; 231default : 232usage (); 233 } 234 }else { 235break ; 236 } 237 } 238if (args .length -i !=2 ) { 239usage (); 240 } 241numberOfRecords =Unsigned16 .fromDecimal (args [i ]); 242out =new FileOutputStream (args [i +1 ]); 243 244outputRecords (out ,useAscii ,startingRecord ,numberOfRecords ,checksum ); 245out .close (); 246if (checksum !=null ) { 247System .out .println (checksum ); 248 } 249 } 250 251}