+/EE360P
distributed systems
git clone https://git.julia.blue/+/EE360P
07ca65d
master
1/** 2* Licensed to the Apache Software Foundation (ASF) under one 3* or more contributor license agreements. See the NOTICE file 4* distributed with this work for additional information 5* regarding copyright ownership. The ASF licenses this file 6* to you under the Apache License, Version 2.0 (the 7* "License"); you may not use this file except in compliance 8* with the License. You may obtain a copy of the License at 9* 10* http://www.apache.org/licenses/LICENSE-2.0 11* 12* Unless required by applicable law or agreed to in writing, software 13* distributed under the License is distributed on an "AS IS" BASIS, 14* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15* See the License for the specific language governing permissions and 16* limitations under the License. 17*/ 18 19package org .apache .hadoop .examples ; 20 21import java .io .DataInput ; 22import java .io .DataOutput ; 23import java .io .IOException ; 24import java .sql .Connection ; 25import java .sql .DriverManager ; 26import java .sql .PreparedStatement ; 27import java .sql .ResultSet ; 28import java .sql .SQLException ; 29import java .sql .Statement ; 30import java .util .Iterator ; 31import java .util .Random ; 32 33import org .apache .commons .logging .Log ; 34import org .apache .commons .logging .LogFactory ; 35import org .apache .hadoop .conf .Configuration ; 36import org .apache .hadoop .conf .Configured ; 37import org .apache .hadoop .io .LongWritable ; 38import org .apache .hadoop .io .NullWritable ; 39import org .apache .hadoop .io .Text ; 40import org .apache .hadoop .io .Writable ; 41import org .apache .hadoop .mapreduce .Job ; 42import org .apache .hadoop .mapreduce .Mapper ; 43import org .apache .hadoop .mapreduce .Reducer ; 44import org .apache .hadoop .mapreduce .lib .db .DBConfiguration ; 45import org .apache .hadoop .mapreduce .lib .db .DBInputFormat ; 46import org .apache .hadoop .mapreduce .lib .db .DBOutputFormat ; 47import org .apache .hadoop .mapreduce .lib .db .DBWritable ; 48import org .apache .hadoop .mapreduce .lib .reduce .LongSumReducer ; 49import org .apache .hadoop .util .StringUtils ; 50import org .apache .hadoop .util .Tool ; 51import org .apache .hadoop .util .ToolRunner ; 52import org .hsqldb .server .Server ; 53 54/** 55* This is a demonstrative program, which uses DBInputFormat for reading 56* the input data from a database, and DBOutputFormat for writing the data 57* to the database. 58* <br> 59* The Program first creates the necessary tables, populates the input table 60* and runs the mapred job. 61* <br> 62* The input data is a mini access log, with a <code><url,referrer,time> 63* </code> schema.The output is the number of pageviews of each url in the log, 64* having the schema <code><url,pageview></code>. 65* 66* When called with no arguments the program starts a local HSQLDB server, and 67* uses this database for storing/retrieving the data. 68* <br> 69* This program requires some additional configuration relating to HSQLDB. 70* The the hsqldb jar should be added to the classpath: 71* <br> 72* <code>export HADOOP_CLASSPATH=share/hadoop/mapreduce/lib-examples/hsqldb-2.0.0.jar</code> 73* <br> 74* And the hsqldb jar should be included with the <code>-libjars</code> 75* argument when executing it with hadoop: 76* <br> 77* <code>-libjars share/hadoop/mapreduce/lib-examples/hsqldb-2.0.0.jar</code> 78*/ 79public class DBCountPageView extends Configured implements Tool { 80 81private static final Log LOG =LogFactory .getLog (DBCountPageView .class ); 82 83private Connection connection ; 84private boolean initialized =false ; 85 86private static final String []AccessFieldNames = {"url" ,"referrer" ,"time" }; 87private static final String []PageviewFieldNames = {"url" ,"pageview" }; 88 89private static final String DB_URL = 90"jdbc:hsqldb:hsql://localhost/URLAccess" ; 91private static final String DRIVER_CLASS ="org.hsqldb.jdbc.JDBCDriver" ; 92 93private Server server ; 94 95private void startHsqldbServer () { 96server =new Server (); 97server .setDatabasePath (0 , 98System .getProperty ("test.build.data" ,"/tmp" ) +"/URLAccess" ); 99server .setDatabaseName (0 ,"URLAccess" ); 100server .start (); 101 } 102 103private void createConnection (String driverClassName 104 ,String url )throws Exception { 105 106Class .forName (driverClassName ); 107connection =DriverManager .getConnection (url ); 108connection .setAutoCommit (false ); 109 } 110 111private void shutdown () { 112try { 113connection .commit (); 114connection .close (); 115 }catch (Throwable ex ) { 116LOG .warn ("Exception occurred while closing connection :" 117 +StringUtils .stringifyException (ex )); 118 }finally { 119try { 120if (server !=null ) { 121server .shutdown (); 122 } 123 }catch (Throwable ex ) { 124LOG .warn ("Exception occurred while shutting down HSQLDB :" 125 +StringUtils .stringifyException (ex )); 126 } 127 } 128 } 129 130private void initialize (String driverClassName ,String url ) 131throws Exception { 132if (!this .initialized ) { 133if (driverClassName .equals (DRIVER_CLASS )) { 134startHsqldbServer (); 135 } 136createConnection (driverClassName ,url ); 137dropTables (); 138createTables (); 139populateAccess (); 140this .initialized =true ; 141 } 142 } 143 144private void dropTables () { 145String dropAccess ="DROP TABLE Access" ; 146String dropPageview ="DROP TABLE Pageview" ; 147Statement st =null ; 148try { 149st =connection .createStatement (); 150st .executeUpdate (dropAccess ); 151st .executeUpdate (dropPageview ); 152connection .commit (); 153st .close (); 154 }catch (SQLException ex ) { 155try {if (st !=null ) {st .close (); } }catch (Exception e ) {} 156 } 157 } 158 159private void createTables ()throws SQLException { 160 161String createAccess = 162"CREATE TABLE " + 163"Access(url VARCHAR(100) NOT NULL," + 164" referrer VARCHAR(100)," + 165" time BIGINT NOT NULL, " + 166" PRIMARY KEY (url, time))" ; 167 168String createPageview = 169"CREATE TABLE " + 170"Pageview(url VARCHAR(100) NOT NULL," + 171" pageview BIGINT NOT NULL, " + 172" PRIMARY KEY (url))" ; 173 174Statement st =connection .createStatement (); 175try { 176st .executeUpdate (createAccess ); 177st .executeUpdate (createPageview ); 178connection .commit (); 179 }finally { 180st .close (); 181 } 182 } 183 184/** 185* Populates the Access table with generated records. 186*/ 187private void populateAccess ()throws SQLException { 188 189PreparedStatement statement =null ; 190try { 191statement =connection .prepareStatement ( 192"INSERT INTO Access(url, referrer, time)" + 193" VALUES (?, ?, ?)" ); 194 195Random random =new Random (); 196 197int time =random .nextInt (50 ) +50 ; 198 199final int PROBABILITY_PRECISION =100 ;// 1 / 100 200final int NEW_PAGE_PROBABILITY =15 ;// 15 / 100 201 202 203//Pages in the site : 204String []pages = {"/a" ,"/b" ,"/c" ,"/d" ,"/e" , 205"/f" ,"/g" ,"/h" ,"/i" ,"/j" }; 206//linkMatrix[i] is the array of pages(indexes) that page_i links to. 207int [][]linkMatrix = {{1 ,5 ,7 }, {0 ,7 ,4 ,6 ,}, {0 ,1 ,7 ,8 }, 208 {0 ,2 ,4 ,6 ,7 ,9 }, {0 ,1 }, {0 ,3 ,5 ,9 }, {0 }, {0 ,1 ,3 }, {0 ,2 ,6 }, {0 ,2 ,6 }}; 209 210//a mini model of user browsing a la pagerank 211int currentPage =random .nextInt (pages .length ); 212String referrer =null ; 213 214for (int i =0 ;i <time ;i ++) { 215 216statement .setString (1 ,pages [currentPage ]); 217statement .setString (2 ,referrer ); 218statement .setLong (3 ,i ); 219statement .execute (); 220 221int action =random .nextInt (PROBABILITY_PRECISION ); 222 223// go to a new page with probability 224// NEW_PAGE_PROBABILITY / PROBABILITY_PRECISION 225if (action <NEW_PAGE_PROBABILITY ) { 226currentPage =random .nextInt (pages .length );// a random page 227referrer =null ; 228 } 229else { 230referrer =pages [currentPage ]; 231action =random .nextInt (linkMatrix [currentPage ].length ); 232currentPage =linkMatrix [currentPage ][action ]; 233 } 234 } 235 236connection .commit (); 237 238 }catch (SQLException ex ) { 239connection .rollback (); 240throw ex ; 241 }finally { 242if (statement !=null ) { 243statement .close (); 244 } 245 } 246 } 247 248/**Verifies the results are correct */ 249private boolean verify ()throws SQLException { 250//check total num pageview 251String countAccessQuery ="SELECT COUNT(*) FROM Access" ; 252String sumPageviewQuery ="SELECT SUM(pageview) FROM Pageview" ; 253Statement st =null ; 254ResultSet rs =null ; 255try { 256st =connection .createStatement (); 257rs =st .executeQuery (countAccessQuery ); 258rs .next (); 259long totalPageview =rs .getLong (1 ); 260 261rs =st .executeQuery (sumPageviewQuery ); 262rs .next (); 263long sumPageview =rs .getLong (1 ); 264 265LOG .info ("totalPageview=" +totalPageview ); 266LOG .info ("sumPageview=" +sumPageview ); 267 268return totalPageview ==sumPageview &&totalPageview !=0 ; 269 }finally { 270if (st !=null ) 271st .close (); 272if (rs !=null ) 273rs .close (); 274 } 275 } 276 277/** Holds a <url, referrer, time > tuple */ 278static class AccessRecord implements Writable ,DBWritable { 279String url ; 280String referrer ; 281long time ; 282 283@ Override 284public void readFields (DataInput in )throws IOException { 285this .url =Text .readString (in ); 286this .referrer =Text .readString (in ); 287this .time =in .readLong (); 288 } 289 290@ Override 291public void write (DataOutput out )throws IOException { 292Text .writeString (out ,url ); 293Text .writeString (out ,referrer ); 294out .writeLong (time ); 295 } 296 297@ Override 298public void readFields (ResultSet resultSet )throws SQLException { 299this .url =resultSet .getString (1 ); 300this .referrer =resultSet .getString (2 ); 301this .time =resultSet .getLong (3 ); 302 } 303@ Override 304public void write (PreparedStatement statement )throws SQLException { 305statement .setString (1 ,url ); 306statement .setString (2 ,referrer ); 307statement .setLong (3 ,time ); 308 } 309 } 310/** Holds a <url, pageview > tuple */ 311static class PageviewRecord implements Writable ,DBWritable { 312String url ; 313long pageview ; 314 315public PageviewRecord (String url ,long pageview ) { 316this .url =url ; 317this .pageview =pageview ; 318 } 319 320@ Override 321public void readFields (DataInput in )throws IOException { 322this .url =Text .readString (in ); 323this .pageview =in .readLong (); 324 } 325@ Override 326public void write (DataOutput out )throws IOException { 327Text .writeString (out ,url ); 328out .writeLong (pageview ); 329 } 330@ Override 331public void readFields (ResultSet resultSet )throws SQLException { 332this .url =resultSet .getString (1 ); 333this .pageview =resultSet .getLong (2 ); 334 } 335@ Override 336public void write (PreparedStatement statement )throws SQLException { 337statement .setString (1 ,url ); 338statement .setLong (2 ,pageview ); 339 } 340@ Override 341public String toString () { 342return url +" " +pageview ; 343 } 344 } 345 346/** 347* Mapper extracts URLs from the AccessRecord (tuples from db), 348* and emits a <url,1> pair for each access record. 349*/ 350static class PageviewMapper extends 351Mapper <LongWritable ,AccessRecord ,Text ,LongWritable > { 352 353LongWritable ONE =new LongWritable (1L ); 354@ Override 355public void map (LongWritable key ,AccessRecord value ,Context context ) 356throws IOException ,InterruptedException { 357Text oKey =new Text (value .url ); 358context .write (oKey ,ONE ); 359 } 360 } 361 362/** 363* Reducer sums up the pageviews and emits a PageviewRecord, 364* which will correspond to one tuple in the db. 365*/ 366static class PageviewReducer extends 367Reducer <Text ,LongWritable ,PageviewRecord ,NullWritable > { 368 369NullWritable n =NullWritable .get (); 370@ Override 371public void reduce (Text key ,Iterable <LongWritable >values , 372Context context )throws IOException ,InterruptedException { 373 374long sum =0L ; 375for (LongWritable value :values ) { 376sum +=value .get (); 377 } 378context .write (new PageviewRecord (key .toString (),sum ),n ); 379 } 380 } 381 382@ Override 383//Usage DBCountPageView [driverClass dburl] 384public int run (String []args )throws Exception { 385 386String driverClassName =DRIVER_CLASS ; 387String url =DB_URL ; 388 389if (args .length >1 ) { 390driverClassName =args [0 ]; 391url =args [1 ]; 392 } 393 394initialize (driverClassName ,url ); 395Configuration conf =getConf (); 396 397DBConfiguration .configureDB (conf ,driverClassName ,url ); 398 399Job job =new Job (conf ); 400 401job .setJobName ("Count Pageviews of URLs" ); 402job .setJarByClass (DBCountPageView .class ); 403job .setMapperClass (PageviewMapper .class ); 404job .setCombinerClass (LongSumReducer .class ); 405job .setReducerClass (PageviewReducer .class ); 406 407DBInputFormat .setInput (job ,AccessRecord .class ,"Access" 408 ,null ,"url" ,AccessFieldNames ); 409 410DBOutputFormat .setOutput (job ,"Pageview" ,PageviewFieldNames ); 411 412job .setMapOutputKeyClass (Text .class ); 413job .setMapOutputValueClass (LongWritable .class ); 414 415job .setOutputKeyClass (PageviewRecord .class ); 416job .setOutputValueClass (NullWritable .class ); 417int ret ; 418try { 419ret =job .waitForCompletion (true ) ?0 :1 ; 420boolean correct =verify (); 421if (!correct ) { 422throw new RuntimeException ("Evaluation was not correct!" ); 423 } 424 }finally { 425shutdown (); 426 } 427return ret ; 428 } 429 430public static void main (String []args )throws Exception { 431int ret =ToolRunner .run (new DBCountPageView (),args ); 432System .exit (ret ); 433 } 434 435}