+/EE360P

distributed systems

git clone https://git.julia.blue/+/EE360P

vijaygarg1init07ca65d

master
13.2 KiB435 linesraw
1/**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements.  See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership.  The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License.  You may obtain a copy of the License at
9 *
10 *     http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19package org.apache.hadoop.examples;
20
21import java.io.DataInput;
22import java.io.DataOutput;
23import java.io.IOException;
24import java.sql.Connection;
25import java.sql.DriverManager;
26import java.sql.PreparedStatement;
27import java.sql.ResultSet;
28import java.sql.SQLException;
29import java.sql.Statement;
30import java.util.Iterator;
31import java.util.Random;
32
33import org.apache.commons.logging.Log;
34import org.apache.commons.logging.LogFactory;
35import org.apache.hadoop.conf.Configuration;
36import org.apache.hadoop.conf.Configured;
37import org.apache.hadoop.io.LongWritable;
38import org.apache.hadoop.io.NullWritable;
39import org.apache.hadoop.io.Text;
40import org.apache.hadoop.io.Writable;
41import org.apache.hadoop.mapreduce.Job;
42import org.apache.hadoop.mapreduce.Mapper;
43import org.apache.hadoop.mapreduce.Reducer;
44import org.apache.hadoop.mapreduce.lib.db.DBConfiguration;
45import org.apache.hadoop.mapreduce.lib.db.DBInputFormat;
46import org.apache.hadoop.mapreduce.lib.db.DBOutputFormat;
47import org.apache.hadoop.mapreduce.lib.db.DBWritable;
48import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer;
49import org.apache.hadoop.util.StringUtils;
50import org.apache.hadoop.util.Tool;
51import org.apache.hadoop.util.ToolRunner;
52import org.hsqldb.server.Server;
53
54/**
55 * This is a demonstrative program, which uses DBInputFormat for reading
56 * the input data from a database, and DBOutputFormat for writing the data 
57 * to the database. 
58 * <br>
59 * The Program first creates the necessary tables, populates the input table 
60 * and runs the mapred job. 
61 * <br> 
62 * The input data is a mini access log, with a <code>&lt;url,referrer,time&gt;
63 * </code> schema.The output is the number of pageviews of each url in the log, 
64 * having the schema <code>&lt;url,pageview&gt;</code>.  
65 * 
66 * When called with no arguments the program starts a local HSQLDB server, and 
67 * uses this database for storing/retrieving the data. 
68 * <br>
69 * This program requires some additional configuration relating to HSQLDB.  
70 * The the hsqldb jar should be added to the classpath:
71 * <br>
72 * <code>export HADOOP_CLASSPATH=share/hadoop/mapreduce/lib-examples/hsqldb-2.0.0.jar</code>
73 * <br>
74 * And the hsqldb jar should be included with the <code>-libjars</code> 
75 * argument when executing it with hadoop:
76 * <br>
77 * <code>-libjars share/hadoop/mapreduce/lib-examples/hsqldb-2.0.0.jar</code>
78 */
79public class DBCountPageView extends Configured implements Tool {
80
81  private static final Log LOG = LogFactory.getLog(DBCountPageView.class);
82  
83  private Connection connection;
84  private boolean initialized = false;
85
86  private static final String[] AccessFieldNames = {"url", "referrer", "time"};
87  private static final String[] PageviewFieldNames = {"url", "pageview"};
88  
89  private static final String DB_URL = 
90    "jdbc:hsqldb:hsql://localhost/URLAccess";
91  private static final String DRIVER_CLASS = "org.hsqldb.jdbc.JDBCDriver";
92  
93  private Server server;
94  
95  private void startHsqldbServer() {
96    server = new Server();
97    server.setDatabasePath(0, 
98        System.getProperty("test.build.data", "/tmp") + "/URLAccess");
99    server.setDatabaseName(0, "URLAccess");
100    server.start();
101  }
102  
103  private void createConnection(String driverClassName
104      , String url) throws Exception {
105
106    Class.forName(driverClassName);
107    connection = DriverManager.getConnection(url);
108    connection.setAutoCommit(false);
109  }
110
111  private void shutdown() {
112    try {
113      connection.commit();
114      connection.close();
115    }catch (Throwable ex) {
116      LOG.warn("Exception occurred while closing connection :"
117          + StringUtils.stringifyException(ex));
118    } finally {
119      try {
120        if(server != null) {
121          server.shutdown();
122        }
123      }catch (Throwable ex) {
124        LOG.warn("Exception occurred while shutting down HSQLDB :"
125            + StringUtils.stringifyException(ex));
126      }
127    }
128  }
129
130  private void initialize(String driverClassName, String url) 
131    throws Exception {
132    if(!this.initialized) {
133      if(driverClassName.equals(DRIVER_CLASS)) {
134        startHsqldbServer();
135      }
136      createConnection(driverClassName, url);
137      dropTables();
138      createTables();
139      populateAccess();
140      this.initialized = true;  
141    }
142  }
143  
144  private void dropTables() {
145    String dropAccess = "DROP TABLE Access";
146    String dropPageview = "DROP TABLE Pageview";
147    Statement st = null;
148    try {
149      st = connection.createStatement();
150      st.executeUpdate(dropAccess);
151      st.executeUpdate(dropPageview);
152      connection.commit();
153      st.close();
154    }catch (SQLException ex) {
155      try { if (st != null) { st.close(); } } catch (Exception e) {}
156    }
157  }
158  
159  private void createTables() throws SQLException {
160
161    String createAccess = 
162      "CREATE TABLE " +
163      "Access(url      VARCHAR(100) NOT NULL," +
164            " referrer VARCHAR(100)," +
165            " time     BIGINT NOT NULL, " +
166            " PRIMARY KEY (url, time))";
167
168    String createPageview = 
169      "CREATE TABLE " +
170      "Pageview(url      VARCHAR(100) NOT NULL," +
171              " pageview     BIGINT NOT NULL, " +
172               " PRIMARY KEY (url))";
173    
174    Statement st = connection.createStatement();
175    try {
176      st.executeUpdate(createAccess);
177      st.executeUpdate(createPageview);
178      connection.commit();
179    } finally {
180      st.close();
181    }
182  }
183
184  /**
185   * Populates the Access table with generated records.
186   */
187  private void populateAccess() throws SQLException {
188
189    PreparedStatement statement = null ;
190    try {
191      statement = connection.prepareStatement(
192          "INSERT INTO Access(url, referrer, time)" +
193          " VALUES (?, ?, ?)");
194
195      Random random = new Random();
196
197      int time = random.nextInt(50) + 50;
198
199      final int PROBABILITY_PRECISION = 100; //  1 / 100 
200      final int NEW_PAGE_PROBABILITY  = 15;  //  15 / 100
201
202
203      //Pages in the site :
204      String[] pages = {"/a", "/b", "/c", "/d", "/e", 
205                        "/f", "/g", "/h", "/i", "/j"};
206      //linkMatrix[i] is the array of pages(indexes) that page_i links to.  
207      int[][] linkMatrix = {{1,5,7}, {0,7,4,6,}, {0,1,7,8}, 
208        {0,2,4,6,7,9}, {0,1}, {0,3,5,9}, {0}, {0,1,3}, {0,2,6}, {0,2,6}};
209
210      //a mini model of user browsing a la pagerank
211      int currentPage = random.nextInt(pages.length); 
212      String referrer = null;
213
214      for(int i=0; i<time; i++) {
215
216        statement.setString(1, pages[currentPage]);
217        statement.setString(2, referrer);
218        statement.setLong(3, i);
219        statement.execute();
220
221        int action = random.nextInt(PROBABILITY_PRECISION);
222
223        // go to a new page with probability 
224        // NEW_PAGE_PROBABILITY / PROBABILITY_PRECISION
225        if(action < NEW_PAGE_PROBABILITY) { 
226          currentPage = random.nextInt(pages.length); // a random page
227          referrer = null;
228        }
229        else {
230          referrer = pages[currentPage];
231          action = random.nextInt(linkMatrix[currentPage].length);
232          currentPage = linkMatrix[currentPage][action];
233        }
234      }
235      
236      connection.commit();
237      
238    }catch (SQLException ex) {
239      connection.rollback();
240      throw ex;
241    } finally {
242      if(statement != null) {
243        statement.close();
244      }
245    }
246  }
247  
248  /**Verifies the results are correct */
249  private boolean verify() throws SQLException {
250    //check total num pageview
251    String countAccessQuery = "SELECT COUNT(*) FROM Access";
252    String sumPageviewQuery = "SELECT SUM(pageview) FROM Pageview";
253    Statement st = null;
254    ResultSet rs = null;
255    try {
256      st = connection.createStatement();
257      rs = st.executeQuery(countAccessQuery);
258      rs.next();
259      long totalPageview = rs.getLong(1);
260
261      rs = st.executeQuery(sumPageviewQuery);
262      rs.next();
263      long sumPageview = rs.getLong(1);
264
265      LOG.info("totalPageview=" + totalPageview);
266      LOG.info("sumPageview=" + sumPageview);
267
268      return totalPageview == sumPageview && totalPageview != 0;
269    }finally {
270      if(st != null)
271        st.close();
272      if(rs != null)
273        rs.close();
274    }
275  }
276  
277  /** Holds a &lt;url, referrer, time &gt; tuple */
278  static class AccessRecord implements Writable, DBWritable {
279    String url;
280    String referrer;
281    long time;
282    
283    @Override
284    public void readFields(DataInput in) throws IOException {
285      this.url = Text.readString(in);
286      this.referrer = Text.readString(in);
287      this.time = in.readLong();
288    }
289    
290    @Override
291    public void write(DataOutput out) throws IOException {
292      Text.writeString(out, url);
293      Text.writeString(out, referrer);
294      out.writeLong(time);
295    }
296    
297    @Override
298    public void readFields(ResultSet resultSet) throws SQLException {
299      this.url = resultSet.getString(1);
300      this.referrer = resultSet.getString(2);
301      this.time = resultSet.getLong(3);
302    }
303    @Override
304    public void write(PreparedStatement statement) throws SQLException {
305      statement.setString(1, url);
306      statement.setString(2, referrer);
307      statement.setLong(3, time);
308    }
309  }
310  /** Holds a &lt;url, pageview &gt; tuple */
311  static class PageviewRecord implements Writable, DBWritable {
312    String url;
313    long pageview;
314   
315    public PageviewRecord(String url, long pageview) {
316      this.url = url;
317      this.pageview = pageview;
318    }
319    
320    @Override
321    public void readFields(DataInput in) throws IOException {
322      this.url = Text.readString(in);
323      this.pageview = in.readLong();
324    }
325    @Override
326    public void write(DataOutput out) throws IOException {
327      Text.writeString(out, url);
328      out.writeLong(pageview);
329    }
330    @Override
331    public void readFields(ResultSet resultSet) throws SQLException {
332      this.url = resultSet.getString(1);
333      this.pageview = resultSet.getLong(2);
334    }
335    @Override
336    public void write(PreparedStatement statement) throws SQLException {
337      statement.setString(1, url);
338      statement.setLong(2, pageview);
339    }
340    @Override
341    public String toString() {
342      return url + " " + pageview;
343    }
344  }
345  
346  /**
347   * Mapper extracts URLs from the AccessRecord (tuples from db), 
348   * and emits a &lt;url,1&gt; pair for each access record. 
349   */
350  static class PageviewMapper extends 
351      Mapper<LongWritable, AccessRecord, Text, LongWritable> {
352    
353    LongWritable ONE = new LongWritable(1L);
354    @Override
355    public void map(LongWritable key, AccessRecord value, Context context)
356        throws IOException, InterruptedException {
357      Text oKey = new Text(value.url);
358      context.write(oKey, ONE);
359    }
360  }
361  
362  /**
363   * Reducer sums up the pageviews and emits a PageviewRecord, 
364   * which will correspond to one tuple in the db.
365   */
366  static class PageviewReducer extends 
367      Reducer<Text, LongWritable, PageviewRecord, NullWritable> {
368    
369    NullWritable n = NullWritable.get();
370    @Override
371    public void reduce(Text key, Iterable<LongWritable> values, 
372        Context context) throws IOException, InterruptedException {
373      
374      long sum = 0L;
375      for(LongWritable value: values) {
376        sum += value.get();
377      }
378      context.write(new PageviewRecord(key.toString(), sum), n);
379    }
380  }
381  
382  @Override
383  //Usage DBCountPageView [driverClass dburl]
384  public int run(String[] args) throws Exception {
385    
386    String driverClassName = DRIVER_CLASS;
387    String url = DB_URL;
388    
389    if(args.length > 1) {
390      driverClassName = args[0];
391      url = args[1];
392    }
393    
394    initialize(driverClassName, url);
395    Configuration conf = getConf();
396
397    DBConfiguration.configureDB(conf, driverClassName, url);
398
399    Job job = new Job(conf);
400        
401    job.setJobName("Count Pageviews of URLs");
402    job.setJarByClass(DBCountPageView.class);
403    job.setMapperClass(PageviewMapper.class);
404    job.setCombinerClass(LongSumReducer.class);
405    job.setReducerClass(PageviewReducer.class);
406
407    DBInputFormat.setInput(job, AccessRecord.class, "Access"
408        , null, "url", AccessFieldNames);
409
410    DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);
411    
412    job.setMapOutputKeyClass(Text.class);
413    job.setMapOutputValueClass(LongWritable.class);
414
415    job.setOutputKeyClass(PageviewRecord.class);
416    job.setOutputValueClass(NullWritable.class);
417    int ret;
418    try {
419      ret = job.waitForCompletion(true) ? 0 : 1;
420      boolean correct = verify();
421      if(!correct) {
422        throw new RuntimeException("Evaluation was not correct!");
423      }
424    } finally {
425      shutdown();    
426    }
427    return ret;
428  }
429
430  public static void main(String[] args) throws Exception {
431    int ret = ToolRunner.run(new DBCountPageView(), args);
432    System.exit(ret);
433  }
434
435}