Coverage Report - net.sf.jmatchparser.util.PrefetchDownloader
 
Classes in this File Line Coverage Branch Coverage Complexity
PrefetchDownloader
0%
0/47
0%
0/18
2,556
PrefetchDownloader$1
0%
0/3
N/A
2,556
PrefetchDownloader$PrefetchEntry
0%
0/6
N/A
2,556
 
 1  
 /*
 2  
  * Copyright (c) 2006 - 2011 Michael Schierl
 3  
  * 
 4  
  * All rights reserved.
 5  
  * 
 6  
  * Redistribution and use in source and binary forms, with or without
 7  
  * modification, are permitted provided that the following conditions
 8  
  * are met:
 9  
  * 
 10  
  * - Redistributions of source code must retain the above copyright notice,
 11  
  *   this list of conditions and the following disclaimer.
 12  
  *   
 13  
  * - Redistributions in binary form must reproduce the above copyright
 14  
  *   notice, this list of conditions and the following disclaimer in the
 15  
  *   documentation and/or other materials provided with the distribution.
 16  
  *   
 17  
  * - Neither name of the copyright holders nor the names of its
 18  
  *   contributors may be used to endorse or promote products derived from
 19  
  *   this software without specific prior written permission.
 20  
  *   
 21  
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND THE CONTRIBUTORS
 22  
  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 23  
  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 24  
  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 25  
  * HOLDERS OR THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 26  
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 27  
  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 28  
  * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 29  
  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
 30  
  * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 31  
  * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 32  
  */
 33  
 package net.sf.jmatchparser.util;
 34  
 
 35  
 import java.io.InputStream;
 36  
 import java.util.HashMap;
 37  
 import java.util.LinkedList;
 38  
 import java.util.Map;
 39  
 import java.util.Queue;
 40  
 
 41  
 /**
 42  
  * Utility class that makes it easy to download files in the background as they
 43  
  * are detected (over slow links) while still doing all of the parsing in a
 44  
  * single thread.
 45  
  * 
 46  
  * The main idea is, that whenever you encounter a file you have to parse later,
 47  
  * you add it to the prefetcher by calling the {@link #download(String, String)}
 48  
  * method. All documents added to the prefetcher are loaded in a configurable
 49  
  * number of background threads (and stored into the {@link CachingDownloader}'s
 50  
  * cache. When ready to parse the new document, you can request the result by
 51  
  * calling {@link #waitFor(String)}, waiting for it if needed.
 52  
  */
 53  0
 public class PrefetchDownloader {
 54  
 
 55  
         private final CachingDownloader cd;
 56  0
         private final Queue<PrefetchEntry> q = new LinkedList<PrefetchEntry>();
 57  0
         private int busy = 0;
 58  0
         private boolean finished = false;
 59  
         private final Map<String, InputStream> results;
 60  
 
 61  
         /**
 62  
          * Create a new prefetch downloader.
 63  
          * 
 64  
          * @param cd
 65  
          *            Caching downloader to use
 66  
          * @param threadcount
 67  
          *            Number of prefetch threads to start
 68  
          * @param cacheResults
 69  
          *            Whether to cache the results in the prefetcher. If not, they
 70  
          *            are only written to the cache and can be retrieved from there
 71  
          */
 72  0
         public PrefetchDownloader(CachingDownloader cd, int threadcount, boolean cacheResults) {
 73  0
                 this.cd = cd;
 74  0
                 this.results = cacheResults ? new HashMap<String, InputStream>() : null;
 75  0
                 for (int i = 0; i < threadcount; i++) {
 76  0
                         new Thread(new Runnable() {
 77  
                                 @Override
 78  
                                 public void run() {
 79  0
                                         runPrefetch();
 80  0
                                 }
 81  
                         }).start();
 82  
                 }
 83  0
         }
 84  
 
 85  
         /**
 86  
          * Add a document to download
 87  
          * 
 88  
          * @param url
 89  
          *            URL of the document
 90  
          * @param cacheName
 91  
          *            Name of the document in the cache
 92  
          * 
 93  
          * @see CachingDownloader#download(String, String)
 94  
          */
 95  
         public synchronized void download(String url, String cacheName) {
 96  0
                 download(url, null, cacheName);
 97  0
         }
 98  
 
 99  
         /**
 100  
          * Add a document to download, including POST data.
 101  
          * 
 102  
          * @param url
 103  
          *            URL of the document
 104  
          * @param postData
 105  
          *            postData to post
 106  
          * @param cacheName
 107  
          *            Name of the document in the cache
 108  
          * 
 109  
          * @see CachingDownloader#download(String, String, String)
 110  
          */
 111  
         public synchronized void download(String url, String postData, String cacheName) {
 112  0
                 q.add(new PrefetchEntry(url, postData, cacheName));
 113  0
                 notifyAll();
 114  0
         }
 115  
 
 116  
         /**
 117  
          * Wait until all prefetching is finished. This is useful if the results do
 118  
          * not have to be parsed, just downloaded, to make sure to wait until
 119  
          * everything requested by now has been downloaded.
 120  
          */
 121  
         public synchronized void waitPrefetch() throws InterruptedException {
 122  0
                 while (busy != 0 || !q.isEmpty())
 123  0
                         wait();
 124  0
         }
 125  
 
 126  
         /**
 127  
          * Wait until a given document has been prefetched.
 128  
          * 
 129  
          * @param cacheName
 130  
          *            Name of the document in the cache
 131  
          * @return The document content as a stream
 132  
          */
 133  
         public synchronized InputStream waitFor(String cacheName) throws InterruptedException {
 134  0
                 if (results == null)
 135  0
                         throw new IllegalStateException("Result caching has been disabled");
 136  0
                 while (!results.containsKey(cacheName))
 137  0
                         wait();
 138  0
                 return results.remove(cacheName);
 139  
         }
 140  
 
 141  
         /**
 142  
          * Wait until all prefetching is finished and shut down the prefetch
 143  
          * threads.
 144  
          */
 145  
         public synchronized void shutdown() throws InterruptedException {
 146  0
                 waitPrefetch();
 147  0
                 finished = true;
 148  0
                 notifyAll();
 149  0
         }
 150  
 
 151  
         private void runPrefetch() {
 152  
                 try {
 153  0
                         PrefetchEntry pe = null;
 154  
                         while (true) {
 155  0
                                 synchronized (this) {
 156  
                                         while (true) {
 157  0
                                                 if (finished)
 158  0
                                                         return;
 159  0
                                                 pe = q.poll();
 160  0
                                                 if (pe == null) {
 161  0
                                                         wait();
 162  
                                                 } else {
 163  
                                                         break;
 164  
                                                 }
 165  
                                         }
 166  0
                                         busy++;
 167  0
                                 }
 168  0
                                 InputStream in = cd.download(pe.url, pe.postData, pe.cacheName);
 169  0
                                 synchronized (this) {
 170  0
                                         busy--;
 171  0
                                         if (results != null)
 172  0
                                                 results.put(pe.cacheName, in);
 173  0
                                         notifyAll();
 174  0
                                 }
 175  0
                         }
 176  0
                 } catch (Exception ex) {
 177  0
                         ex.printStackTrace();
 178  
                 }
 179  0
         }
 180  
 
 181  0
         private static class PrefetchEntry {
 182  
                 private final String url;
 183  
                 private final String cacheName;
 184  
                 private final String postData;
 185  
 
 186  0
                 public PrefetchEntry(String url, String postData, String cacheName) {
 187  0
                         this.url = url;
 188  0
                         this.postData = postData;
 189  0
                         this.cacheName = cacheName;
 190  0
                 }
 191  
         }
 192  
 }