View Javadoc

1   package org.apache.torque.util;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *   http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import java.io.IOException;
23  import java.io.ObjectInputStream;
24  import java.io.Serializable;
25  import java.lang.reflect.Method;
26  import java.sql.Connection;
27  import java.sql.SQLException;
28  import java.util.ArrayList;
29  import java.util.Hashtable;
30  import java.util.Iterator;
31  import java.util.List;
32  import java.util.Set;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.torque.Torque;
37  import org.apache.torque.TorqueException;
38  
39  import com.workingdogs.village.DataSetException;
40  import com.workingdogs.village.QueryDataSet;
41  
42  /***
43   * This class can be used to retrieve a large result set from a database query.
44   * The query is started and then rows are returned a page at a time.  The <code>
45   * LargeSelect</code> is meant to be placed into the Session or User.Temp, so
46   * that it can be used in response to several related requests.  Note that in
47   * order to use <code>LargeSelect</code> you need to be willing to accept the
48   * fact that the result set may become inconsistent with the database if updates
49   * are processed subsequent to the queries being executed.  Specifying a memory
50   * page limit of 1 will give you a consistent view of the records but the totals
51   * may not be accurate and the performance will be terrible.  In most cases
52   * the potential for inconsistencies data should not cause any serious problems
53   * and performance should be pretty good (but read on for further warnings).
54   *
55   * <p>The idea here is that the full query result would consume too much memory
56   * and if displayed to a user the page would be too long to be useful.  Rather
57   * than loading the full result set into memory, a window of data (the memory
58   * limit) is loaded and retrieved a page at a time.  If a request occurs for
59   * data that falls outside the currently loaded window of data then a new query
60   * is executed to fetch the required data.  Performance is optimized by
61   * starting a thread to execute the database query and fetch the results.  This
62   * will perform best when paging forwards through the data, but a minor
63   * optimization where the window is moved backwards by two rather than one page
64   * is included for when a user pages past the beginning of the window.
65   *
66   * <p>As the query is performed in in steps, it is often the case that the total
67   * number of records and pages of data is unknown.  <code>LargeSelect</code>
68   * provides various methods for indicating how many records and pages it is
69   * currently aware of and for presenting this information to users.
70   *
71   * <p><code>LargeSelect</code> utilises the <code>Criteria</code> methods
72   * <code>setOffset()</code> and <code>setLimit()</code> to limit the amount of
73   * data retrieved from the database - these values are either passed through to
74   * the DBMS when supported (efficient with the caveat below) or handled by
75   * the Village API when it is not (not so efficient).  At time of writing
76   * <code>Criteria</code> will only pass the offset and limit through to MySQL
77   * and PostgreSQL (with a few changes to <code>DBOracle</code> and <code>
78   * BasePeer</code> Oracle support can be implemented by utilising the <code>
79   * rownum</code> pseudo column).
80   *
81   * <p>As <code>LargeSelect</code> must re-execute the query each time the user
82   * pages out of the window of loaded data, you should consider the impact of
83   * non-index sort orderings and other criteria that will require the DBMS to
84   * execute the entire query before filtering down to the offset and limit either
85   * internally or via Village.
86   *
87   * <p>The memory limit defaults to 5 times the page size you specify, but
88   * alternative constructors and the class method <code>setMemoryPageLimit()
89   * </code> allow you to override this for a specific instance of
90   * <code>LargeSelect</code> or future instances respectively.
91   *
92   * <p>Some of the constructors allow you to specify the name of the class to use
93   * to build the returnd rows.  This works by using reflection to find <code>
94   * addSelectColumns(Criteria)</code> and <code>populateObjects(List)</code>
95   * methods to add the necessary select columns to the criteria (only if it
96   * doesn't already contain any) and to convert query results from Village
97   * <code>Record</code> objects to a class defined within the builder class.
98   * This allows you to use any of the Torque generated Peer classes, but also
99   * makes it fairly simple to construct business object classes that can be used
100  * for this purpose (simply copy and customise the <code>addSelectColumns()
101  * </code>, <code>populateObjects()</code>, <code>row2Object()</code> and <code>
102  * populateObject()</code> methods from an existing Peer class).
103  *
104  * <p>Typically you will create a <code>LargeSelect</code> using your <code>
105  * Criteria</code> (perhaps created from the results of a search parameter
106  * page), page size, memory page limit and return class name (for which you may
107  * have defined a business object class before hand) and place this in user.Temp
108  * thus:
109  *
110  * <pre>
111  *     data.getUser().setTemp("someName", largeSelect);
112  * </pre>
113  *
114  * <p>In your template you will then use something along the lines of:
115  *
116  * <pre>
117  *    #set($largeSelect = $data.User.getTemp("someName"))
118  *    #set($searchop = $data.Parameters.getString("searchop"))
119  *    #if($searchop.equals("prev"))
120  *      #set($recs = $largeSelect.PreviousResults)
121  *    #else
122  *      #if($searchop.equals("goto"))
123  *        #set($recs = $largeSelect.getPage($data.Parameters.getInt("page", 1)))
124  *      #else
125  *        #set($recs = $largeSelect.NextResults)
126  *      #end
127  *    #end
128  * </pre>
129  *
130  * <p>...to move through the records.  <code>LargeSelect</code> implements a
131  * number of convenience methods that make it easy to add all of the necessary
132  * bells and whistles to your template.
133  *
134  * @author <a href="mailto:john.mcnally@clearink.com">John D. McNally</a>
135  * @author <a href="mailto:seade@backstagetech.com.au">Scott Eade</a>
136  * @version $Id: LargeSelect.java 476550 2006-11-18 16:08:37Z tfischer $
137  */
138 public class LargeSelect implements Runnable, Serializable
139 {
140     /*** Serial version */
141     private static final long serialVersionUID = -1166842932571491942L;
142 
143     /*** The number of records that a page consists of.  */
144     private int pageSize;
145     /*** The maximum number of records to maintain in memory. */
146     private int memoryLimit;
147 
148     /*** The record number of the first record in memory. */
149     private transient int blockBegin = 0;
150     /*** The record number of the last record in memory. */
151     private transient int blockEnd;
152     /*** How much of the memory block is currently occupied with result data. */
153     private volatile int currentlyFilledTo = -1;
154 
155     /*** The SQL query that this <code>LargeSelect</code> represents. */
156     private String query;
157     /*** The database name to get from Torque. */
158     private String dbName;
159 
160     /*** The memory store of records. */
161     private transient List results = null;
162 
163     /*** The thread that executes the query. */
164     private transient Thread thread = null;
165     /***
166      * A flag used to kill the thread when the currently executing query is no
167      * longer required.
168      */
169     private transient volatile boolean killThread = false;
170     /*** A flag that indicates whether or not the query thread is running. */
171     private transient volatile boolean threadRunning = false;
172     /***
173      * An indication of whether or not the current query has completed
174      * processing.
175      */
176     private transient volatile boolean queryCompleted = false;
177     /***
178      * An indication of whether or not the totals (records and pages) are at
179      * their final values.
180      */
181     private transient boolean totalsFinalized = false;
182 
183     /*** The cursor position in the result set. */
184     private int position;
185     /*** The total number of pages known to exist. */
186     private int totalPages = -1;
187     /*** The total number of records known to exist. */
188     private int totalRecords = 0;
189 
190     /*** The criteria used for the query. */
191     private Criteria criteria = null;
192     /*** The last page of results that were returned. */
193     private transient List lastResults;
194 
195     /***
196      * The class that is possibly used to construct the criteria and used
197      * to transform the Village Records into the desired OM or business objects.
198      */
199     private Class returnBuilderClass = null;
200     /***
201      * A reference to the method in the return builder class that will
202      * convert the Village Records to the desired class.
203      */
204     private transient Method populateObjectsMethod = null;
205 
206     /***
207      * The default value ("&gt;") used to indicate that the total number of
208      * records or pages is unknown.
209      */
210     public static final String DEFAULT_MORE_INDICATOR = "&gt;";
211 
212     /***
213      * The value used to indicate that the total number of records or pages is
214      * unknown (default: "&gt;"). You can use <code>setMoreIndicator()</code>
215      * to change this to whatever value you like (e.g. "more than").
216      */
217     private static String moreIndicator = DEFAULT_MORE_INDICATOR;
218 
219     /***
220      * The default value for the maximum number of pages of data to be retained
221      * in memory.
222      */
223     public static final int DEFAULT_MEMORY_LIMIT_PAGES = 5;
224 
225     /***
226      * The maximum number of pages of data to be retained in memory.  Use
227      * <code>setMemoryPageLimit()</code> to provide your own value.
228      */
229     private static int memoryPageLimit = DEFAULT_MEMORY_LIMIT_PAGES;
230 
231     /***
232      * The number of milliseconds to sleep when the result of a query
233      * is not yet available.
234      */
235     private static final int QUERY_NOT_COMPLETED_SLEEP_TIME = 500;
236 
237     /***
238      * The number of milliseconds to sleep before retrying to stop a query.
239      */
240     private static final int QUERY_STOP_SLEEP_TIME = 100;
241 
242     /*** A place to store search parameters that relate to this query. */
243     private Hashtable params = null;
244 
245     /*** Logging */
246     private static Log log = LogFactory.getLog(LargeSelect.class);
247 
248     /***
249      * Creates a LargeSelect whose results are returned as a <code>List</code>
250      * containing a maximum of <code>pageSize</code> Village <code>Record</code>
251      * objects at a time, maintaining a maximum of
252      * <code>LargeSelect.memoryPageLimit</code> pages of results in memory.
253      *
254      * @param criteria object used by BasePeer to build the query.  In order to
255      * allow this class to utilise database server implemented offsets and
256      * limits (when available), the provided criteria must not have any limit or
257      * offset defined.
258      * @param pageSize number of rows to return in one block.
259      * @throws IllegalArgumentException if <code>criteria</code> uses one or
260      * both of offset and limit, or if <code>pageSize</code> is less than 1;
261      */
262     public LargeSelect(Criteria criteria, int pageSize)
263     {
264         this(criteria, pageSize, LargeSelect.memoryPageLimit);
265     }
266 
267     /***
268      * Creates a LargeSelect whose results are returned as a <code>List</code>
269      * containing a maximum of <code>pageSize</code> Village <code>Record</code>
270      * objects at a time, maintaining a maximum of <code>memoryPageLimit</code>
271      * pages of results in memory.
272      *
273      * @param criteria object used by BasePeer to build the query.  In order to
274      * allow this class to utilise database server implemented offsets and
275      * limits (when available), the provided criteria must not have any limit or
276      * offset defined.
277      * @param pageSize number of rows to return in one block.
278      * @param memoryPageLimit maximum number of pages worth of rows to be held
279      * in memory at one time.
280      * @throws IllegalArgumentException if <code>criteria</code> uses one or
281      * both of offset and limit, or if <code>pageSize</code> or
282      * <code>memoryLimitPages</code> are less than 1;
283      */
284     public LargeSelect(Criteria criteria, int pageSize, int memoryPageLimit)
285     {
286         init(criteria, pageSize, memoryPageLimit);
287     }
288 
289     /***
290      * Creates a LargeSelect whose results are returned as a <code>List</code>
291      * containing a maximum of <code>pageSize</code> objects of the type
292      * defined within the class named <code>returnBuilderClassName</code> at a
293      * time, maintaining a maximum of <code>LargeSelect.memoryPageLimit</code>
294      * pages of results in memory.
295      *
296      * @param criteria object used by BasePeer to build the query.  In order to
297      * allow this class to utilise database server implemented offsets and
298      * limits (when available), the provided criteria must not have any limit or
299      * offset defined.  If the criteria does not include the definition of any
300      * select columns the <code>addSelectColumns(Criteria)</code> method of
301      * the class named as <code>returnBuilderClassName</code> will be used to
302      * add them.
303      * @param pageSize number of rows to return in one block.
304      * @param returnBuilderClassName The name of the class that will be used to
305      * build the result records (may implement <code>addSelectColumns(Criteria)
306      * </code> and must implement <code>populateObjects(List)</code>).
307      * @throws IllegalArgumentException if <code>criteria</code> uses one or
308      * both of offset and limit, if <code>pageSize</code> is less than 1, or if
309      * problems are experienced locating and invoking either one or both of
310      * <code>addSelectColumns(Criteria)</code> and <code> populateObjects(List)
311      * </code> in the class named <code>returnBuilderClassName</code>.
312      */
313     public LargeSelect(
314             Criteria criteria,
315             int pageSize,
316             String returnBuilderClassName)
317     {
318         this(
319             criteria,
320             pageSize,
321             LargeSelect.memoryPageLimit,
322             returnBuilderClassName);
323     }
324 
325     /***
326      * Creates a LargeSelect whose results are returned as a <code>List</code>
327      * containing a maximum of <code>pageSize</code> objects of the type
328      * defined within the class named <code>returnBuilderClassName</code> at a
329      * time, maintaining a maximum of <code>memoryPageLimit</code> pages of
330      * results in memory.
331      *
332      * @param criteria object used by BasePeer to build the query.  In order to
333      * allow this class to utilise database server implemented offsets and
334      * limits (when available), the provided criteria must not have any limit or
335      * offset defined.  If the criteria does not include the definition of any
336      * select columns the <code>addSelectColumns(Criteria)</code> method of
337      * the class named as <code>returnBuilderClassName</code> will be used to
338      * add them.
339      * @param pageSize number of rows to return in one block.
340      * @param memoryPageLimit maximum number of pages worth of rows to be held
341      * in memory at one time.
342      * @param returnBuilderClassName The name of the class that will be used to
343      * build the result records (may implement <code>addSelectColumns(Criteria)
344      * </code> and must implement <code>populateObjects(List)</code>).
345      * @throws IllegalArgumentException if <code>criteria</code> uses one or
346      * both of offset and limit, if <code>pageSize</code> or <code>
347      * memoryLimitPages</code> are less than 1, or if problems are experienced
348      * locating and invoking either one or both of <code>
349      * addSelectColumns(Criteria)</code> and <code> populateObjects(List)</code>
350      * in the class named <code>returnBuilderClassName</code>.
351      */
352     public LargeSelect(
353             Criteria criteria,
354             int pageSize,
355             int memoryPageLimit,
356             String returnBuilderClassName)
357     {
358         try
359         {
360             this.returnBuilderClass = Class.forName(returnBuilderClassName);
361 
362             // Add the select columns if necessary.
363             if (criteria.getSelectColumns().size() == 0)
364             {
365                 Class[] argTypes = { Criteria.class };
366                 Method selectColumnAdder =
367                     returnBuilderClass.getMethod("addSelectColumns", argTypes);
368                 Object[] theArgs = { criteria };
369                 selectColumnAdder.invoke(returnBuilderClass.newInstance(),
370                         theArgs);
371             }
372         }
373         catch (Exception e)
374         {
375             throw new IllegalArgumentException(
376                     "The class named as returnBuilderClassName does not "
377                     + "provide the necessary facilities - see javadoc.");
378         }
379 
380         init(criteria, pageSize, memoryPageLimit);
381     }
382 
383     /***
384      * Access the populateObjects method.
385      *
386      * @throws SecurityException if the security manager does not allow
387      *         access to the method.
388      * @throws NoSuchMethodException if the poulateObjects method does not
389      *         exist.
390      */
391     private Method getPopulateObjectsMethod()
392             throws NoSuchMethodException
393     {
394         if (null == populateObjectsMethod)
395         {
396             Class[] argTypes = { List.class };
397             populateObjectsMethod
398                     = returnBuilderClass.getMethod("populateObjects", argTypes);
399         }
400         return populateObjectsMethod;
401     }
402 
403     /***
404      * Called by the constructors to start the query.
405      *
406      * @param criteria Object used by <code>BasePeer</code> to build the query.
407      * In order to allow this class to utilise database server implemented
408      * offsets and limits (when available), the provided criteria must not have
409      * any limit or offset defined.
410      * @param pageSize number of rows to return in one block.
411      * @param memoryLimitPages maximum number of pages worth of rows to be held
412      * in memory at one time.
413      * @throws IllegalArgumentException if <code>criteria</code> uses one or
414      * both of offset and limit and if <code>pageSize</code> or
415      * <code>memoryLimitPages</code> are less than 1;
416      */
417     private void init(Criteria criteria, int pageSize, int memoryLimitPages)
418     {
419         if (criteria.getOffset() != 0 || criteria.getLimit() != -1)
420         {
421             throw new IllegalArgumentException(
422                     "criteria must not use Offset and/or Limit.");
423         }
424 
425         if (pageSize < 1)
426         {
427             throw new IllegalArgumentException(
428                     "pageSize must be greater than zero.");
429         }
430 
431         if (memoryLimitPages < 1)
432         {
433             throw new IllegalArgumentException(
434                     "memoryPageLimit must be greater than zero.");
435         }
436 
437         this.pageSize = pageSize;
438         this.memoryLimit = pageSize * memoryLimitPages;
439         this.criteria = criteria;
440         dbName = criteria.getDbName();
441         blockEnd = blockBegin + memoryLimit - 1;
442         startQuery(pageSize);
443     }
444 
445     /***
446      * Retrieve a specific page, if it exists.
447      *
448      * @param pageNumber the number of the page to be retrieved - must be
449      * greater than zero.  An empty <code>List</code> will be returned if
450      * <code>pageNumber</code> exceeds the total number of pages that exist.
451      * @return a <code>List</code> of query results containing a maximum of
452      * <code>pageSize</code> results.
453      * @throws IllegalArgumentException when <code>pageNo</code> is not
454      * greater than zero.
455      * @throws TorqueException if invoking the <code>populateObjects()<code>
456      * method runs into problems or a sleep is unexpectedly interrupted.
457      */
458     public List getPage(int pageNumber) throws TorqueException
459     {
460         if (pageNumber < 1)
461         {
462             throw new IllegalArgumentException(
463                     "pageNumber must be greater than zero.");
464         }
465         return getResults((pageNumber - 1) * pageSize);
466     }
467 
468     /***
469      * Gets the next page of rows.
470      *
471      * @return a <code>List</code> of query results containing a maximum of
472      * <code>pageSize</code> reslts.
473      * @throws TorqueException if invoking the <code>populateObjects()<code>
474      * method runs into problems or a sleep is unexpectedly interrupted.
475      */
476     public List getNextResults() throws TorqueException
477     {
478         if (!getNextResultsAvailable())
479         {
480             return getCurrentPageResults();
481         }
482         return getResults(position);
483     }
484 
485     /***
486      * Provide access to the results from the current page.
487      *
488      * @return a <code>List</code> of query results containing a maximum of
489      * <code>pageSize</code> reslts.
490      * @throws TorqueException if invoking the <code>populateObjects()<code>
491      * method runs into problems or a sleep is unexpectedly interrupted.
492      */
493     public List getCurrentPageResults() throws TorqueException
494     {
495         return null == lastResults && position > 0
496                 ? getResults(position) : lastResults;
497     }
498 
499     /***
500      * Gets the previous page of rows.
501      *
502      * @return a <code>List</code> of query results containing a maximum of
503      * <code>pageSize</code> reslts.
504      * @throws TorqueException if invoking the <code>populateObjects()<code>
505      * method runs into problems or a sleep is unexpectedly interrupted.
506      */
507     public List getPreviousResults() throws TorqueException
508     {
509         if (!getPreviousResultsAvailable())
510         {
511             return getCurrentPageResults();
512         }
513 
514         int start;
515         if (position - 2 * pageSize < 0)
516         {
517             start = 0;
518         }
519         else
520         {
521             start = position - 2 * pageSize;
522         }
523         return getResults(start);
524     }
525 
526     /***
527      * Gets a page of rows starting at a specified row.
528      *
529      * @param start the starting row.
530      * @return a <code>List</code> of query results containing a maximum of
531      * <code>pageSize</code> reslts.
532      * @throws TorqueException if invoking the <code>populateObjects()<code>
533      * method runs into problems or a sleep is unexpectedly interrupted.
534      */
535     private List getResults(int start) throws TorqueException
536     {
537         return getResults(start, pageSize);
538     }
539 
540     /***
541      * Gets a block of rows starting at a specified row and containing a
542      * specified number of rows.
543      *
544      * @param start the starting row.
545      * @param size the number of rows.
546      * @return a <code>List</code> of query results containing a maximum of
547      * <code>pageSize</code> reslts.
548      * @throws IllegalArgumentException if <code>size &gt; memoryLimit</code> or
549      * <code>start</code> and <code>size</code> result in a situation that is
550      * not catered for.
551      * @throws TorqueException if invoking the <code>populateObjects()<code>
552      * method runs into problems or a sleep is unexpectedly interrupted.
553      */
554     private synchronized List getResults(int start, int size)
555             throws TorqueException
556     {
557         if (log.isDebugEnabled())
558         {
559             log.debug("getResults(start: " + start
560                     + ", size: " + size + ") invoked.");
561         }
562 
563         if (size > memoryLimit)
564         {
565             throw new IllegalArgumentException("size (" + size
566                     + ") exceeds memory limit (" + memoryLimit + ").");
567         }
568 
569         // Request was for a block of rows which should be in progess.
570         // If the rows have not yet been returned, wait for them to be
571         // retrieved.
572         if (start >= blockBegin && (start + size - 1) <= blockEnd)
573         {
574             if (log.isDebugEnabled())
575             {
576                 log.debug("getResults(): Sleeping until "
577                         + "start+size-1 (" + (start + size - 1)
578                         + ") > currentlyFilledTo (" + currentlyFilledTo
579                         + ") && !queryCompleted (!" + queryCompleted + ")");
580             }
581             while (((start + size - 1) > currentlyFilledTo) && !queryCompleted)
582             {
583                 try
584                 {
585                     Thread.sleep(QUERY_NOT_COMPLETED_SLEEP_TIME);
586                 }
587                 catch (InterruptedException e)
588                 {
589                     throw new TorqueException("Unexpected interruption", e);
590                 }
591             }
592         }
593 
594         // Going in reverse direction, trying to limit db hits so assume user
595         // might want at least 2 sets of data.
596         else if (start < blockBegin && start >= 0)
597         {
598             if (log.isDebugEnabled())
599             {
600                 log.debug("getResults(): Paging backwards as start (" + start
601                         + ") < blockBegin (" + blockBegin + ") && start >= 0");
602             }
603             stopQuery();
604             if (memoryLimit >= 2 * size)
605             {
606                 blockBegin = start - size;
607                 if (blockBegin < 0)
608                 {
609                     blockBegin = 0;
610                 }
611             }
612             else
613             {
614                 blockBegin = start;
615             }
616             blockEnd = blockBegin + memoryLimit - 1;
617             startQuery(size);
618             // Re-invoke getResults() to provide the wait processing.
619             return getResults(start, size);
620         }
621 
622         // Assume we are moving on, do not retrieve any records prior to start.
623         else if ((start + size - 1) > blockEnd)
624         {
625             if (log.isDebugEnabled())
626             {
627                 log.debug("getResults(): Paging past end of loaded data as "
628                         + "start+size-1 (" + (start + size - 1)
629                         + ") > blockEnd (" + blockEnd + ")");
630             }
631             stopQuery();
632             blockBegin = start;
633             blockEnd = blockBegin + memoryLimit - 1;
634             startQuery(size);
635             // Re-invoke getResults() to provide the wait processing.
636             return getResults(start, size);
637         }
638 
639         else
640         {
641             throw new IllegalArgumentException("Parameter configuration not "
642                     + "accounted for.");
643         }
644 
645         int fromIndex = start - blockBegin;
646         int toIndex = fromIndex + Math.min(size, results.size() - fromIndex);
647 
648         if (log.isDebugEnabled())
649         {
650             log.debug("getResults(): Retrieving records from results elements "
651                     + "start-blockBegin (" + fromIndex + ") through "
652                     + "fromIndex + Math.min(size, results.size() - fromIndex) ("
653                     + toIndex + ")");
654         }
655 
656         List returnResults;
657 
658         synchronized (results)
659         {
660             returnResults = new ArrayList(results.subList(fromIndex, toIndex));
661         }
662 
663         if (null != returnBuilderClass)
664         {
665             // Invoke the populateObjects() method
666             Object[] theArgs = { returnResults };
667             try
668             {
669                 returnResults = (List) getPopulateObjectsMethod().invoke(
670                         returnBuilderClass.newInstance(), theArgs);
671             }
672             catch (Exception e)
673             {
674                 throw new TorqueException("Unable to populate results", e);
675             }
676         }
677         position = start + size;
678         lastResults = returnResults;
679         return returnResults;
680     }
681 
682     /***
683      * A background thread that retrieves the rows.
684      */
685     public void run()
686     {
687         boolean dbSupportsNativeLimit;
688         boolean dbSupportsNativeOffset;
689         try
690         {
691             dbSupportsNativeLimit
692                     = (Torque.getDB(dbName).supportsNativeLimit());
693             dbSupportsNativeOffset
694                     = (Torque.getDB(dbName).supportsNativeOffset());
695         }
696         catch (TorqueException e)
697         {
698             log.error("run() : Exiting :", e);
699             // we cannot execute further because Torque is not initialized
700             // correctly
701             return;
702         }
703 
704         int size;
705         if (dbSupportsNativeLimit && dbSupportsNativeOffset)
706         {
707             // retrieve one page at a time
708             size = pageSize;
709         }
710         else
711         {
712             // retrieve the whole block at once and add the offset,
713             // and add one record to check if we have reached the end of the
714             // data
715             size = blockBegin + memoryLimit + 1;
716         }
717         /* The connection to the database. */
718         Connection conn = null;
719         /*** Used to retrieve query results from Village. */
720         QueryDataSet qds = null;
721 
722         try
723         {
724             // Add 1 to memory limit to check if the query ends on a page break.
725             results = new ArrayList(memoryLimit + 1);
726 
727             // Use the criteria to limit the rows that are retrieved to the
728             // block of records that fit in the predefined memoryLimit.
729             if (dbSupportsNativeLimit)
730             {
731                 if (dbSupportsNativeOffset)
732                 {
733                     criteria.setOffset(blockBegin);
734                     // Add 1 to memory limit to check if the query ends on a
735                     // page break.
736                     criteria.setLimit(memoryLimit + 1);
737                 }
738                 else
739                 {
740                     criteria.setLimit(blockBegin + memoryLimit + 1);
741                 }
742             }
743             query = BasePeer.createQueryString(criteria);
744 
745             // Get a connection to the db.
746             conn = Torque.getConnection(dbName);
747 
748             // Execute the query.
749             if (log.isDebugEnabled())
750             {
751                 log.debug("run(): query = " + query);
752                 log.debug("run(): memoryLimit = " + memoryLimit);
753                 log.debug("run(): blockBegin = " + blockBegin);
754                 log.debug("run(): blockEnd = " + blockEnd);
755             }
756             qds = new QueryDataSet(conn, query);
757 
758             // Continue getting rows one page at a time until the memory limit
759             // is reached, all results have been retrieved, or the rest
760             // of the results have been determined to be irrelevant.
761             while (!killThread
762                 && !qds.allRecordsRetrieved()
763                 && currentlyFilledTo + pageSize <= blockEnd)
764             {
765                 // This caters for when memoryLimit is not a multiple of
766                 //  pageSize which it never is because we always add 1 above.
767                 // not applicable if the db has no native limit where this
768                 // was already considered
769                 if ((currentlyFilledTo + pageSize) >= blockEnd
770                         && dbSupportsNativeLimit)
771                 {
772                     // Add 1 to check if the query ends on a page break.
773                     size = blockEnd - currentlyFilledTo + 1;
774                 }
775 
776                 if (log.isDebugEnabled())
777                 {
778                     log.debug("run(): Invoking BasePeer.getSelectResults(qds, "
779                             + size + ", false)");
780                 }
781 
782                 List tempResults
783                         = BasePeer.getSelectResults(qds, size, false);
784 
785                 int startIndex = dbSupportsNativeOffset ? 0 : blockBegin;
786 
787                 synchronized (results)
788                 {
789                     for (int i = startIndex, n = tempResults.size(); i < n; i++)
790                     {
791                         results.add(tempResults.get(i));
792                     }
793                 }
794 
795                 if (dbSupportsNativeLimit && dbSupportsNativeOffset)
796                 {
797                     currentlyFilledTo += tempResults.size();
798                 }
799                 else
800                 {
801                     currentlyFilledTo = tempResults.size() - 1 - blockBegin;
802                 }
803 
804                 boolean perhapsLastPage = true;
805 
806                 // If the extra record was indeed found then we know we are not
807                 // on the last page but we must now get rid of it.
808                 if ((dbSupportsNativeLimit
809                         && (results.size() == memoryLimit + 1))
810                     || (!dbSupportsNativeLimit
811                             && currentlyFilledTo >= memoryLimit))
812                 {
813                     synchronized (results)
814                     {
815                         results.remove(currentlyFilledTo--);
816                     }
817                     perhapsLastPage = false;
818                 }
819 
820                 if (results.size() > 0
821                     && blockBegin + currentlyFilledTo >= totalRecords)
822                 {
823                     // Add 1 because index starts at 0
824                     totalRecords = blockBegin + currentlyFilledTo + 1;
825                 }
826 
827                 // if the db has limited the datasets, we must retrieve all
828                 // datasets. If not, we are always finished because we fetch
829                 // the whole block at once.
830                 if (qds.allRecordsRetrieved()
831                         || !dbSupportsNativeLimit)
832                 {
833                     queryCompleted = true;
834                     // The following ugly condition ensures that the totals are
835                     // not finalized when a user does something like requesting
836                     // a page greater than what exists in the database.
837                     if (perhapsLastPage
838                         && getCurrentPageNumber() <= getTotalPages())
839                     {
840                         totalsFinalized = true;
841                     }
842                 }
843                 qds.clearRecords();
844             }
845 
846             if (log.isDebugEnabled())
847             {
848                 log.debug("run(): While loop terminated because either:");
849                 log.debug("run(): 1. qds.allRecordsRetrieved(): "
850                         + qds.allRecordsRetrieved());
851                 log.debug("run(): 2. killThread: " + killThread);
852                 log.debug("run(): 3. !(currentlyFilledTo + size <= blockEnd): !"
853                         + (currentlyFilledTo + pageSize <= blockEnd));
854                 log.debug("run(): - currentlyFilledTo: " + currentlyFilledTo);
855                 log.debug("run(): - size: " + pageSize);
856                 log.debug("run(): - blockEnd: " + blockEnd);
857                 log.debug("run(): - results.size(): " + results.size());
858             }
859         }
860         catch (TorqueException e)
861         {
862             log.error(e);
863         }
864         catch (SQLException e)
865         {
866             log.error(e);
867         }
868         catch (DataSetException e)
869         {
870             log.error(e);
871         }
872         finally
873         {
874             try
875             {
876                 if (qds != null)
877                 {
878                     qds.close();
879                 }
880                 Torque.closeConnection(conn);
881             }
882             catch (SQLException e)
883             {
884                 log.error(e);
885             }
886             catch (DataSetException e)
887             {
888                 log.error(e);
889             }
890             threadRunning = false;
891         }
892     }
893 
894     /***
895      * Starts a new thread to retrieve the result set.
896      *
897      * @param initialSize the initial size for each block.
898      */
899     private synchronized void startQuery(int initialSize)
900     {
901         if (!threadRunning)
902         {
903             pageSize = initialSize;
904             currentlyFilledTo = -1;
905             queryCompleted = false;
906             thread = new Thread(this);
907             thread.start();
908             threadRunning = true;
909         }
910     }
911 
912     /***
913      * Used to stop filling the memory with the current block of results, if it
914      * has been determined that they are no longer relevant.
915      *
916      * @throws TorqueException if a sleep is interrupted.
917      */
918     private synchronized void stopQuery() throws TorqueException
919     {
920         if (threadRunning)
921         {
922             killThread = true;
923             while (thread.isAlive())
924             {
925                 try
926                 {
927                     Thread.sleep(QUERY_STOP_SLEEP_TIME);
928                 }
929                 catch (InterruptedException e)
930                 {
931                     throw new TorqueException("Unexpected interruption", e);
932                 }
933             }
934             killThread = false;
935         }
936     }
937 
938     /***
939      * Retrieve the number of the current page.
940      *
941      * @return the current page number.
942      */
943     public int getCurrentPageNumber()
944     {
945         return position / pageSize;
946     }
947 
948     /***
949      * Retrieve the total number of search result records that are known to
950      * exist (this will be the actual value when the query has completeted (see
951      * <code>getTotalsFinalized()</code>).  The convenience method
952      * <code>getRecordProgressText()</code> may be more useful for presenting to
953      * users.
954      *
955      * @return the number of result records known to exist (not accurate until
956      * <code>getTotalsFinalized()</code> returns <code>true</code>).
957      */
958     public int getTotalRecords()
959     {
960         return totalRecords;
961     }
962 
963     /***
964      * Provide an indication of whether or not paging of results will be
965      * required.
966      *
967      * @return <code>true</code> when multiple pages of results exist.
968      */
969     public boolean getPaginated()
970     {
971         // Handle a page memory limit of 1 page.
972         if (!getTotalsFinalized())
973         {
974             return true;
975         }
976         return blockBegin + currentlyFilledTo + 1 > pageSize;
977     }
978 
979     /***
980      * Retrieve the total number of pages of search results that are known to
981      * exist (this will be the actual value when the query has completeted (see
982      * <code>getQyeryCompleted()</code>).  The convenience method
983      * <code>getPageProgressText()</code> may be more useful for presenting to
984      * users.
985      *
986      * @return the number of pages of results known to exist (not accurate until
987      * <code>getTotalsFinalized()</code> returns <code>true</code>).
988      */
989     public int getTotalPages()
990     {
991         if (totalPages > -1)
992         {
993             return totalPages;
994         }
995 
996         int tempPageCount =  getTotalRecords() / pageSize
997                 + (getTotalRecords() % pageSize > 0 ? 1 : 0);
998 
999         if (getTotalsFinalized())
1000         {
1001             totalPages = tempPageCount;
1002         }
1003 
1004         return tempPageCount;
1005     }
1006 
1007     /***
1008      * Retrieve the page size.
1009      *
1010      * @return the number of records returned on each invocation of
1011      * <code>getNextResults()</code>/<code>getPreviousResults()</code>.
1012      */
1013     public int getPageSize()
1014     {
1015         return pageSize;
1016     }
1017 
1018     /***
1019      * Provide access to indicator that the total values for the number of
1020      * records and pages are now accurate as opposed to known upper limits.
1021      *
1022      * @return <code>true</code> when the totals are known to have been fully
1023      * computed.
1024      */
1025     public boolean getTotalsFinalized()
1026     {
1027         return totalsFinalized;
1028     }
1029 
1030     /***
1031      * Provide a way of changing the more pages/records indicator.
1032      *
1033      * @param moreIndicator the indicator to use in place of the default
1034      * ("&gt;").
1035      */
1036     public static void setMoreIndicator(String moreIndicator)
1037     {
1038         LargeSelect.moreIndicator = moreIndicator;
1039     }
1040 
1041     /***
1042      * Retrieve the more pages/records indicator.
1043      */
1044     public static String getMoreIndicator()
1045     {
1046         return LargeSelect.moreIndicator;
1047     }
1048 
1049     /***
1050      * Sets the multiplier that will be used to compute the memory limit when a
1051      * constructor with no memory page limit is used - the memory limit will be
1052      * this number multiplied by the page size.
1053      *
1054      * @param memoryPageLimit the maximum number of pages to be in memory
1055      * at one time.
1056      */
1057     public static void setMemoryPageLimit(int memoryPageLimit)
1058     {
1059         LargeSelect.memoryPageLimit = memoryPageLimit;
1060     }
1061 
1062     /***
1063      * Retrieves the multiplier that will be used to compute the memory limit
1064      * when a constructor with no memory page limit is used - the memory limit
1065      * will be this number multiplied by the page size.
1066      */
1067     public static int getMemoryPageLimit()
1068     {
1069         return LargeSelect.memoryPageLimit;
1070     }
1071 
1072     /***
1073      * A convenience method that provides text showing progress through the
1074      * selected rows on a page basis.
1075      *
1076      * @return progress text in the form of "1 of &gt; 5" where "&gt;" can be
1077      * configured using <code>setMoreIndicator()</code>.
1078      */
1079     public String getPageProgressText()
1080     {
1081         StringBuffer result = new StringBuffer();
1082         result.append(getCurrentPageNumber());
1083         result.append(" of ");
1084         if (!totalsFinalized)
1085         {
1086             result.append(moreIndicator);
1087             result.append(" ");
1088         }
1089         result.append(getTotalPages());
1090         return result.toString();
1091     }
1092 
1093     /***
1094      * Provides a count of the number of rows to be displayed on the current
1095      * page - for the last page this may be less than the configured page size.
1096      *
1097      * @return the number of records that are included on the current page of
1098      * results.
1099      * @throws TorqueException if invoking the <code>populateObjects()<code>
1100      * method runs into problems or a sleep is unexpectedly interrupted.
1101      */
1102     public int getCurrentPageSize() throws TorqueException
1103     {
1104         if (null == getCurrentPageResults())
1105         {
1106             return 0;
1107         }
1108         return getCurrentPageResults().size();
1109     }
1110 
1111     /***
1112      * Provide the record number of the first row included on the current page.
1113      *
1114      * @return The record number of the first row of the current page.
1115      */
1116     public int getFirstRecordNoForPage()
1117     {
1118         if (getCurrentPageNumber() < 1)
1119         {
1120             return 0;
1121         }
1122         return (getCurrentPageNumber() - 1) * getPageSize() + 1;
1123     }
1124 
1125     /***
1126      * Provide the record number of the last row included on the current page.
1127      *
1128      * @return the record number of the last row of the current page.
1129      * @throws TorqueException if invoking the <code>populateObjects()<code>
1130      * method runs into problems or a sleep is unexpectedly interrupted.
1131      */
1132     public int getLastRecordNoForPage() throws TorqueException
1133     {
1134         if (0 == getCurrentPageNumber())
1135         {
1136             return 0;
1137         }
1138         return (getCurrentPageNumber() - 1) * getPageSize()
1139                 + getCurrentPageSize();
1140     }
1141 
1142     /***
1143      * A convenience method that provides text showing progress through the
1144      * selected rows on a record basis.
1145      *
1146      * @return progress text in the form of "26 - 50 of &gt; 250" where "&gt;"
1147      * can be configured using <code>setMoreIndicator()</code>.
1148      * @throws TorqueException if invoking the <code>populateObjects()<code>
1149      * method runs into problems or a sleep is unexpectedly interrupted.
1150      */
1151     public String getRecordProgressText() throws TorqueException
1152     {
1153         StringBuffer result = new StringBuffer();
1154         result.append(getFirstRecordNoForPage());
1155         result.append(" - ");
1156         result.append(getLastRecordNoForPage());
1157         result.append(" of ");
1158         if (!totalsFinalized)
1159         {
1160             result.append(moreIndicator);
1161             result.append(" ");
1162         }
1163         result.append(getTotalRecords());
1164         return result.toString();
1165     }
1166 
1167     /***
1168      * Indicates if further result pages are available.
1169      *
1170      * @return <code>true</code> when further results are available.
1171      */
1172     public boolean getNextResultsAvailable()
1173     {
1174         if (!totalsFinalized || getCurrentPageNumber() < getTotalPages())
1175         {
1176             return true;
1177         }
1178         return false;
1179     }
1180 
1181     /***
1182      * Indicates if previous results pages are available.
1183      *
1184      * @return <code>true</code> when previous results are available.
1185      */
1186     public boolean getPreviousResultsAvailable()
1187     {
1188         if (getCurrentPageNumber() <= 1)
1189         {
1190             return false;
1191         }
1192         return true;
1193     }
1194 
1195     /***
1196      * Indicates if any results are available.
1197      *
1198      * @return <code>true</code> of any results are available.
1199      */
1200     public boolean hasResultsAvailable()
1201     {
1202         return getTotalRecords() > 0;
1203     }
1204 
1205     /***
1206      * Clear the query result so that the query is reexecuted when the next page
1207      * is retrieved.  You may want to invoke this method if you are returning to
1208      * a page after performing an operation on an item in the result set.
1209      *
1210      * @throws TorqueException if a sleep is interrupted.
1211      */
1212     public synchronized void invalidateResult() throws TorqueException
1213     {
1214         stopQuery();
1215         blockBegin = 0;
1216         blockEnd = 0;
1217         currentlyFilledTo = -1;
1218         results = null;
1219         // TODO Perhaps store the oldPosition and immediately restart the
1220         // query.
1221         // oldPosition = position;
1222         position = 0;
1223         totalPages = -1;
1224         totalRecords = 0;
1225         queryCompleted = false;
1226         totalsFinalized = false;
1227         lastResults = null;
1228     }
1229 
1230     /***
1231      * Retrieve a search parameter.  This acts as a convenient place to store
1232      * parameters that relate to the LargeSelect to make it easy to get at them
1233      * in order to repopulate search parameters on a form when the next page of
1234      * results is retrieved - they in no way effect the operation of
1235      * LargeSelect.
1236      *
1237      * @param name the search parameter key to retrieve.
1238      * @return the value of the search parameter.
1239      */
1240     public String getSearchParam(String name)
1241     {
1242         return getSearchParam(name, null);
1243     }
1244 
1245     /***
1246      * Retrieve a search parameter.  This acts as a convenient place to store
1247      * parameters that relate to the LargeSelect to make it easy to get at them
1248      * in order to repopulate search parameters on a form when the next page of
1249      * results is retrieved - they in no way effect the operation of
1250      * LargeSelect.
1251      *
1252      * @param name the search parameter key to retrieve.
1253      * @param defaultValue the default value to return if the key is not found.
1254      * @return the value of the search parameter.
1255      */
1256     public String getSearchParam(String name, String defaultValue)
1257     {
1258         if (null == params)
1259         {
1260             return defaultValue;
1261         }
1262         String value = (String) params.get(name);
1263         return null == value ? defaultValue : value;
1264     }
1265 
1266     /***
1267      * Set a search parameter.  If the value is <code>null</code> then the
1268      * key will be removed from the parameters.
1269      *
1270      * @param name the search parameter key to set.
1271      * @param value the value of the search parameter to store.
1272      */
1273     public void setSearchParam(String name, String value)
1274     {
1275         if (null == value)
1276         {
1277             removeSearchParam(name);
1278         }
1279         else
1280         {
1281             if (null != name)
1282             {
1283                 if (null == params)
1284                 {
1285                     params = new Hashtable();
1286                 }
1287                 params.put(name, value);
1288             }
1289         }
1290     }
1291 
1292     /***
1293      * Remove a value from the search parameters.
1294      *
1295      * @param name the search parameter key to remove.
1296      */
1297     public void removeSearchParam(String name)
1298     {
1299         if (null != params)
1300         {
1301             params.remove(name);
1302         }
1303     }
1304 
1305     /***
1306      * Deserialize this LargeSelect instance.
1307      *
1308      * @param inputStream The serialization input stream.
1309      * @throws IOException
1310      * @throws ClassNotFoundException
1311      */
1312     private void readObject(ObjectInputStream inputStream)
1313             throws IOException, ClassNotFoundException
1314     {
1315         inputStream.defaultReadObject();
1316 
1317         // avoid NPE because of Tomcat de-serialization of sessions
1318         if (Torque.isInit())
1319         {
1320             startQuery(pageSize);
1321         }
1322     }
1323 
1324     /***
1325      * Provide something useful for debugging purposes.
1326      *
1327      * @return some basic information about this instance of LargeSelect.
1328      */
1329     public String toString()
1330     {
1331         StringBuffer result = new StringBuffer();
1332         result.append("LargeSelect - TotalRecords: ");
1333         result.append(getTotalRecords());
1334         result.append(" TotalsFinalised: ");
1335         result.append(getTotalsFinalized());
1336         result.append("\nParameters:");
1337         if (null == params || params.size() == 0)
1338         {
1339             result.append(" No parameters have been set.");
1340         }
1341         else
1342         {
1343             Set keys = params.keySet();
1344             for (Iterator iter = keys.iterator(); iter.hasNext();)
1345             {
1346                 String key = (String) iter.next();
1347                 String val = (String) params.get(key);
1348                 result.append("\n ").append(key).append(": ").append(val);
1349             }
1350         }
1351         return result.toString();
1352     }
1353 
1354 }