Changeset 2200


Ignore:
Timestamp:
02/01/07 15:35:19 (13 years ago)
Author:
melissa
Message:

Added logic to RandomAccessStream that keeps track of how many files we have
open - files are then (automatically) closed and reopened as needed.
Unfortunately, there is a slight penalty in terms of memory, but it's not
significant and should be trivial to fix.

Location:
trunk/loci/formats
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • trunk/loci/formats/RandomAccessStream.java

    r2178 r2200  
    2626 
    2727import java.io.*; 
    28 import java.util.Vector; 
     28import java.util.*; 
    2929 
    3030/** 
    3131 * RandomAccessStream provides methods for "intelligent" reading of files and 
    32  * byte arrays. 
     32 * byte arrays.  It also automagically deals with closing and reopening files 
     33 * to prevent an IOException caused by too many open files. 
    3334 * 
    3435 * @author Melissa Linkert, linkert at wisc.edu 
     
    4546  protected static final int MAX_HISTORY = 50; 
    4647 
     48  /** Maximum number of open files. */ 
     49  protected static final int MAX_FILES = 128; 
     50 
    4751  /** Indicators for most efficient method of reading. */ 
    4852  protected static final int DIS = 0; 
    4953  protected static final int RAF = 1; 
    5054  protected static final int ARRAY = 2; 
     55 
     56  // -- Static fields -- 
     57 
     58  /** Hashtable of all files that have been opened at some point. */ 
     59  private static Hashtable fileCache = new Hashtable(); 
     60 
     61  /** Number of currently open files. */ 
     62  private static int openFiles = 0; 
    5163 
    5264  // -- Fields -- 
     
    109121    fp = 0; 
    110122    afp = 0; 
     123    fileCache.put(this, Boolean.TRUE); 
     124    openFiles++; 
     125    if (openFiles > MAX_FILES) cleanCache(); 
    111126  } 
    112127 
    113128  /** Constructs a random access stream around the given byte array. */ 
    114129  public RandomAccessStream(byte[] array) throws IOException { 
     130    // this doesn't use a file descriptor, so we don't need to add it to the 
     131    // file cache 
    115132    raf = new RABytes(array); 
    116133    fp = 0; 
     
    121138 
    122139  /** Return the underlying InputStream */ 
    123   public DataInputStream getInputStream() { return dis; } 
     140  public DataInputStream getInputStream() {  
     141    try { 
     142      if (fileCache.get(this) == Boolean.FALSE) reopen(); 
     143    } 
     144    catch (IOException e) { 
     145      return null; 
     146    } 
     147    return dis;  
     148  } 
    124149 
    125150  /** 
     
    140165 
    141166  /** Gets the number of bytes in the file. */ 
    142   public long length() throws IOException { return raf.length(); } 
     167  public long length() throws IOException {  
     168    if (fileCache.get(this) == Boolean.FALSE) reopen(); 
     169    return raf.length();  
     170  } 
    143171 
    144172  /** Gets the current (absolute) file pointer. */ 
     
    151179    if (dis != null) dis.close(); 
    152180    dis = null; 
    153     fp = 0; 
    154     mark = 0; 
     181    fileCache.put(this, Boolean.FALSE); 
     182    openFiles--; 
    155183  } 
    156184 
     
    345373 
    346374  public int available() throws IOException { 
     375    if (fileCache.get(this) == Boolean.FALSE) reopen(); 
    347376    return dis != null ? dis.available() + ext : 
    348377      (int) (length() - getFilePointer()); 
     
    350379 
    351380  public void mark(int readLimit) { 
     381    try { 
     382      if (fileCache.get(this) == Boolean.FALSE) reopen(); 
     383    } 
     384    catch (IOException e) { } 
    352385    dis.mark(readLimit); 
    353386  } 
     
    356389 
    357390  public void reset() throws IOException { 
     391    if (fileCache.get(this) == Boolean.FALSE) reopen(); 
    358392    dis.reset(); 
    359393    fp = (int) (length() - dis.available()); 
    360394  } 
    361395 
    362   // -- Helper methods -- 
     396  // -- Helper methods - I/O -- 
    363397 
    364398  /** Naive heuristic for determining a "good" buffer size for the DIS. */ 
     
    395429   */ 
    396430  protected int checkEfficiency(int toRead) throws IOException { 
     431    if (fileCache.get(this) == Boolean.FALSE) reopen(); 
    397432    int oldBufferSize = 0; 
    398433    if (recent != null) { 
     
    480515  } 
    481516 
     517  // -- Helper methods - cache management -- 
     518 
     519  /** Re-open a file that has been closed */ 
     520  private void reopen() throws IOException { 
     521    File f = new File(Location.getMappedId(file)); 
     522    f = f.getAbsoluteFile(); 
     523    if (f.exists()) { 
     524      raf = new RAFile(f, "r"); 
     525      dis = new DataInputStream(new BufferedInputStream( 
     526        new FileInputStream(Location.getMappedId(file)), MAX_OVERHEAD)); 
     527    } 
     528    else { 
     529      raf = new RAUrl(Location.getMappedId(file), "r"); 
     530    } 
     531    fileCache.put(this, Boolean.TRUE); 
     532    openFiles++; 
     533    if (openFiles > MAX_FILES) cleanCache(); 
     534  } 
     535 
     536  /** If we have too many open files, close 1/4 of them. */ 
     537  private void cleanCache() { 
     538    int toClose = MAX_FILES - 10; 
     539    RandomAccessStream[] files = (RandomAccessStream[])  
     540      fileCache.keySet().toArray(new RandomAccessStream[0]); 
     541    int closed = 0; 
     542    int ndx = 0; 
     543    int blahCounter = 0; 
     544    int oldOpen = openFiles; 
     545     
     546    while (closed < toClose) { 
     547      if (!this.equals(files[ndx]) &&  
     548        !fileCache.get(files[ndx]).equals(Boolean.FALSE))  
     549      { 
     550        try { files[ndx].close(); } 
     551        catch (IOException e) { e.printStackTrace(); } 
     552        blahCounter++; 
     553        closed++; 
     554      } 
     555      ndx++; 
     556    } 
     557  } 
     558 
    482559} 
  • trunk/loci/formats/in/EPSReader.java

    r2182 r2200  
    4141 
    4242  /** Current file. */ 
    43   protected BufferedReader in; 
     43  protected RandomAccessStream in; 
    4444 
    4545  /** Image width. */ 
     
    178178    if (debug) debug("EPSReader.initFile(" + id + ")"); 
    179179    super.initFile(id); 
    180     in = new BufferedReader(new FileReader(id)); 
     180    in = new RandomAccessStream(id); 
    181181    String line = in.readLine(); 
    182182    if (!line.trim().startsWith("%!PS")) { 
  • trunk/loci/formats/in/ImarisReader.java

    r2182 r2200  
    243243    if (!open) return true; // not allowed to check the file contents 
    244244    try { 
    245       DataInputStream dis = new DataInputStream(new FileInputStream(name)); 
     245      RandomAccessStream dis = new DataInputStream(new FileInputStream(name)); 
    246246      byte[] b = new byte[4]; 
    247247      dis.readFully(b); 
Note: See TracChangeset for help on using the changeset viewer.