Changeset 5732


Ignore:
Timestamp:
12/08/09 00:14:45 (10 years ago)
Author:
curtis
Message:

Major speed-up to initialization for new-style ND2 files.

Location:
trunk/components
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • trunk/components/bio-formats/src/loci/formats/in/NativeND2Reader.java

    r5680 r5732  
    220220    if (in.read() == -38 && in.read() == -50) { 
    221221      // newer version of ND2 - doesn't use JPEG2000 
     222      status("Searching for blocks"); 
    222223 
    223224      isJPEG = false; 
     
    234235      Vector<int[]> customDataLengths = new Vector<int[]>(); 
    235236 
    236       while (in.getFilePointer() < in.length() - 1 && 
    237         in.getFilePointer() >= 0) 
     237      // search for blocks 
     238      byte[] sigBytes = {-38, -50, -66, 10}; // 0xDACEBE0A 
     239      final String sig = new String(sigBytes); 
     240      while (in.getFilePointer() < in.length() - 1 && in.getFilePointer() >= 0) 
    238241      { 
    239         int b1 = in.read(); 
    240         int b2 = in.read(); 
    241         int b3 = in.read(); 
    242         int b4 = in.read(); 
    243         while (b1 != -38 || b2 != -50 || b3 != -66 || b4 != 10) { 
    244           if (in.getFilePointer() >= in.length() - 1) break; 
    245           b1 = b2; 
    246           b2 = b3; 
    247           b3 = b4; 
    248           b4 = in.read(); 
    249         } 
    250         if (in.getFilePointer() >= in.length() - 1) break; 
     242        in.findString(false, 1024, sig); // empirically, 1KB blocks work well 
     243        if (in.getFilePointer() > in.length() - 24) break; 
    251244 
    252245        int lenOne = in.readInt(); 
     
    255248        in.skipBytes(4); 
    256249 
     250        long fp = in.getFilePointer(); 
    257251        String blockType = in.readString(12); 
    258         long fp = in.getFilePointer() - 12; 
     252 
     253        int percent = (int) (100 * fp / in.length()); 
     254        status("Parsing block '" + blockType + "' " + percent + "%"); 
     255 
    259256        int skip = len - 12 - lenOne * 2; 
    260257        if (skip <= 0) skip += lenOne * 2; 
  • trunk/components/common/src/loci/common/RandomAccessInputStream.java

    r5729 r5732  
    227227   */ 
    228228  public String findString(String... terminators) throws IOException { 
    229     return findString(DEFAULT_BLOCK_SIZE, terminators); 
    230   } 
    231  
    232   /** 
    233    * Reads a string ending with one of the given terminating substrings, 
    234    * using the specified block size for buffering. 
    235    * 
     229    return findString(true, DEFAULT_BLOCK_SIZE, terminators); 
     230  } 
     231 
     232  /** 
     233   * Reads or skips a string ending with 
     234   * one of the given terminating substrings. 
     235   * 
     236   * @param saveString Whether to collect the string from the current file 
     237   *   pointer to the terminating bytes, and return it. If false, returns null. 
    236238   * @param terminators The strings for which to search. 
     239   * 
     240   * @throws IOException If saveString flag is set 
     241   *   and the maximum search length (512 MB) is exceeded. 
     242   * 
     243   * @return The string from the initial position through the end of the 
     244   *   terminating sequence, or through the end of the stream if no 
     245   *   terminating sequence is found, or null if saveString flag is unset. 
     246   */ 
     247  public String findString(boolean saveString, String... terminators) 
     248    throws IOException 
     249  { 
     250    return findString(saveString, DEFAULT_BLOCK_SIZE, terminators); 
     251  } 
     252 
     253  /** 
     254   * Reads a string ending with one of the given terminating 
     255   * substrings, using the specified block size for buffering. 
     256   * 
    237257   * @param blockSize The block size to use when reading bytes in chunks. 
    238    * 
    239    * @throws IOException If the maximum search length (512 MB) is exceeded. 
     258   * @param terminators The strings for which to search. 
    240259   * 
    241260   * @return The string from the initial position through the end of the 
     
    246265    throws IOException 
    247266  { 
     267    return findString(true, blockSize, terminators); 
     268  } 
     269 
     270  /** 
     271   * Reads or skips a string ending with one of the given terminating 
     272   * substrings, using the specified block size for buffering. 
     273   * 
     274   * @param saveString Whether to collect the string from the current file 
     275   *   pointer to the terminating bytes, and return it. If false, returns null. 
     276   * @param blockSize The block size to use when reading bytes in chunks. 
     277   * @param terminators The strings for which to search. 
     278   * 
     279   * @throws IOException If saveString flag is set 
     280   *   and the maximum search length (512 MB) is exceeded. 
     281   * 
     282   * @return The string from the initial position through the end of the 
     283   *   terminating sequence, or through the end of the stream if no 
     284   *   terminating sequence is found, or null if saveString flag is unset. 
     285   */ 
     286  public String findString(boolean saveString, int blockSize, 
     287    String... terminators) throws IOException 
     288  { 
    248289    StringBuilder out = new StringBuilder(); 
    249290    long startPos = getFilePointer(); 
     291    long bytesDropped = 0; 
    250292    long inputLen = length(); 
    251293    long maxLen = inputLen - startPos; 
    252     if (maxLen > MAX_SEARCH_SIZE) maxLen = MAX_SEARCH_SIZE; 
     294    boolean tooLong = saveString && maxLen > MAX_SEARCH_SIZE; 
     295    if (tooLong) maxLen = MAX_SEARCH_SIZE; 
    253296    boolean match = false; 
     297    int maxTermLen = 0; 
     298    for (String term : terminators) { 
     299      int len = term.length(); 
     300      if (len > maxTermLen) maxTermLen = len; 
     301    } 
    254302 
    255303    InputStreamReader in = new InputStreamReader(this); 
    256304    char[] buf = new char[blockSize]; 
    257     int i = 0; 
    258     while (i < maxLen) { 
    259       long pos = startPos + i; 
     305    long loc = 0; 
     306    while (loc < maxLen) { 
     307      long pos = startPos + loc; 
    260308      int num = blockSize; 
    261309      if (pos + blockSize > inputLen) num = (int) (inputLen - pos); 
     310 
     311      // if we're not saving the string, drop any old, unnecessary output 
     312      if (!saveString) { 
     313        int outLen = out.length(); 
     314        if (outLen >= maxTermLen) { 
     315          int dropIndex = outLen - maxTermLen + 1; 
     316          String last = out.substring(dropIndex, outLen); 
     317          out.setLength(0); 
     318          out.append(last); 
     319          bytesDropped += dropIndex; 
     320        } 
     321      } 
    262322 
    263323      // read block from stream 
     
    268328      out.append(buf, 0, r); 
    269329 
    270       // check output 
    271       int[] indices = new int[terminators.length]; 
     330      // check output, returning smallest possible string 
     331      int min = Integer.MAX_VALUE, tagLen = 0; 
    272332      for (int t=0; t<terminators.length; t++) { 
    273         int tagLen = terminators[t].length(); 
    274         indices[t] = out.indexOf(terminators[t], i == 0 ? 0 : i - tagLen); 
    275         if (!match) { 
    276           match = indices[t] >= 0; 
     333        int len = terminators[t].length(); 
     334        int start = (int) (loc - bytesDropped - len); 
     335        int value = out.indexOf(terminators[t], start < 0 ? 0 : start); 
     336        if (value >= 0 && value < min) { 
     337          match = true; 
     338          min = value; 
     339          tagLen = len; 
    277340        } 
    278341      } 
    279342 
    280       // return smallest possible string 
    281  
    282343      if (match) { 
    283         int min = Integer.MAX_VALUE; 
    284         int minIndex = Integer.MAX_VALUE; 
    285         for (int t=0; t<indices.length; t++) { 
    286           if (indices[t] >= 0 && indices[t] < min) { 
    287             min = indices[t]; 
    288             minIndex = t; 
    289           } 
     344        // reset stream to proper location 
     345        seek(startPos + bytesDropped + min + tagLen); 
     346 
     347        // trim output string 
     348        if (saveString) { 
     349          out.setLength(min + tagLen); 
     350          return out.toString(); 
    290351        } 
    291         int tagLen = terminators[minIndex].length(); 
    292         seek(startPos + min + tagLen); // reset stream to proper location 
    293         out.setLength(min + tagLen); // trim output 
    294         break; 
    295       } 
    296  
    297       i += r; 
    298     } 
    299  
    300     if (!match) throw new IOException("Maximum search length reached."); 
    301  
    302     return out.toString(); 
     352        else return null; 
     353      } 
     354 
     355      loc += r; 
     356    } 
     357 
     358    // no match 
     359    if (tooLong) throw new IOException("Maximum search length reached."); 
     360    return null; 
    303361  } 
    304362 
Note: See TracChangeset for help on using the changeset viewer.