Speeding up FileIO - Double Buffered File Copy?

Question

We are trying to speed up file copy from disk to tape, and I need a little more speed. I have tried playing with the size of the buffer, but that isn't changing much (makeing it slower if anything).
I'm trying to make a double buffered file copy and I can't figure out how to do it. I figured this would be a good place to get speed. Right now, my write is very simple:
byte buffer = new buffer[8 * 1024 * 1024];
FileInputStream in = new FileInputStream(srcFile);
while(true) {
  int amountRead = in.read(buffer);
  if (amountRead == -1) { break; }
  write(buffer, 0, length);
}So what i need to do it be able to read and write at the same time. So I was thinking that I could either make the write method a sperate thread, or some how make threaded buffers that read while the other is being writen. Has anyone tackled this problem before?
If this isn't the right way to speed up File IO, can you let me know other ideas? Thanks in advance!
Andrew

Answer 1

Once again: I wish I could claim credit for these classes, but they were in fact posted a year or so ago by someone lese. If I had the name I would give credit.
I've used these for two heavy-duty applications with never a problem.
<code>
package pipes;
import java.io.IOException;
import java.io.InputStream;
* This class is equivalent to <code>java.io.PipedInputStream</code>. In the
* interface it only adds a constructor which allows for specifying the buffer
* size. Its implementation, however, is much simpler and a lot more efficient
* than its equivalent. It doesn't rely on polling. Instead it uses proper
* synchronization with its counterpart PipedOutputStream.
* Multiple readers can read from this stream concurrently. The block asked for
* by a reader is delivered completely, or until the end of the stream if less
* is available. Other readers can't come in between.
public class PipedInputStream extends InputStream {
byte[] buffer;
boolean closed = false;
int readLaps = 0;
int readPosition = 0;
PipedOutputStream source;
int writeLaps = 0;
int writePosition = 0;
* Creates an unconnected PipedInputStream with a default buffer size.
* @exception IOException
public PipedInputStream() throws IOException {
this(null);
* Creates a PipedInputStream with a default buffer size and connects it to
* source.
* @exception IOException It was already connected.
public PipedInputStream(PipedOutputStream source) throws IOException {
this(source, 0x10000);
* Creates a PipedInputStream with buffer size <code>bufferSize</code> and
* connects it to <code>source</code>.
* @exception IOException It was already connected.
public PipedInputStream(PipedOutputStream source, int bufferSize) throws IOException {
if (source != null) {
connect(source);
buffer = new byte[bufferSize];
* Return the number of bytes of data available from this stream without blocking.
public int available() throws IOException {
// The circular buffer is inspected to see where the reader and the writer
// are located.
return writePosition > readPosition ? // The writer is in the same lap.
writePosition - readPosition : (writePosition < readPosition ? // The writer is in the next lap.
buffer.length - readPosition + 1 + writePosition :
// The writer is at the same position or a complete lap ahead.
(writeLaps > readLaps ? buffer.length : 0)
* Closes the pipe.
* @exception IOException The pipe is not connected.
public void close() throws IOException {
if (source == null) {
throw new IOException("Unconnected pipe");
synchronized (buffer) {
closed = true;
// Release any pending writers.
buffer.notifyAll();
* Connects this input stream to an output stream.
* @exception IOException The pipe is already connected.
public void connect(PipedOutputStream source) throws IOException {
if (this.source != null) {
throw new IOException("Pipe already connected");
this.source = source;
source.sink = this;
* Closes the input stream if it is open.
protected void finalize() throws Throwable {
close();
* Unsupported - does nothing.
public void mark(int readLimit) {
return;
* returns whether or not mark is supported.
public boolean markSupported() {
return false;
* reads a byte of data from the input stream.
* @return the byte read, or -1 if end-of-stream was reached.
public int read() throws IOException {
byte[] b = new byte[0];
int result = read(b);
return result == -1 ? -1 : b[0];
* Reads data from the input stream into a buffer.
* @exception IOException
public int read(byte[] b) throws IOException {
return read(b, 0, b.length);
* Reads data from the input stream into a buffer, starting at the specified offset,
* and for the length requested.
* @exception IOException The pipe is not connected.
public int read(byte[] b, int off, int len) throws IOException {
if (source == null) {
throw new IOException("Unconnected pipe");
synchronized (buffer) {
if (writePosition == readPosition && writeLaps == readLaps) {
if (closed) {
return -1;
// Wait for any writer to put something in the circular buffer.
try {
buffer.wait();
catch (InterruptedException e) {
throw new IOException(e.getMessage());
// Try again.
return read(b, off, len);
// Don't read more than the capacity indicated by len or what's available
// in the circular buffer.
int amount = Math.min(len,
(writePosition > readPosition ? writePosition : buffer.length) - readPosition);
System.arraycopy(buffer, readPosition, b, off, amount);
readPosition += amount;
if (readPosition == buffer.length) {
// A lap was completed, so go back.
readPosition = 0;
++readLaps;
// The buffer is only released when the complete desired block was
// obtained.
if (amount < len) {
int second = read(b, off + amount, len - amount);
return second == -1 ? amount : amount + second;
} else {
buffer.notifyAll();
return amount;
package pipes;
import java.io.IOException;
import java.io.OutputStream;
* This class is equivalent to java.io.PipedOutputStream. In the
* interface it only adds a constructor which allows for specifying the buffer
* size. Its implementation, however, is much simpler and a lot more efficient
* than its equivalent. It doesn't rely on polling. Instead it uses proper
* synchronization with its counterpart PipedInputStream.
* Multiple writers can write in this stream concurrently. The block written
* by a writer is put in completely. Other writers can't come in between.
public class PipedOutputStream extends OutputStream {
PipedInputStream sink;
* Creates an unconnected PipedOutputStream.
* @exception IOException
public PipedOutputStream() throws IOException {
this(null);
* Creates a PipedOutputStream with a default buffer size and connects it to
* <code>sink</code>.
* @exception IOException It was already connected.
public PipedOutputStream(PipedInputStream sink) throws IOException {
this(sink, 0x10000);
* Creates a PipedOutputStream with buffer size <code>bufferSize</code> and
* connects it to <code>sink</code>.
* @exception IOException It was already connected.
public PipedOutputStream(PipedInputStream sink, int bufferSize) throws IOException {
if (sink != null) {
connect(sink);
sink.buffer = new byte[bufferSize];
* Closes the input stream.
* @exception IOException The pipe is not connected.
public void close() throws IOException {
if (sink == null) {
throw new IOException("Unconnected pipe");
synchronized (sink.buffer) {
sink.closed = true;
flush();
* Connects the output stream to an input stream.
* @exception IOException The pipe is already connected.
public void connect(PipedInputStream sink) throws IOException {
if (this.sink != null) {
throw new IOException("Pipe already connected");
this.sink = sink;
sink.source = this;
* Closes the output stream if it is open.
protected void finalize() throws Throwable {
close();
* forces any buffered data to be written.
* @exception IOException
public void flush() throws IOException {
synchronized (sink.buffer) {
// Release all readers.
sink.buffer.notifyAll();
* writes a byte of data to the output stream.
* @exception IOException
public void write(int b) throws IOException {
write(new byte[] {(byte) b});
* Writes a buffer of data to the output stream.
* @exception IOException
public void write(byte[] b) throws IOException {
write(b, 0, b.length);
* writes data to the output stream from a buffer, starting at the named offset,
* and for the named length.
* @exception IOException The pipe is not connected or a reader has closed
* it.
public void write(byte[] b, int off, int len) throws IOException {
if (sink == null) {
throw new IOException("Unconnected pipe");
if (sink.closed) {
throw new IOException("Broken pipe");
synchronized (sink.buffer) {
     if (sink.writePosition == sink.readPosition &&
     sink.writeLaps > sink.readLaps) {
     // The circular buffer is full, so wait for some reader to consume
     // something.
     try {
     sink.buffer.wait();
     catch (InterruptedException e) {
     throw new IOException(e.getMessage());
     // Try again.
     write(b, off, len);
     return;
     // Don't write more than the capacity indicated by len or the space
     // available in the circular buffer.
     int amount = Math.min(len,
     (sink.writePosition < sink.readPosition ?
     sink.readPosition : sink.buffer.length)
- sink.writePosition);
     System.arraycopy(b, off, sink.buffer, sink.writePosition, amount);
     sink.writePosition += amount;
     if (sink.writePosition == sink.buffer.length) {
     sink.writePosition = 0;
     ++sink.writeLaps;
     // The buffer is only released when the complete desired block was
     // written.
     if (amount < len) {
     write(b, off + amount, len - amount);
     } else {
     sink.buffer.notifyAll();
</code>