DataOutputStream appears to be buffering and then overflowing?? 10 Duke Pts

Question

Hi folks,
Hope someone can help. I am trying to put together a file upload applet (yes I know....security this and signing that and so on...). It "works" (erm...kind of) using the JFileChooser and JProgressMonitorInputStream to carry out an HTTP post to a server side CGI script.
I have it "working" in so far as with small files it does actually complete although usually some time after the progress bar shows it completed.
With larger files though the progress bar still rushes through too quickly but then it crashes with an OutOfMemory exception (at around 40MB).
It looks to me almost as if the DataOutputStream is buffering and returning (not blocking) before the data has actually been sent. I have tried including a .flush() after the write but it made no difference.
Why does the DataOutputStream return from write before the data has actually been sent?
If anyone can help in any way it will be massively appreciated! I have included code below its not neat since I have been bashing it around trying to make it work.
What I am looking for is to have the progress bar show the progress as the contents of the file is uploaded to the server via the HTTP post.
package videoupload;
import java.awt.*;
import java.awt.event.*;
import javax.swing.*;
import java.net.*;
import java.io.*;
public class VideoUpload extends JApplet implements ActionListener
  class HTTPPoster extends SwingWorker {
    String sBoundary="2db8c22f75474a58cd13fa2d3425017015d392ce";
    URL u;
    public URLConnection postConnect(URL urlTarget)
      URLConnection c = null;
      try {
        c = urlTarget.openConnection();
        // post multipart data
        c.setDoOutput(true);
        c.setDoInput(true);
        c.setUseCaches(false);
      } catch (Exception ex)
         ex.printStackTrace();
      return c;
    public void postHeaders(URLConnection pCon)
      // set some request headers
      pCon.setRequestProperty("Connection", "Keep-Alive");
      pCon.setRequestProperty("HTTP_REFERER", "http://www.<THEWEBSITE>.com/");
      pCon.setRequestProperty("Content-Type",
                              "multipart/form-data; boundary="+sBoundary);
    public DataOutputStream getFormBodyOutput(URLConnection pCon)
      DataOutputStream dos = null;
      try
        dos = new DataOutputStream(pCon.getOutputStream());
        sendBoundary(dos);
      } catch (Exception ex)
        ex.printStackTrace();
      return dos;
    public void sendBoundary(DataOutputStream dos)
      try
        dos.writeBytes("--"+sBoundary+"\r\n");
      } catch (Exception ex)
        ex.printStackTrace();
    public void postFile(DataOutputStream dos, String name, File f)
      try
        dos.writeBytes(
            "Content-Disposition: form-data; name=\"" + name +
            "\"; filename=\"" + f.getName() +
            "\"\r\nContent-Type: application/octet-stream\r\n\r\n");
        System.out.println("Opening - "+f.getAbsoluteFile());
        FileInputStream fis = new FileInputStream(f.getAbsoluteFile());
        ProgressMonitorInputStream pmis = new ProgressMonitorInputStream(
            content, f.getName(), fis);
        ProgressMonitor pm = pmis.getProgressMonitor();
        pm.setMillisToDecideToPopup(500);
        byte[] bt = new byte[1024];
        int cnt = pmis.read(bt);
        while (cnt == bt.length)
          dos.write(bt, 0, cnt);
          cnt = pmis.read(bt);
        // send the last bit to the server
        dos.write(bt, 0, cnt);
        // now close the file and let the web server know this is the end of this form part
        pmis.close();
        fis.close();
        sendBoundary(dos);
      } catch (Exception ex)
        ex.printStackTrace();
    public void endPost(DataOutputStream dos)
      try
        dos.writeBytes(
            "\r\n--" + sBoundary + "--\r\n\r\n");
        dos.flush();
        dos.close();
      } catch (Exception ex)
        ex.printStackTrace();
    public void getResponse(URLConnection pCon)
      try
        DataInputStream dis =
            new DataInputStream(
                new BufferedInputStream(pCon.getInputStream()));
        String sIn = dis.readLine();
        while (sIn != null)
          if (sIn != null)
            System.out.println(sIn);
          sIn = dis.readLine();
      } catch (Exception ex)
        ex.printStackTrace();
    public Object construct()
      try
          u = new URL("http://www.<THEWEBSITE>.com/cgi-bin/upload.cgi");
          // System.out's here for debugging only...
          System.out.println("Connect");
          URLConnection pCon = postConnect(u);
          System.out.println("Send headers");
          postHeaders(pCon);
          System.out.println("Begin body");
          DataOutputStream dos = getFormBodyOutput(pCon);
          System.out.println("Send file");
          this.postFile(dos, "file", f);
          System.out.println("End body");
          endPost(dos);
          System.out.println("Get response");
          getResponse(pCon);
          System.out.println("Done");
        catch (Exception ex)
          ex.printStackTrace();
      return null;
  Container content;
  File f;
  protected boolean isStandalone = false;
  protected BorderLayout borderLayout1 = new BorderLayout();
  String uploadPath;
  //Get a parameter value
  public String getParameter(String key, String def)
    return isStandalone ? System.getProperty(key, def) :
        (getParameter(key) != null ? getParameter(key) : def);
  //Construct the applet
  public VideoUpload()
  //Initialize the applet
  public void init()
    try
      UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());
    catch (Exception e)
      System.err.println("Error setting native LAF: " + e);
    content = getContentPane();
    content.setBackground(Color.white);
    content.setLayout(new FlowLayout());
    // Come back to this later
    JTextField txtFileName = new JTextField();
    txtFileName.addActionListener(this);
    content.add(txtFileName);
    JButton btnUpload = new JButton("Upload");
    btnUpload.setMnemonic('u');
    btnUpload.addActionListener(this);
    content.add(btnUpload);
    //Create a file chooser
    final JFileChooser fc = new JFileChooser();
    int returnVal = fc.showOpenDialog(content);
    f=fc.getSelectedFile();
    try
      uploadPath = this.getParameter("uploadPath", "/");
    catch (Exception e)
      e.printStackTrace();
  public void actionPerformed(ActionEvent e){
    HTTPPoster worker = new HTTPPoster();
    worker.start();
  //Start the applet
  public void start()
  //Stop the applet
  public void stop()
  //Destroy the applet
  public void destroy()
  //Get Applet information
  public String getAppletInfo()
    return "Applet Information";
  //Get parameter info
  public String[][] getParameterInfo()
    java.lang.String[][] pinfo =
        {"uploadPath", "String", ""},
    return pinfo;
  //Main method
  public static void main(String[] args)
    VideoUpload applet = new VideoUpload();
    applet.isStandalone = true;
    Frame frame;
    frame = new Frame();
    frame.setTitle("Applet Frame");
    frame.add(applet, BorderLayout.CENTER);
    applet.init();
    applet.start();
    frame.setSize(400, 320);
    Dimension d = Toolkit.getDefaultToolkit().getScreenSize();
    frame.setLocation( (d.width - frame.getSize().width) / 2,
                      (d.height - frame.getSize().height) / 2);
    frame.setVisible(true);
}

Answer 1

Well after much ploughing through internet I have found much to my dismay that URLConnection is not as great as it may appear as it does buffer all output prior to sending.
This is fine for everything but file transfers, especially large ones as memory can be an issue.
Answered own question, doh!