Get content from html page

Question

Hey guys, Im looking at accessing a webpage, downloading the content then stripping out the parts i want.
http://sunsolve.sun.com/search/document.do?assetkey=1-34-9-1
For example, I would like to be left with just the patches and their information, not the heading and intro. Where should i start?

Answer 1

here is some class that can read an URL:import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.io.OutputStream;
import java.net.URLEncoder;
public class test {
     public static void main(String args[]) {
          if(args.length!=0){
               new test(args);
          new test();
     public test() {
          this.openURL("http://www.google.com",null);
     public test(String[] args){
          int i = 0;
          while(i<args.length){
               // do something with the encoding, I am assuing utf-8
               // but the openURL method can check the header for you
               try{
                    System.out.println(new String(this.openURL(args,null),"UTF-8"));
               }catch(Exception e){
                    e.printStackTrace();
               i++;
     public byte[] openURL(String urlpath,URL u) {
          // it is VERRY importaint to read the entire response
          // if you want to connect to the same server again
          // this is because closing the inputstream does not close the socket
          // and response data from a previous request could be mixed up with the current
          InputStream is;
          OutputStream os;
          byte[] buf = new byte[1024];
          URLConnection urlc = null;
          try {
               URL a = null;
               if(u!=null){
                    a = u;
               }else{
                    a = new URL(urlpath);
               urlc = a.openConnection();
               urlc.setDoOutput(false);
          // either setDoOutput to false or Post some info
//               os = urlc.getOutputStream();
//               String name = "key="+URLEncoder.encode("value", "UTF-8");
//               os.write(name.getBytes("UTF-8"));
//               os.close();
               is = urlc.getInputStream();
               int len = 0;
               ByteArrayOutputStream bos = new ByteArrayOutputStream();
               while ((len = is.read(buf)) > 0) {
                    bos.write(buf, 0, len);
               // close the inputstream
               is.close();
               return bos.toByteArray();
          } catch (Exception e) {
               e.printStackTrace();
               try {
                    // now failing to read the inputstream does not mean the server did not send
                    // any data, here is how you can read that data, this is needed for the same
                    // reason mentioned above.
                    ((HttpURLConnection) urlc).getResponseCode();
                    InputStream es = ((HttpURLConnection) urlc).getErrorStream();
                    int ret = 0;
                    // read the response body
                    while ((ret = es.read(buf)) > 0) {
                    // close the errorstream
                    es.close();
               } catch (IOException ex) {
                    ex.printStackTrace();
                    // deal with the exception
          return new byte[0];
Here is some code to set a proxy// IF YOUR PROXY NEEDS AUTHENTICATION
//The base64encoder is part of the w3c tools
//download jigsaw and look for the base64,,, file
//http://www.google.nl/search?hl=nl&q=site%3Aw3c.org+jigsaw&lr=
//compiled it and put it in [jre home]/lib/ext
//put this jar file in the classpath when you compile
String proxyUrl = "myproxy";
String user = "myUser";
String password = "myPassword";
               URLConnection conn = url.openConnection();
               if(proxyUrl!=null){
                    System.getProperties().put( "proxySet", "true" );
                    System.getProperties().put( "proxyHost", proxyUrl );
                    System.getProperties().put( "proxyPort", "80" );
                    pwd = user + ":" + password;
                    Base64Encoder enc  = new Base64Encoder(password);
                    encodedPassword = enc.processString() ;
                    // optional
                    conn.setRequestProperty( "Proxy-Authorization", encodedPassword );
               // start opening output or inputstream on the connection