How to, login into the yahoo groups/clubs,....

Question

Hi,
I would like to write a offline browser utility to follow up my yahoo groups and clubs.
I took several attempts before I wrote this mail and I discoverd that it are the redirections and cookies that cause the problems.
Does anyone know a class capable to download a URL dispite of cookies and redirections?
I tried with the swing-brower that comes as example with the JDK but he has also problems with the cookies and redirections.
Greetings
Frank

Answer 1

Hi,
I found the sollution to my problem with the help of the sniplet of madhu77 "Dealing with cookies" from nov 8, 2000 10:05 AM
I worte a class to login to the yahoo clubs, I just added the hidden parameters from HTML form, and two methods, one to retrieve the login page and a second to get a HashSet of String(URLs) to the clubs your a member of. The constructor accepts login name and password as parameters.
import java.net.*;
import java.io.*;
import java.util.*;
public class YahooClubsLogin
private String loginName="";
private String password="";
private String loginPage="";
public YahooClubsLogin(String pLoginName,String pPassword)
try
loginName=pLoginName;
password=pPassword;
String firstcookie="";
URL u = new URL("http://clubs.yahoo.com/clubs.html");
URLConnection uc = u.openConnection();
HttpURLConnection huc = (HttpURLConnection)uc;
huc.setFollowRedirects(false);
String cookie1 = huc.getHeaderField("set-cookie");
if(cookie1!=null)
{ int index = cookie1.indexOf(";");
if(index > 0)
firstcookie = cookie1.substring(0, index);
URL loginurl = new URL("http://login.yahoo.com/config/login");
URLConnection con = loginurl.openConnection();
HttpURLConnection httpcon = (HttpURLConnection)con;
httpcon.setDoInput(true);
httpcon.setDoOutput(true);
httpcon.setUseCaches(false);
httpcon.setRequestProperty("Content-Type","application/x-www-form-urlencoded");
httpcon.setRequestProperty("Cookie", firstcookie);
httpcon.setFollowRedirects(false);
OutputStream raw = httpcon.getOutputStream();
OutputStream buffered = new BufferedOutputStream(raw);
OutputStreamWriter out = new OutputStreamWriter(buffered);
String query = ".tries"+"="+URLEncoder.encode("1")+"&"+
".done"+"="+URLEncoder.encode("http://clubs.yahoo.com/clubs/")+"&"+
".src"+"="+URLEncoder.encode("grp")+"&"+
".intl"+"="+URLEncoder.encode("us")+"&"+
"login"+"="+URLEncoder.encode(loginName)+"&"+
"passwd"+"="+URLEncoder.encode(password)+"&"+
".persistent"+"="+URLEncoder.encode("Y")+"&"+
".chkP"+"="+URLEncoder.encode("Y")+"&"+
"Submit"+"="+URLEncoder.encode("Login");
out.write(query);
out.write("\r\n");
out.flush();
out.close();
for(int p=0; p<15 ; p++)
String header2 = httpcon.getHeaderField(p);
String key2 =httpcon.getHeaderFieldKey(p);
String location = httpcon.getHeaderField("location");
int j=1;
int count = 0;
String[] cookies = new String[4];
while(httpcon.getHeaderFieldKey(j)!= null)
String key =
httpcon.getHeaderFieldKey(j).toLowerCase();
if(key.indexOf("set-cookie")!=-1)
String header = httpcon.getHeaderField(j);
int ind = header.indexOf(";");
if(ind > 0)
String cookie = header.substring(0, ind);
cookies[count] = cookie;
count++;
j++;
String cookiestring = new String();
for(int k=0; k<count; k++)
cookiestring = cookiestring.concat(cookies[k]+";");
URL verifyurl = new URL(location);
URLConnection loccon = verifyurl.openConnection();
HttpURLConnection lcon = (HttpURLConnection)loccon;
lcon.setFollowRedirects(false);
lcon.setRequestProperty("Cookie", cookiestring);
for(int r=0; r<15 ; r++)
String header3 = lcon.getHeaderField(r);
String key3 = lcon.getHeaderFieldKey(r);
int code = lcon.getResponseCode();
String response =lcon.getResponseMessage();
String location2 = lcon.getHeaderField("location");
String secondcookie = new String();
String cookie2 = lcon.getHeaderField("set-cookie");
if(cookie2!=null)
int index2 = cookie2.indexOf(";");
if(index2 > 0)
secondcookie = cookie2.substring(0, index2);
secondcookie = cookiestring+";"+secondcookie;
URL myurl = new URL(location2);
URLConnection mycon = myurl.openConnection();
HttpURLConnection hmycon = (HttpURLConnection)mycon;
hmycon.setFollowRedirects(false);
hmycon.setRequestProperty("Cookie", secondcookie);
for(int s=0; s<15 ; s++)
String header4 = hmycon.getHeaderField(s);
String key4 = hmycon.getHeaderFieldKey(s);
InputStream data = hmycon.getInputStream();
InputStream in = new BufferedInputStream(data);
Reader r = new InputStreamReader(in);
StringBuffer tmpPage=new StringBuffer(20000);
int c;
while ((c = r.read())!= -1)
tmpPage.append((char)c);
loginPage = tmpPage.toString();
catch(Exception e)
String getLoginPage()
return loginPage;
HashSet getClubURLs()
HashSet urls = new HashSet();
int loginNamePos = loginPage.indexOf(loginName);
int endOfClubsTable = loginPage.indexOf("/table>",loginNamePos);
int positionOfClub = loginPage.indexOf("http://clubs.yahoo.com/clubs/",loginNamePos);
while (positionOfClub > loginNamePos && positionOfClub < endOfClubsTable )
int positionEndClub = loginPage.indexOf("\">",positionOfClub);
urls.add(loginPage.substring(positionOfClub,positionEndClub));
positionOfClub = loginPage.indexOf("http://clubs.yahoo.com/clubs/",positionEndClub);
return urls;