Coding Problem 6 Now Defined in Scripting Languages/Bioinformatics WIKI

Question

In the "Scripting Languages and Bioinformatics" WIKI, I have completed a new page here:
https://wiki.sdn.sap.com/wiki/display/EmTech/Bio-InformaticBasicsPartII-AlignmentToolsforRelatingProteinGenestoProteinPrimaryStructures
and also defined Coding Problem 6 at the end of this page.
This problem involves scripting an interactive session with a web site and parsing what the web site returns at various points duing the session.
Am looking foward to a script that solves this problem, if anyone feels like spending the time to do it.

Answer 1

<?
// sequence to search ---------------------------------------------
$bquery      = "MTQIIKIDPLNPEIDKIKIAADVIRNGGTVAFPTETVYGLGANAFDG";
$bquery     .= "NACLKIFQAKNRPVDNPLIVHIADFNQLFEVAKDIPDKVLEIAQIVW";
$bquery     .= "PGPLTFVLKKTERVPKEVTAGLDTVAVRMPAHPIALQLIRESGVPIA";
$bquery     .= "APSANLATRPSPTKAEDVIVDLNGRVDVIIDGGHTFFGVESTIINVT";
$bquery     .= "VEPPVLLRPGPFTIEELKKLFGEIVIPEFAQGKKEAEIALAPGMKYK";
$bquery     .= "HYAPNTRLLLVENRNIFKDVVSLLSKKYKVALLIPKELSKEFEGLQQ";
$bquery     .= "IILGSDENLYEVARNLFDSFRELDKLNVDLGIMIGFPERGIGFAIMN";
$bquery     .= "RARKASGFSIIKAISDVYKYVNI";
// url for form1 --------------------------------------------------
$url = "http://blast.ncbi.nlm.nih.gov:80/Blast.cgi";
// parameters for form1 -------------------------------------------
$bdb         = "protein";
$bgeneticcode= "1";
$bquery_from = "";
$bquery_to   = "";
$bjobtitle   = "ACW-" . time();
$bdatabase   = "nr";
$bblastprog  = "tblastn";
$bpagetype   = "BlastSearch";
$pf  = "";
$pf .= "QUERY=" . $bquery;
$pf .= "&db=" . $bdb;
$pf .= "&QUERY_FROM=" . $bquery_from;
$pf .= "&QUERY_TO=" . $bquery_to;
$pf .= "&JOB_TITLE=" . $bjobtitle;
$pf .= "&DATABASE=" . $bdatabase;
$pf .= "&BLAST_PROGRAMS=" . $bblastprog;
$pf .= "&PAGE_TYPE=" . $bpagetype;
$pf .= "&GENETIC_CODE=1";
$pf .= "&DBTYPE=";
$pf .= "&EQ_MENU=";
$pf .= "&EQ_TEXT=";
$pf .= "&NEWWIN=";
$pf .= "&MATCH_SCORES=";
$pf .= "&MAX_NUM_SEQ=100";
$pf .= "&EXPECT=10";
$pf .= "&WORD_SIZE=3";
$pf .= "&MATRIX_NAME=BLOSUM62";
$pf .= "&GAPCOSTS=11 1";
$pf .= "&COMPOSITION_BASED_STATISTICS=2";
$pf .= "&REPEATS=";
$pf .= "&FILTER=";
$pf .= "&LCASE_MASK=";
$pf .= "&TEMPLATE_LENGTH=";
$pf .= "&TEMPLATE_TYPE=";
$pf .= "&I_THRESH=0.005";
$pf .= "&CLIENT=web";
$pf .= "&SERVICE=plain";
$pf .= "&CMD=request";
$pf .= "&PAGE=Translations";
$pf .= "&PROGRAM=tblastn";
$pf .= "&MEGABLAST=";
$pf .= "&RUN_PSIBLAST=";
$pf .= "&SELECTED_PROG_TYPE=tblastn";
$pf .= "&SAVED_SEARCH=true";
$pf .= "&BLAST_SPEC=";
$pf .= "&QUERY_BELIEVE_DEFLINE=";
$pf .= "&DB_DIR_PREFIX=";
$pf .= "&SHOW_OVERVIEW=true";
$pf .= "&SHOW_LINKOUT=true";
$pf .= "&GET_SEQUENCE=true";
$pf .= "&FORMAT_OBJECT=Alignment";
$pf .= "&FORMAT_TYPE=HTML";
$pf .= "&ALIGNMENT_VIEW=Pairwise";
$pf .= "&MASK_CHAR=2";
$pf .= "&MASK_COLOR=1";
$pf .= "&DESCRIPTIONS=100";
$pf .= "&ALIGNMENTS=100";
$pf .= "&NEW_VIEW=true";
$pf .= "&OLD_BLAST=true";
$pf .= "&NCBI_GI=false";
$pf .= "&SHOW_CDS_FEATURE=false";
$pf .= "&NUM_OVERVIEW=100";
$pf .= "&FORMAT_EQ_TEXT=";
$pf .= "&FORMAT_ORGANISM=";
$pf .= "&EXPECT_LOW=";
$pf .= "&EXPECT_HIGH=true";
$pf .= "&QUERY_INDEX=0";
// define the HTTP connection -----------------------------
$ch = curl_init();
curl_setopt($ch,CURLOPT_URL,$url);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch,CURLOPT_POSTFIELDS, $pf);
curl_setopt($ch,CURLOPT_COOKIEJAR,
dirname(__FILE__).'/cookie.txt');
curl_setopt($ch,CURLOPT_FOLLOWLOCATION,1);
curl_setopt($ch, CURLOPT_HEADER , 1);
curl_setopt($ch, CURLOPT_RETURNTRANSFER,1);
// call it ------------------------------------------------
$my_result = curl_exec($ch);
// get the request ID; the query takes some time and this ID
// allows to identify the original request
$query = "/<tr><td>Request ID<\/td><td> <b>([0-9A-Z]*)<\/b><\/td><\/tr>/";
preg_match($query, $my_result, $request_id);
// query for the result page every 2 seconds...when the result
// is ready, we recognize it by its signature
$not_ready = true;
$result_url = "http://blast.ncbi.nlm.nih.gov/Blast.cgi?CMD=Get&VIEW_RESULTS=FromRes&RID=" . $request_id[1];
while($not_ready){
  $temp_page = null;
  curl_setopt($ch,CURLOPT_URL,$result_url);
  curl_setopt($ch, CURLOPT_POST, 0);
  $my_result = curl_exec($ch);
  $query = "/This page will be automatically updated in <b>(\d*)<\/b> seconds/";
  preg_match($query, $my_result, $temp_page);
  if(isset($temp_page[1])) {  sleep(2); }
  else { $not_ready = false; }
// parse the results page; split it into result fragments first --------
$pat = "/";
$pat .= "><input type=\"checkbox\" name=\"getSeqGi\"([^\n]*)\n";
$pat .= "Length=\d*\n\n";
$pat .= "[\w\s]*:\n";
$pat .= "(\s*<a[^>]*>[^<]*<\/a>\n)*\n";
$pat .= "([^\n]*\n){3}\n";
$pat .= "(([^\n]*\n){3}\n[^n])*";
$pat .= "/";
preg_match_all($pat, $my_result, $fragments);
// 'fine parse the result fragments and get some important parameters -----
foreach($fragments[0] as $fragment){
  $query = "/<a href=\"http:\/\/(www\.ncbi\.nlm\.nih\.gov\/entrez\/query\.fcgi\?";
  $query .= "[^>]*)>([^<]*)<\/a><a[^>]*>[^<]*<\/a>\s*<a[^>]*><img[^>]*><\/a>([^\n]*)/";
  preg_match($query, $fragment, $u);
  $query  = "/\s*Score\s*=\s*(\d*) bits\s*\(\d*\),\s*Expect\s*=\s*([^,]*),";
  $query .= "[^\n]*\n\s*Identities\s*=\s*\d*\/\d*\s*\((\d*%)\)[^\n]*\n\s*Frame\s*=\s*([^\n]*)\n/";
  preg_match($query, $fragment, $u1);
  preg_match_all("/Sbjct\s*(\d*)\s*[A-Z\-]*\s*(\d*)\n/", $fragment, $positions);
  $fr["url"]        = $u[1];
  $fr["shname"]     = $u[2];
  $fr["name"]       = $u[3];
  $fr["score"]      = $u1[1];
  $fr["expect"]     = $u1[2];
  $fr["identities"] = $u1[3];
  $fr["frame"]      = $u1[4];
// find out if we have to reverse the strand direction or not; adjust
// lower and upper limit
  if ($fr["frame"] >= 0) {
    $fr["high"] =  $positions[2][count($positions)-1]; $fr["low"] = $positions[1][0]; $fr["reverse"] = ""; }
  else {
    $fr["high"] =  $positions[1][0]; $fr["low"] = $positions[2][count($positions)-1]; $fr["reverse"] = true;}
  $frx[] = $fr;
// we call the final details form for the top ranked result
// here we could loop over several results
curl_setopt($ch,CURLOPT_URL,$frx[0]["url"]);
curl_setopt($ch, CURLOPT_POST, 0);
$my_result = curl_exec($ch);
// parse the final details form to get the necessary parameters to execute
// the final details form
preg_match("/<input name=\"WebEnv\" type=\"hidden\" value=\"([^\"]*)\"/",
           $my_result, $u5);
preg_match("/<input name=\"query_key\" type=\"hidden\" value=\"([^\"]*)\"/",
           $my_result, $u6);
preg_match("/<input name=\"db\" type=\"hidden\" value=\"([^\"]*)\"/",
           $my_result, $u7);
preg_match("/<input name=\"qty\" type=\"hidden\" value=\"([^\"]*)\"/",
           $my_result, $u8);
preg_match("/<input name=\"c_start\" type=\"hidden\" value=\"([^\"]*)\"/",
           $my_result, $u9);
preg_match(
  "/<select name=\"dopt\".*<option value=\"([^\"]*)\" selected=\"1\">/",
  $my_result, $u10);
preg_match(
  "/<select name=\"dispmax\".*<option value=\"([^\"]*)\" selected=\"1\">/",
  $my_result, $u11);
preg_match(
  "/<select name=\"sendto\".*<option value=\"([^\"]*)\" selected=\"1\">/",
  $my_result, $u12);
preg_match(
  "/<input type=\"hidden\" id=\"([^\"]*)\" name=\"fmt_mask\".*value=\"([^\"]*)\"/",
  $my_result, $u13);
preg_match(
  "/<input type=\"checkbox\" id=\"truncate\" name=\"truncate\" value=\"([^\"]*)\"\/>/",
  $my_result, $u14);
// fill in the necessary parameters parsed out now and earlier
$of  = "WebEnv=" . $u5[1];
$of .= "&query_key=" . $u6[1];
$of .= "&db=" . $u7[1];
$of .= "&qty=" . $u8[1];
$of .= "&c_start=" . $u9[1];
$of .= "&dopt=" . $u10[1];
$of .= "&dispmax=" . $u11[1];
$of .= "&sendto=" . $u12[1];
$of .= "&fmt_mask=" . $u13[1];
$of .= "&truncate=" . $u14[1];
$of .= "&less_feat=";
$of .= "&from=" . $fr["low"];
$of .= "&to=" . $fr["high"];
$of .= "&strand=" . $fr["reverse"];
$of .= "&extrafeatpresent=1";
$of .= "&ef_SNP=1";
$of .= "&ef_CDD=8";
$of .= "&ef_MGC=16";
$of .= "&ef_HPRD=32";
$of .= "&ef_STS=64";
$of .= "&ef_tRNA=128";
$of .= "&ef_microRNA=256";
$of .= "&ef_Exon=512";
$of .= "&submit=Refresh";
$ourl = "http://www.ncbi.nlm.nih.gov/entrez/viewer.fcgi?" . $of;
// execute the final details form
curl_setopt($ch,CURLOPT_URL,$ourl);
curl_setopt($ch, CURLOPT_POST, 0);
curl_setopt($ch,CURLOPT_POSTFIELDS, $of);
curl_setopt($ch,CURLOPT_FOLLOWLOCATION,1);
curl_setopt($ch, CURLOPT_HEADER , 1);
curl_setopt($ch, CURLOPT_RETURNTRANSFER,1);
$my_result = curl_exec($ch);
// parse the result page and get the final ORIGIN snippet
preg_match("/ORIGIN\s*\n\s*<a name=\"[^\"]*\"><\/a>\s*([^\/]*)\//", $my_result, $u99);
// output the final result ----------------------------------
echo "\n" . preg_replace("/\s{2,}/", "\n", preg_replace("/\d+\s/", "", $u99[1]));
?>
Edited by: Anton Wenzelhuemer on Aug 6, 2008 8:08 PM (Re-Formatted)