Mercurial > hg > ywww

changeset 35:86f79bc1d142

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
refactor to split out shareable fn to actually do an Amazon API request
author Charlie Root
date Fri, 04 Jan 2019 12:54:12 -0500
parents 5cae8d572998
children e570b199108b
files xml/doAmazonRequest.inc xml/getAmazonInfo.php
diffstat 2 files changed, 228 insertions(+), 229 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/xml/doAmazonRequest.inc	Fri Jan 04 12:54:12 2019 -0500
@@ -0,0 +1,77 @@
+<?php
+
+function doAmazonRequest($ext, $parameters, $try)
+{
+  $public_key ="AKIAIHTNWC7L6LOUY4LQ";
+  $private_key="zWQlIzndJDtXNfxEXH7K7YR7hzv3u77lOcqfqPde";
+  $requestURI = $_SERVER['REQUEST_URI'];
+  $requestIP = $_SERVER['REMOTE_ADDR'];
+  $file_data=$ext;
+  ksort($parameters);
+  foreach ($parameters as $i=>$d) {
+    $file_data.='&'.$i.'='.$d;
+  }
+  //file_put_contents('/var/ywww/debug/phpDebug',
+  //		"nai1: $requestIP $file_data\n",FILE_APPEND);
+  $gotit=0;
+  $url=aws_signed_request($ext,$parameters,$public_key,$private_key);  
+  $crl = curl_init();
+  $timeout = 5;
+  curl_setopt ($crl, CURLOPT_URL,$url);
+  curl_setopt ($crl, CURLOPT_ENCODING , "gzip"); 
+  curl_setopt ($crl, CURLOPT_RETURNTRANSFER, 1);
+  curl_setopt ($crl, CURLOPT_CONNECTTIMEOUT, $timeout);
+  for ($i=1; $i<=$try; $i++) {
+    $semaphore = new SyncSemaphore("Amazon");
+    $gotit = $semaphore->lock(1000);
+    if (!$gotit) {
+      file_put_contents('/var/ywww/debug/phpDebug',
+			"Got it: ".(int)$gotit." $requestIP 1\n",FILE_APPEND);
+      $gotit=$semaphore->lock(1000);
+      file_put_contents('/var/ywww/debug/phpDebug',
+			"Got it: ".(int)$gotit." $requestIP 1a\n",FILE_APPEND);
+    }
+    $output = curl_exec($crl);
+    //file_put_contents('/var/ywww/debug/phpDebug',
+    //	      "nai2: ".strlen($output)."\n",FILE_APPEND);
+
+    curl_close($crl);
+    usleep(500000);
+    if ($gotit) {
+      $semaphore->unlock();
+    }
+    else {
+      file_put_contents('/var/ywww/debug/phpDebug',
+			"W/o lock for $requestIP 1b\n",FILE_APPEND);
+    }
+    $mm=array();
+    $xml = new SimpleXMLElement($output);	
+    if (preg_match("/<Error>/",$output,$mm)) {
+      $resName=$xml->getName();
+      $code=$xml->Error->Code;
+      if ($code) {
+	$message=$xml->Error->Message;
+      }
+      else {
+	$code=$xml->Items->Request->Errors->Error->Code;
+	$message=$xml->Items->Request->Errors->Error->Message;
+      }
+      file_put_contents('/var/ywww/debug/phpDebug',"Losing $i: ".$mm[0].", $resName, $code, $requestIP, $requestURI\n",FILE_APPEND);
+      if ($code!='RequestThrottled') {
+	file_put_contents('/var/ywww/debug/phpDebug',"message: $message\n".
+			  print_r($parameters,TRUE)."\n",FILE_APPEND);
+	if ($code=="") {
+	  file_put_contents('/var/ywww/debug/phpDebug',"error elt:\n$output\n",FILE_APPEND);
+	}
+	throw new Exception($code);
+      }
+    }
+    else {
+      return $xml;
+    }
+    usleep(200000); // Try to reduce throttling until we get a 
+    // principled solution in place
+  }
+  throw new Exception("ThrottledRepeatedly");
+}
+?>
--- a/xml/getAmazonInfo.php	Fri Jan 04 12:53:10 2019 -0500
+++ b/xml/getAmazonInfo.php	Fri Jan 04 12:54:12 2019 -0500
@@ -1,5 +1,7 @@
 <?php
 
+include_once "doAmazonRequest.inc";
+
 function getAmazonDet($isbn,$go,$localeIn)
 {
   $Adefault=array(
@@ -40,32 +42,18 @@
     break;
   };		
 
-  for ($i=1; $i<=3; $i++)  {
-    try {  
-      return getAmazonDetRemote($go,$language,$locale,$loc,
-				$page,$operation,$searchparameter,
-				$searchparameterdata,
-				$show_array,$show_url,$show_xml);
-    }
-    catch (Exception $e) {
-      $code=$e->getMessage();
-      file_put_contents('/var/ywww/debug/phpDebug',
-			"Bang: $code $i\n",FILE_APPEND);
-      if ($code=='RequestThrottled') {
-	usleep(200000); // Try to reduce throttling until we get a 
-	// principled solution in place
-	$bail=False;
-      }
-      else {
-	$bail=True;
-      }
-    }
-    if ($bail) { break; }
+  try {  
+    return getAmazonDetRemote($go,$language,$locale,$loc,
+			      $page,$operation,$searchparameter,
+			      $searchparameterdata,
+			      $show_array,$show_url,$show_xml);
   }
-  return getAmazonDetLocal($go,$language,$loc,
-			   $page,$operation,$searchparameter,
-			   $searchparameterdata,
-			   $show_array,$show_url,$show_xml);
+  catch (Exception $e) {
+    return getAmazonDetLocal($go,$language,$loc,
+			     $page,$operation,$searchparameter,
+			     $searchparameterdata,
+			     $show_array,$show_url,$show_xml);
+  }
 }
 
 function getAmazonDetRemote($go,$language,$locale,$loc,
@@ -118,8 +106,6 @@
 
   //if(go != 1)
   //include "aws_signed_request.php";  
-  $public_key ="AKIAIHTNWC7L6LOUY4LQ";
-  $private_key="zWQlIzndJDtXNfxEXH7K7YR7hzv3u77lOcqfqPde";
   //this is the data that is used to form the request for AWS
   //this is the part that is search specific
   $parameters=array(
@@ -140,86 +126,24 @@
     //'SearchIndex'     =>$searchindex      ,  //Books for example.
     $parameters['SearchIndex']=$searchindex;
   }
-  $requestURI = $_SERVER['REQUEST_URI'];
-  $requestIP = $_SERVER['REMOTE_ADDR'];
-  $ext=$Aserver[$locale]['ext'];  
-  $file_data=$ext;
-  ksort($parameters);
-  foreach ($parameters as $i=>$d) {
-    $file_data.='&'.$i.'='.$d;
-  }
-  $gotit=0;
-  $url=aws_signed_request($ext,$parameters,$public_key,$private_key);  
-  $crl = curl_init();
-  $timeout = 5;
-  curl_setopt ($crl, CURLOPT_URL,$url);
-  curl_setopt ($crl, CURLOPT_ENCODING , "gzip"); 
-  curl_setopt ($crl, CURLOPT_RETURNTRANSFER, 1);
-  curl_setopt ($crl, CURLOPT_CONNECTTIMEOUT, $timeout);
-  $semaphore = new SyncSemaphore("Amazon");
-  $gotit = $semaphore->lock(1000);
-  if (!$gotit) {
-    file_put_contents('/var/ywww/debug/phpDebug',
-		      "Got it: ".(int)$gotit." $requestIP 1\n",FILE_APPEND);
-    $gotit=$semaphore->lock(1000);
-    file_put_contents('/var/ywww/debug/phpDebug',
-		      "Got it: ".(int)$gotit." $requestIP 1a\n",FILE_APPEND);
-  }
-  $output = curl_exec($crl);
-  curl_close($crl);
-  usleep(500000);
-  if ($gotit) {
-    $semaphore->unlock();
-  }
-  else {
-    file_put_contents('/var/ywww/debug/phpDebug',
-		      "W/o lock for $requestIP 1b\n",FILE_APPEND);
-  }
-  $review = "";
-  $review1 = "";
-  $review2 = "";
-  $review3 = "";
-  // HST added this
-  $mm=array();
-  if (preg_match("/<Error>/",$output,$mm)) {
-    $xml = new SimpleXMLElement($output);	
-    $resName=$xml->getName();
-    $code=$xml->Error->Code;
-    if (!$code) {
-      $code=$xml->Items->Request->Errors->Error->Code;
-    }
-    file_put_contents('/var/ywww/debug/phpDebug',"Losing: ".$mm[0].", $resName, $code, $requestIP, $requestURI\n",FILE_APPEND);
-    if ($code!='RequestThrottled') {
-      file_put_contents('/var/ywww/debug/phpDebug',"parms: $go\nurl: $url\n".
-			print_r($parameters,TRUE)."\n",FILE_APPEND);
-      if ($code=="") {
-	file_put_contents('/var/ywww/debug/phpDebug',"error elt:\n$output\n",FILE_APPEND);
-      }
-    }
-    throw new Exception($code);
-  }
-  else {
-    $xml = new SimpleXMLElement($output);	
-    set_error_handler(function () {
-	global $output;
-	file_put_contents('/var/ywww/debug/phpDebug',
-			  "Caught one?: ".$searchparameterdata,
-			  FILE_APPEND);
-	file_put_contents('/var/ywww/debug/phpDebug',
-			  print_r($output, TRUE)."\n",
-			  FILE_APPEND);
-      } );
-    $review = $xml->Items->Item->CustomerReviews->IFrameURL;
-    // The above is failing repeatedly -- 
-    //PHP Notice: Trying to get property of non-object in
-    // /var/ywww/xml/getAmazonInfo.php on line [109]
-    // See the dumped structure at the end of this file for the
-    // cause
-    restore_error_handler();
-    //echo $review;
-  }
+  $xml=doAmazonRequest($Aserver[$locale]['ext'],$parameters,3);  // may throw exception
+  set_error_handler(function () {
+      global $output;
+      file_put_contents('/var/ywww/debug/phpDebug',
+			"Caught one?: ".$searchparameterdata,
+			FILE_APPEND);
+      file_put_contents('/var/ywww/debug/phpDebug',
+			print_r($output, TRUE)."\n",
+			FILE_APPEND);
+    } );
+  $review = $xml->Items->Item->CustomerReviews->IFrameURL;
+  restore_error_handler();
+  //echo $review;
   if ($review != "")
     {
+      $review1 = "";
+      $review2 = "";
+      $review3 = "";
       $text = @file_get_contents($review . "&truncate=300");
       $removeTop1 = preg_replace('~<div class="crIFrameHeaderLeftColumn">(.*?)<div class="crIFrameHeaderHistogram">~si', '', $text);
       $removeTop2 = preg_replace('~<div class="crIFrameHeaderHistogram">(.*?)<div class="crIframeReviewList">~si', '', $text);
@@ -310,158 +234,156 @@
     if ($item && $item->ItemAttributes && $item->ItemAttributes->Title) {
       $title = $item->ItemAttributes->Title; }
     else { $title = "[no title]"; };
-    file_put_contents('/var/ywww/debug/phpDebug',"win: |$errorCode| ".
-		      $title."\n",
+    file_put_contents('/var/ywww/debug/phpDebug',"win: $title\n",
 		      FILE_APPEND);
     return $output;
   }
-  else
-    {
-      if ($xml->Items->Item) {
-	$title = $xml->Items->Item[0]->ItemAttributes->Title;
-	$author = $xml->Items->Item[0]->ItemAttributes->Author;
-	$binding = $xml->Items->Item[0]->ItemAttributes->Binding;
-	$dewey = $xml->Items->Item[0]->ItemAttributes->DeweyDecimalNumber;
-	if($dewey == "")
-	  $dewey = "null";
-	$imageURL = $xml->Items->Item[0]->MediumImage->URL;
-	$salesRank = $xml->Items->Item[0]->SalesRank;
-	$pubDate = $xml->Items->Item[0]->ItemAttributes->PublicationDate;
-	if (strlen($pubDate)==4) { $pubDate=$pubDate."-01-01";}
-	if (strlen($pubDate)==7) { $pubDate=$pubDate."-01";}
-	if (strlen($pubDate)==0) {
-	  $pubDate="null";
-	}
-	else {
-	  $pubDate="\"$pubDate\"";
-	}
-	$publisher = $xml->Items->Item[0]->ItemAttributes->Publisher;
+  else {
+    if ($xml->Items->Item) {
+      $title = $xml->Items->Item[0]->ItemAttributes->Title;
+      $author = $xml->Items->Item[0]->ItemAttributes->Author;
+      $binding = $xml->Items->Item[0]->ItemAttributes->Binding;
+      $dewey = $xml->Items->Item[0]->ItemAttributes->DeweyDecimalNumber;
+      if($dewey == "")
+	$dewey = "null";
+      $imageURL = $xml->Items->Item[0]->MediumImage->URL;
+      $salesRank = $xml->Items->Item[0]->SalesRank;
+      $pubDate = $xml->Items->Item[0]->ItemAttributes->PublicationDate;
+      if (strlen($pubDate)==4) { $pubDate=$pubDate."-01-01";}
+      if (strlen($pubDate)==7) { $pubDate=$pubDate."-01";}
+      if (strlen($pubDate)==0) {
+	$pubDate="null";
       }
       else {
-	$title = $salesRank = "";
-	$dewey = "null";
+	$pubDate="\"$pubDate\"";
       }
+      $publisher = $xml->Items->Item[0]->ItemAttributes->Publisher;
+    }
+    else {
+      $title = $salesRank = "";
+      $dewey = "null";
+    }
 			
-      $genreID = "";
-      $genre = "";		
-      $genArr = array();
+    $genreID = "";
+    $genre = "";		
+    $genArr = array();
 
-      if ($xml->Items->Item[0] && $xml->Items->Item[0]->BrowseNodes) {
-	for($i=0;$i<sizeof($xml->Items->Item->BrowseNodes->BrowseNode);$i++){	
-	  //sexy recursive function
-	  findGenre($xml->Items->Item->BrowseNodes->BrowseNode[$i], $genreID, $genre);
-				
-	  if($genre != "")
-	    $genArr[strval($genreID)] = strval($genre);
-	  //$genArr[$i] = array(strval($genreID) => strval($genre));
-
-	  //echo $genre;
-	  //echo $genreID;
+    if ($xml->Items->Item[0] && $xml->Items->Item[0]->BrowseNodes) {
+      for($i=0;$i<sizeof($xml->Items->Item->BrowseNodes->BrowseNode);$i++){	
+	//sexy recursive function
+	findGenre($xml->Items->Item->BrowseNodes->BrowseNode[$i], $genreID, $genre);
 				
-	  $genre = "";
-	  $genreID = "";
-	}
+	if($genre != "")
+	  $genArr[strval($genreID)] = strval($genre);
+	//$genArr[$i] = array(strval($genreID) => strval($genre));
+
+	//echo $genre;
+	//echo $genreID;
+				
+	$genre = "";
+	$genreID = "";
       }
-			
-      $g1 = "null";
-      $g2 = "null";
-      $g3 = "null";
-      $loop = 1;
+    }
 			
-      foreach ($genArr as $key => $value) {
-	//echo "$key => $value";
-	if ($key>2047) {
-	  //HST added
-	  break;
-	}
-	$queryG = "CALL b_addBrowseNode($key,\"$value\")";	//add the name value pair for genre to new table	
-	//echo $queryG;
-	include "../../private/db.php";
-	$resG = mysqli_query($link, $queryG);
-	mysqli_close($link);
+    $g1 = "null";
+    $g2 = "null";
+    $g3 = "null";
+    $loop = 1;
+			
+    foreach ($genArr as $key => $value) {
+      //echo "$key => $value";
+      if ($key>2047) {
+	//HST added
+	break;
+      }
+      $queryG = "CALL b_addBrowseNode($key,\"$value\")";	//add the name value pair for genre to new table	
+      //echo $queryG;
+      include "../../private/db.php";
+      $resG = mysqli_query($link, $queryG);
+      mysqli_close($link);
 				
-	switch ($loop) {
-	case 1:
-	  $g1 = $key;
-	  break;
-	case 2:
-	  $g2 = $key;
-	  break;
-	case 3:
-	  $g3 = $key;
-	  break;
-	}
+      switch ($loop) {
+      case 1:
+	$g1 = $key;
+	break;
+      case 2:
+	$g2 = $key;
+	break;
+      case 3:
+	$g3 = $key;
+	break;
+      }
 				
-	$loop++;
-      }
+      $loop++;
+    }
 			
-      if($salesRank == "")
-	$salesRank = "null";
+    if($salesRank == "")
+      $salesRank = "null";
 			
-      if($title != "")
-	{
-	  include "../../private/db.php";
-	  if ($publisher->count()==0) {
-	    $publisher="null";
+    if($title != "")
+      {
+	include "../../private/db.php";
+	if ($publisher->count()==0) {
+	  $publisher="null";
+	}
+	else {
+	  $publisher=mysqli_real_escape_string($link,$publisher);
+	  if (strlen($publisher)>30) {
+	    $publisher=rtrim(substr($publisher,0,30),"\\");
 	  }
-	  else {
-	    $publisher=mysqli_real_escape_string($link,$publisher);
-	    if (strlen($publisher)>30) {
-	      $publisher=rtrim(substr($publisher,0,30),"\\");
-	    }
-	    $publisher="\"".$publisher."\"";
-	  }
-	  if ($author->count()==0) {
-	    $author="unknown";
-	  }
-	  else {
-	    $author=mysqli_real_escape_string($link,$author);
-	    if (strlen($author)>30) {
-	      $author=rtrim(substr($author,0,30),"\\");
-	    }
+	  $publisher="\"".$publisher."\"";
+	}
+	if ($author->count()==0) {
+	  $author="unknown";
+	}
+	else {
+	  $author=mysqli_real_escape_string($link,$author);
+	  if (strlen($author)>30) {
+	    $author=rtrim(substr($author,0,30),"\\");
 	  }
-	  $author="\"".$author."\"";
-	  $title=mysqli_real_escape_string($link,$title);
-	  if (strlen($title)>100) {
-	    $title=rtrim(substr($title,0,100),"\\");
-	  }
-	  $title="\"".$title."\"";
-	  $review1 = mysqli_real_escape_string($link,$review1);
-	  if (strlen($review1)>500) { $review1=rtrim(substr($review1,0,500),"\\");}
-	  $review2 = mysqli_real_escape_string($link,$review2);
-	  if (strlen($review2)>500) { $review2=rtrim(substr($review2,0,500),"\\");}
-	  $review3 = mysqli_real_escape_string($link,$review3);
-	  if (strlen($review3)>500) { $review3=rtrim(substr($review3,0,500),"\\");}
+	}
+	$author="\"".$author."\"";
+	$title=mysqli_real_escape_string($link,$title);
+	if (strlen($title)>100) {
+	  $title=rtrim(substr($title,0,100),"\\");
+	}
+	$title="\"".$title."\"";
+	$review1 = mysqli_real_escape_string($link,$review1);
+	if (strlen($review1)>500) { $review1=rtrim(substr($review1,0,500),"\\");}
+	$review2 = mysqli_real_escape_string($link,$review2);
+	if (strlen($review2)>500) { $review2=rtrim(substr($review2,0,500),"\\");}
+	$review3 = mysqli_real_escape_string($link,$review3);
+	if (strlen($review3)>500) { $review3=rtrim(substr($review3,0,500),"\\");}
 			
-	  $queryInsert = "CALL b_addNewBook(\"$searchparameterdata\",$title, $author,\"$binding\",\"$imageURL\", $dewey, $salesRank,$pubDate,$publisher,$g1,$g2,$g3,$loc)";	             
-	  //echo $queryInsert;
+	$queryInsert = "CALL b_addNewBook(\"$searchparameterdata\",$title, $author,\"$binding\",\"$imageURL\", $dewey, $salesRank,$pubDate,$publisher,$g1,$g2,$g3,$loc)";	             
+	//echo $queryInsert;
 				
-	  $res = mysqli_query($link, $queryInsert);
+	$res = mysqli_query($link, $queryInsert);
+	if (!$res) {
+	  $err=mysqli_error( $link );
+	  mysqli_close($link);
+	  file_put_contents('/var/ywww/debug/phpDebug',
+			    "anb failed: $queryInsert\n$err\n",
+			    FILE_APPEND);
+	  exit($err);
+	}
+	$queryInsertReviews = "CALL b_insertReviews(\"$searchparameterdata\",\"$review1\",\"$review2\",\"$review3\")";
+	if($review1 != "") {
+	  $res = mysqli_query($link, $queryInsertReviews);
 	  if (!$res) {
 	    $err=mysqli_error( $link );
 	    mysqli_close($link);
 	    file_put_contents('/var/ywww/debug/phpDebug',
-			      "anb failed: $queryInsert\n$err\n",
+			      "anr failed: $queryInsertReviews\n",
 			      FILE_APPEND);
 	    exit($err);
 	  }
-	  $queryInsertReviews = "CALL b_insertReviews(\"$searchparameterdata\",\"$review1\",\"$review2\",\"$review3\")";
-	  if($review1 != "") {
-	    $res = mysqli_query($link, $queryInsertReviews);
-	    if (!$res) {
-	      $err=mysqli_error( $link );
-	      mysqli_close($link);
-	      file_put_contents('/var/ywww/debug/phpDebug',
-				"anr failed: $queryInsertReviews\n",
-				FILE_APPEND);
-	      exit($err);
-	    }
-	  }
-	  mysqli_close($link);	//do not remove. reset is needed otherwise mysqli_fetch_array doesn't work after first loop		
 	}
+	mysqli_close($link);	//do not remove. reset is needed otherwise mysqli_fetch_array doesn't work after first loop		
+      }
 
-      echo $output;
-    }
+    echo $output;
+  }
 }
 
 function getAmazonDetLocal($go,$language,$loc,