diff xml/getAmazonInfo.php @ 23:d606320ec331

post-5.7-upgrade efforts to reduce dropped connections, db insertion fails
author Charlie Root
date Sun, 30 Dec 2018 07:00:09 -0500
parents 46382face560
children 828895488948
line wrap: on
line diff
--- a/xml/getAmazonInfo.php	Sun Dec 30 06:59:13 2018 -0500
+++ b/xml/getAmazonInfo.php	Sun Dec 30 07:00:09 2018 -0500
@@ -113,17 +113,17 @@
         curl_setopt ($crl, CURLOPT_RETURNTRANSFER, 1);
         curl_setopt ($crl, CURLOPT_CONNECTTIMEOUT, $timeout);
 	$semaphore = new SyncSemaphore("Amazon");
-	$gotit = $semaphore->lock(4000);
+	$gotit = $semaphore->lock(1000);
 	if (!$gotit) {
 	  file_put_contents('/var/ywww/debug/phpDebug',
 			"Got it: ".(int)$gotit." $requestIP 1\n",FILE_APPEND);
-	  $gotit=$semaphore->lock(5000);
+	  $gotit=$semaphore->lock(1000);
 	  file_put_contents('/var/ywww/debug/phpDebug',
 		        "Got it: ".(int)$gotit." $requestIP 1a\n",FILE_APPEND);
 	}
         $output = curl_exec($crl);
         curl_close($crl);
-	usleep(1000000);
+	usleep(500000);
 	if ($gotit) {
 	  $semaphore->unlock();
 	}
@@ -136,13 +136,17 @@
 	$review2 = "";
 	$review3 = "";
 	// HST added this
-	if (preg_match("<Error>",$output)) {
+	$mm=array();
+	if (preg_match("<Error>",$output,$mm)) {
 	  $xml = new SimpleXMLElement($output);	
 	  $resName=$xml->getName();
 	  $code=$xml->Error->Code;
-	  file_put_contents('/var/ywww/debug/phpDebug',"Losing: $resName, $code, $requestIP, $requestURI\n",FILE_APPEND);
+	  if (!$code) {
+	    $code=$xml->Items->Request->Errors->Error->Code;
+	  }
+	  file_put_contents('/var/ywww/debug/phpDebug',"Losing: ".$mm[0].", $resName, $code, $requestIP, $requestURI\n",FILE_APPEND);
 	  if ($code=='RequestThrottled') {
-	      sleep(5); // Try to reduce throttling until we get a 
+	      usleep(200000); // Try to reduce throttling until we get a 
 	      // principled solution in place
 	  }
 	  else {
@@ -178,7 +182,7 @@
 		$text = @file_get_contents($review . "&truncate=300");
 		$removeTop1 = preg_replace('~<div class="crIFrameHeaderLeftColumn">(.*?)<div class="crIFrameHeaderHistogram">~si', '', $text);
 		$removeTop2 = preg_replace('~<div class="crIFrameHeaderHistogram">(.*?)<div class="crIframeReviewList">~si', '', $text);
-		if (preg_match('~<body[^>]*>(.*?)</body>~si', $removeTop2, $body)){ $getBody = trim($body[1]); }
+		if (preg_match('~<body[^>]*>(.*?)</body>~si', $removeTop2, $body)){ $getBody = trim($body[1]); } else { $getBody = $removeTop2;}
 		$removeDiv = preg_replace('~<div[^>]*>(.*?)</div>~si', '', $getBody);
 		$removeCloseDivs = preg_replace('/<\/div>/','', $removeDiv);
 		$setBoundary = str_replace('<!-- BOUNDARY -->','BOTTOM-TOP', $removeCloseDivs);
@@ -340,6 +344,10 @@
 			
 	      foreach ($genArr as $key => $value) {
 		//echo "$key => $value";
+		if ($key>2047) {
+		  //HST added
+		  break;
+		}
 		$queryG = "CALL b_addBrowseNode($key,\"$value\")";	//add the name value pair for genre to new table	
 		//echo $queryG;
 		include "../../private/db.php";
@@ -400,7 +408,7 @@
 		if ( mysqli_num_rows( $res ) > 0 )
 		{
 			$rows=mysqli_fetch_array($res, MYSQLI_ASSOC);
-						
+			mysqli_close($link);	//do not remove. reset is needed otherwise mysqli_fetch_array doesn't work after first loop						
 			$output .=  "<ASIN>" . $searchparameterdata . "</ASIN>";
 			$output .=  "<Title>" . htmlspecialchars($rows["Title"]) . "</Title>";			
 			$output .=  "<Author>" . htmlspecialchars($rows["Author"]) . "</Author>";
@@ -416,7 +424,10 @@
 			$output .=  "<ProductGroup>Book</ProductGroup>";
 			$output .=  "<Error>AccountLimitExceeded</Error>";
 			
-		}		
+		}
+		else {
+		  mysqli_close($link);	//do not remove. reset is needed otherwise mysqli_fetch_array doesn't work after first loop
+		}
 		$output .=  "</Details>";
 		echo $output;
 	}