Mercurial > hg > ywww

changeset 23:d606320ec331

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
post-5.7-upgrade efforts to reduce dropped connections, db insertion fails
author Charlie Root
date Sun, 30 Dec 2018 07:00:09 -0500
parents 69c37b58d091
children 341d8c681357
files php/read/getAllSavedBooks.php php/read/getBookDetails.php php/read/similar.php uk/index.php us/index.php xml/amazonBookSearch.php xml/getAmazonInfo.php
diffstat 7 files changed, 88 insertions(+), 35 deletions(-) [+]
line wrap: on
line diff
--- a/php/read/getAllSavedBooks.php	Sun Dec 30 06:59:13 2018 -0500
+++ b/php/read/getAllSavedBooks.php	Sun Dec 30 07:00:09 2018 -0500
@@ -32,6 +32,7 @@
 
 $prevListName = "";
 
+if ($results) {
 while($line = mysqli_fetch_assoc($results)) {
 
 	$listName = htmlspecialchars(trim($line["SavedListName"]));
@@ -58,12 +59,14 @@
 		
 	$savedBooks++;
 }
-	
-	if($savedBooks != 0)
-	{
-		$output .=  "</ListDetails>";
-		$output .=  "</List>";
-	}
+ $results->free();
+ }
+mysqli_close($link);	
+if($savedBooks != 0)
+  {
+    $output .=  "</ListDetails>";
+    $output .=  "</List>";
+  }
 $output .=  "</results>";
 
 if(isset($ret))
@@ -71,7 +74,7 @@
 else
 	echo $output;
 
-mysqli_close($link);
+
 
 
 ?>
\ No newline at end of file
--- a/php/read/getBookDetails.php	Sun Dec 30 06:59:13 2018 -0500
+++ b/php/read/getBookDetails.php	Sun Dec 30 07:00:09 2018 -0500
@@ -69,6 +69,7 @@
 	if ( mysqli_num_rows( $res ) > 0 )
 	{
 		$rows=mysqli_fetch_array($res, MYSQLI_ASSOC);
+		mysqli_close($link);	//do not remove. reset is needed otherwise mysqli_fetch_array doesn't work after first loop
 		$timestamp=$rows['Timestamp'];
 		//echo $timestamp;
 		date_default_timezone_set('UTC');
@@ -97,7 +98,9 @@
 			$output .=  "<ProductGroup>Book</ProductGroup>";
 		}
 	}
-	mysqli_close($link);	//do not remove. reset is needed otherwise mysqli_fetch_array doesn't work after first loop
+	else {
+	  mysqli_close($link);	//do not remove. reset is needed otherwise mysqli_fetch_array doesn't work after first loop
+	}
 	include "../../../private/db.php";
 	
 	if($proceed == true)
@@ -207,17 +210,17 @@
   	        $gotit=0;
 		$requestIP = $_SERVER['REMOTE_ADDR'];
 		$semaphore = new SyncSemaphore("Amazon");
-		$gotit = $semaphore->lock(4000);
+		$gotit = $semaphore->lock(1000);
 		if (!$gotit) {
 		  file_put_contents('/var/ywww/debug/phpDebug',
            	  "Got it: ".(int)$gotit." $requestIP 2\n",FILE_APPEND);
-		  $gotit=$semaphore->lock(5000);
+		  $gotit=$semaphore->lock(1000);
 		  file_put_contents('/var/ywww/debug/phpDebug',
 		    "Got it: ".(int)$gotit." $requestIP 2a\n",FILE_APPEND);
 		}
 		$out = curl_exec($crl);
 		curl_close($crl);
-		usleep(1000000);
+		usleep(500000);
 		if ($gotit) {
 		  $semaphore->unlock();
 		}
@@ -226,13 +229,16 @@
 				    "W/o lock for $requestIP 2b\n",FILE_APPEND);
 		}
 		//echo $out;
-		
-		if (preg_match("<Error>",$out)) {
+		$mm=array();
+		if (preg_match("<Error>",$out,$mm)) {
 		  $xml = new SimpleXMLElement($out);	
 		  $resName=$xml->getName();
 		  $code=$xml->Error->Code;
+		  if (!$code) {
+		    $code=$xml->Items->Request->Errors->Error->Code;
+		  }
 		  $requestIP = $_SERVER['REMOTE_ADDR'];
-		  file_put_contents('/var/ywww/debug/phpDebug',"Losing gBD: $resName, $code, $requestIP\n",FILE_APPEND);
+		  file_put_contents('/var/ywww/debug/phpDebug',"Losing gBD: ".$mm[0].", $resName, $code, $requestIP\n",FILE_APPEND);
 		  if ($code=='RequestThrottled') {
 		    usleep(200000); // Try to reduce throttling until we get a 
 		    // principled solution in place
@@ -278,6 +284,16 @@
 		  $pubDate = $xml->Items->Item->ItemAttributes->PublicationDate;
 		  $publisher = $xml->Items->Item->ItemAttributes->Publisher;
 		  restore_error_handler();
+		  if ($publisher and strlen($publisher)>30) {
+		    $publisher=substr($publisher,0,30);
+		  }
+		  if ($title and strlen($title)>100) {
+		    $title=substr($title,0,100);
+		  }
+		  $publisher=mysqli_real_escape_string($link,$publisher);
+		  $title=mysqli_real_escape_string($link,$title);
+		  if (strlen($pubDate)==4) { $pubDate=$pubDate."-01-01";}
+		  if (strlen($pubDate)==7) { $pubDate=$pubDate."-01";}
 		  /*echo $title;
 		   echo $author;
 		   echo $binding;
@@ -316,6 +332,10 @@
 		
 		    foreach ($genArr as $key => $value) {
 		      //echo "$key => $value";
+		      if ($key>2047) {
+			//HST added
+			break;
+		      }
 		      $queryG = "CALL b_addBrowseNode($key,\"$value\")";	//add the name value pair for genre to new table	
 		      //echo $queryG;
 		      $resG = mysqli_query($link, $queryG);
@@ -335,7 +355,7 @@
 		      $loop++;
 		    }
 		  }
-		
+		  
 		  if($salesRank == "")
 		    $salesRank = "null";
 		
@@ -346,7 +366,15 @@
 		  $queryInsert = "CALL b_addNewBook(\"$isbn\",\"$title\", \"$author\",\"$binding\",\"$imageURL\", $dewey, $salesRank,\"$pubDate\",\"$publisher\",$g1,$g2,$g3,$loc)";	
 		  //echo $queryInsert;
 		
-		  $resG = mysqli_query($link, $queryInsert) or exit( mysqli_error( $link ));
+		  $res = mysqli_query($link, $queryInsert);
+				if (!$res) {
+				  $err=mysqli_error( $link );
+				  mysqli_close($link);
+				  file_put_contents('/var/ywww/debug/phpDebug',
+					"anb failed: $err, $pubDate, $g2, $publisher, $title\n",
+					FILE_APPEND);
+				  exit($err);
+				}
 		  mysqli_close($link);	//do not remove. reset is needed otherwise mysqli_fetch_array doesn't work after first loop		
 
 		
--- a/php/read/similar.php	Sun Dec 30 06:59:13 2018 -0500
+++ b/php/read/similar.php	Sun Dec 30 07:00:09 2018 -0500
@@ -108,17 +108,17 @@
         curl_setopt ($crl, CURLOPT_CONNECTTIMEOUT, $timeout);
 	$gotit=0;
 	  $semaphore = new SyncSemaphore("Amazon");
-	  $gotit = $semaphore->lock(4000);
+	  $gotit = $semaphore->lock(1000);
 	  if (!$gotit) {
 	    file_put_contents('/var/ywww/debug/phpDebug',
 			      "Got it: ".(int)$gotit." 4\n",FILE_APPEND);
-	    $gotit = $semaphore->lock(5000);
+	    $gotit = $semaphore->lock(1000);
 	    file_put_contents('/var/ywww/debug/phpDebug',
-		        "Got it: ".(int)$gotit." $requestIP 4a\n",FILE_APPEND);
+		        "Got it: ".(int)$gotit." 4a\n",FILE_APPEND);
 	  }
         $ret = curl_exec($crl);
         curl_close($crl);
-	usleep(1000000);
+	usleep(500000);
 	if ($gotit) {
 	  $semaphore->unlock();
 	}
--- a/uk/index.php	Sun Dec 30 06:59:13 2018 -0500
+++ b/uk/index.php	Sun Dec 30 07:00:09 2018 -0500
@@ -127,17 +127,24 @@
 				include "../../private/db.php"; 
 				$query = "CALL b_getBookInfo('$isbn', $countryCode)";
 				//echo $query;
-				$res = mysqli_query($link, $query) or exit( mysqli_error( $link ));
+				$res = mysqli_query($link, $query);
+				if (!$res) {
+				  $err=mysqli_error( $link );
+				  mysqli_close($link);
+				  exit($err);
+				}
 				include "../xml/aws_signed_request.php";
 				include "../xml/getAmazonInfo.php";
 				if ( mysqli_num_rows( $res ) > 0 )
 				{
 					$rows=mysqli_fetch_array($res, MYSQLI_ASSOC);		
+					mysqli_close($link);
 					$title = $rows["Title"];
 					$author = $rows["Author"];
 				}		
 				else
 				{
+				  mysqli_close($link);
 					$bookDetails = getAmazonDet($isbn,1,'uk'); 
 					$bookXML = new SimpleXMLElement($bookDetails);
 					if ($bookXML->Items && $bookXML->Items->Item[0] &&
@@ -175,12 +182,13 @@
 					if ( mysqli_num_rows( $res ) > 0 )
 					{
 						$rows=mysqli_fetch_array($res, MYSQLI_ASSOC);		
+						mysqli_close($link);
 						$titleL = $rows["Title"];
 						$authorL = $rows["Author"];
-						mysqli_close($link);
 					}		
 					else
 					{					
+						mysqli_close($link);
 						$bookLDetails = getAmazonDet($linkedISBN,1,'uk'); 
 						$bookLXML = new SimpleXMLElement($bookLDetails);
 						if ($bookLXML->Items &&
--- a/us/index.php	Sun Dec 30 06:59:13 2018 -0500
+++ b/us/index.php	Sun Dec 30 07:00:09 2018 -0500
@@ -133,11 +133,13 @@
 				if ( mysqli_num_rows( $res ) > 0 )
 				{
 					$rows=mysqli_fetch_array($res, MYSQLI_ASSOC);		
+					mysqli_close($link);
 					$title = $rows["Title"];
 					$author = $rows["Author"];
 				}		
 				else
 				{
+				  mysqli_close($link);
 					$bookDetails = getAmazonDet($isbn,1,'us'); 
 					$bookXML = new SimpleXMLElement($bookDetails);
 					// HST added this
@@ -206,12 +208,13 @@
 				  if ( mysqli_num_rows( $res ) > 0 )
 				    {
 				      $rows=mysqli_fetch_array($res, MYSQLI_ASSOC);		
+				      mysqli_close($link);
 				      $titleL = $rows["Title"];
 				      $authorL = $rows["Author"];
-				      mysqli_close($link);
 				    }		
 				  else
 				    {					
+				      mysqli_close($link);
 				      $bookLDetails = getAmazonDet($linkedISBN,1,'us'); 
 				      $bookLXML = new SimpleXMLElement($bookLDetails);
 				      // HST added this
--- a/xml/amazonBookSearch.php	Sun Dec 30 06:59:13 2018 -0500
+++ b/xml/amazonBookSearch.php	Sun Dec 30 07:00:09 2018 -0500
@@ -95,14 +95,14 @@
         curl_setopt ($crl, CURLOPT_CONNECTTIMEOUT, $timeout);
 	$gotit=0;
 	  $semaphore = new SyncSemaphore("Amazon");
-	  $gotit = $semaphore->lock(4000);
+	  $gotit = $semaphore->lock(1000);
 	  file_put_contents('/var/ywww/debug/phpDebug',
 			    "Got it: ".(int)$gotit." 3\n",FILE_APPEND);
-	  if (!$gotit) { $gotit = $semaphore->lock(20000); }
+	  if (!$gotit) { $gotit = $semaphore->lock(1000); }
         $ret = curl_exec($crl);
         curl_close($crl);
 	if ($gotit) {
-	  usleep(1000000);
+	  usleep(500000);
 	  file_put_contents('/var/ywww/debug/phpDebug',"Unlocking 3\n",FILE_APPEND);
 	  $semaphore->unlock();
 	}
--- a/xml/getAmazonInfo.php	Sun Dec 30 06:59:13 2018 -0500
+++ b/xml/getAmazonInfo.php	Sun Dec 30 07:00:09 2018 -0500
@@ -113,17 +113,17 @@
         curl_setopt ($crl, CURLOPT_RETURNTRANSFER, 1);
         curl_setopt ($crl, CURLOPT_CONNECTTIMEOUT, $timeout);
 	$semaphore = new SyncSemaphore("Amazon");
-	$gotit = $semaphore->lock(4000);
+	$gotit = $semaphore->lock(1000);
 	if (!$gotit) {
 	  file_put_contents('/var/ywww/debug/phpDebug',
 			"Got it: ".(int)$gotit." $requestIP 1\n",FILE_APPEND);
-	  $gotit=$semaphore->lock(5000);
+	  $gotit=$semaphore->lock(1000);
 	  file_put_contents('/var/ywww/debug/phpDebug',
 		        "Got it: ".(int)$gotit." $requestIP 1a\n",FILE_APPEND);
 	}
         $output = curl_exec($crl);
         curl_close($crl);
-	usleep(1000000);
+	usleep(500000);
 	if ($gotit) {
 	  $semaphore->unlock();
 	}
@@ -136,13 +136,17 @@
 	$review2 = "";
 	$review3 = "";
 	// HST added this
-	if (preg_match("<Error>",$output)) {
+	$mm=array();
+	if (preg_match("<Error>",$output,$mm)) {
 	  $xml = new SimpleXMLElement($output);	
 	  $resName=$xml->getName();
 	  $code=$xml->Error->Code;
-	  file_put_contents('/var/ywww/debug/phpDebug',"Losing: $resName, $code, $requestIP, $requestURI\n",FILE_APPEND);
+	  if (!$code) {
+	    $code=$xml->Items->Request->Errors->Error->Code;
+	  }
+	  file_put_contents('/var/ywww/debug/phpDebug',"Losing: ".$mm[0].", $resName, $code, $requestIP, $requestURI\n",FILE_APPEND);
 	  if ($code=='RequestThrottled') {
-	      sleep(5); // Try to reduce throttling until we get a 
+	      usleep(200000); // Try to reduce throttling until we get a 
 	      // principled solution in place
 	  }
 	  else {
@@ -178,7 +182,7 @@
 		$text = @file_get_contents($review . "&truncate=300");
 		$removeTop1 = preg_replace('~<div class="crIFrameHeaderLeftColumn">(.*?)<div class="crIFrameHeaderHistogram">~si', '', $text);
 		$removeTop2 = preg_replace('~<div class="crIFrameHeaderHistogram">(.*?)<div class="crIframeReviewList">~si', '', $text);
-		if (preg_match('~<body[^>]*>(.*?)</body>~si', $removeTop2, $body)){ $getBody = trim($body[1]); }
+		if (preg_match('~<body[^>]*>(.*?)</body>~si', $removeTop2, $body)){ $getBody = trim($body[1]); } else { $getBody = $removeTop2;}
 		$removeDiv = preg_replace('~<div[^>]*>(.*?)</div>~si', '', $getBody);
 		$removeCloseDivs = preg_replace('/<\/div>/','', $removeDiv);
 		$setBoundary = str_replace('<!-- BOUNDARY -->','BOTTOM-TOP', $removeCloseDivs);
@@ -340,6 +344,10 @@
 			
 	      foreach ($genArr as $key => $value) {
 		//echo "$key => $value";
+		if ($key>2047) {
+		  //HST added
+		  break;
+		}
 		$queryG = "CALL b_addBrowseNode($key,\"$value\")";	//add the name value pair for genre to new table	
 		//echo $queryG;
 		include "../../private/db.php";
@@ -400,7 +408,7 @@
 		if ( mysqli_num_rows( $res ) > 0 )
 		{
 			$rows=mysqli_fetch_array($res, MYSQLI_ASSOC);
-						
+			mysqli_close($link);	//do not remove. reset is needed otherwise mysqli_fetch_array doesn't work after first loop						
 			$output .=  "<ASIN>" . $searchparameterdata . "</ASIN>";
 			$output .=  "<Title>" . htmlspecialchars($rows["Title"]) . "</Title>";			
 			$output .=  "<Author>" . htmlspecialchars($rows["Author"]) . "</Author>";
@@ -416,7 +424,10 @@
 			$output .=  "<ProductGroup>Book</ProductGroup>";
 			$output .=  "<Error>AccountLimitExceeded</Error>";
 			
-		}		
+		}
+		else {
+		  mysqli_close($link);	//do not remove. reset is needed otherwise mysqli_fetch_array doesn't work after first loop
+		}
 		$output .=  "</Details>";
 		echo $output;
 	}