Mercurial > hg > ywww

changeset 21:46382face560

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
use a semaphore to try to cut down on RequestThrottled fails
author Charlie Root
date Thu, 25 Oct 2018 09:42:12 -0400
parents 7679346abfdb
children 69c37b58d091
files .hgignore php/read/getBookDetails.php php/read/similar.php xml/amazonBookSearch.php xml/getAmazonInfo.php
diffstat 5 files changed, 89 insertions(+), 15 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/.hgignore	Thu Oct 25 09:42:12 2018 -0400
@@ -0,0 +1,1 @@
+test
--- a/php/read/getBookDetails.php	Thu Oct 25 09:40:25 2018 -0400
+++ b/php/read/getBookDetails.php	Thu Oct 25 09:42:12 2018 -0400
@@ -204,16 +204,35 @@
 		curl_setopt ($crl, CURLOPT_ENCODING , "gzip"); 
 		curl_setopt ($crl, CURLOPT_RETURNTRANSFER, 1);
 		curl_setopt ($crl, CURLOPT_CONNECTTIMEOUT, $timeout);
+  	        $gotit=0;
+		$requestIP = $_SERVER['REMOTE_ADDR'];
+		$semaphore = new SyncSemaphore("Amazon");
+		$gotit = $semaphore->lock(4000);
+		if (!$gotit) {
+		  file_put_contents('/var/ywww/debug/phpDebug',
+           	  "Got it: ".(int)$gotit." $requestIP 2\n",FILE_APPEND);
+		  $gotit=$semaphore->lock(5000);
+		  file_put_contents('/var/ywww/debug/phpDebug',
+		    "Got it: ".(int)$gotit." $requestIP 2a\n",FILE_APPEND);
+		}
 		$out = curl_exec($crl);
 		curl_close($crl);
-
+		usleep(1000000);
+		if ($gotit) {
+		  $semaphore->unlock();
+		}
+		else {
+		  file_put_contents('/var/ywww/debug/phpDebug',
+				    "W/o lock for $requestIP 2b\n",FILE_APPEND);
+		}
 		//echo $out;
 		
 		if (preg_match("<Error>",$out)) {
 		  $xml = new SimpleXMLElement($out);	
 		  $resName=$xml->getName();
 		  $code=$xml->Error->Code;
-		  file_put_contents('/var/ywww/debug/phpDebug',"Losing gBD: $resName, $code\n",FILE_APPEND);
+		  $requestIP = $_SERVER['REMOTE_ADDR'];
+		  file_put_contents('/var/ywww/debug/phpDebug',"Losing gBD: $resName, $code, $requestIP\n",FILE_APPEND);
 		  if ($code=='RequestThrottled') {
 		    usleep(200000); // Try to reduce throttling until we get a 
 		    // principled solution in place
--- a/php/read/similar.php	Thu Oct 25 09:40:25 2018 -0400
+++ b/php/read/similar.php	Thu Oct 25 09:42:12 2018 -0400
@@ -106,8 +106,26 @@
         curl_setopt ($crl, CURLOPT_URL,$url);
         curl_setopt ($crl, CURLOPT_RETURNTRANSFER, 1);
         curl_setopt ($crl, CURLOPT_CONNECTTIMEOUT, $timeout);
+	$gotit=0;
+	  $semaphore = new SyncSemaphore("Amazon");
+	  $gotit = $semaphore->lock(4000);
+	  if (!$gotit) {
+	    file_put_contents('/var/ywww/debug/phpDebug',
+			      "Got it: ".(int)$gotit." 4\n",FILE_APPEND);
+	    $gotit = $semaphore->lock(5000);
+	    file_put_contents('/var/ywww/debug/phpDebug',
+		        "Got it: ".(int)$gotit." $requestIP 4a\n",FILE_APPEND);
+	  }
         $ret = curl_exec($crl);
         curl_close($crl);
+	usleep(1000000);
+	if ($gotit) {
+	  $semaphore->unlock();
+	}
+	else {
+	  file_put_contents('/var/ywww/debug/phpDebug',
+			    "Unlocking 4\n",FILE_APPEND);
+	}
         //echo $ret;
 		
 $xml = new SimpleXMLElement($ret);
--- a/xml/amazonBookSearch.php	Thu Oct 25 09:40:25 2018 -0400
+++ b/xml/amazonBookSearch.php	Thu Oct 25 09:42:12 2018 -0400
@@ -93,6 +93,17 @@
         curl_setopt ($crl, CURLOPT_ENCODING , "gzip");
 	curl_setopt ($crl, CURLOPT_RETURNTRANSFER, 1);
         curl_setopt ($crl, CURLOPT_CONNECTTIMEOUT, $timeout);
+	$gotit=0;
+	  $semaphore = new SyncSemaphore("Amazon");
+	  $gotit = $semaphore->lock(4000);
+	  file_put_contents('/var/ywww/debug/phpDebug',
+			    "Got it: ".(int)$gotit." 3\n",FILE_APPEND);
+	  if (!$gotit) { $gotit = $semaphore->lock(20000); }
         $ret = curl_exec($crl);
         curl_close($crl);
+	if ($gotit) {
+	  usleep(1000000);
+	  file_put_contents('/var/ywww/debug/phpDebug',"Unlocking 3\n",FILE_APPEND);
+	  $semaphore->unlock();
+	}
         echo $ret;
--- a/xml/getAmazonInfo.php	Thu Oct 25 09:40:25 2018 -0400
+++ b/xml/getAmazonInfo.php	Thu Oct 25 09:42:12 2018 -0400
@@ -83,30 +83,54 @@
 	'ReviewSort'	  =>'-HelpfulVotes'
   );
 
+$requestURI = $_SERVER['REQUEST_URI'];
+$requestIP = $_SERVER['REMOTE_ADDR'];
+// if ($requestIP=="173.161.113.65" || $requestIP=="141.8.132.25") {
+// 	    $delay=60;
+// 	    file_put_contents('/var/ywww/debug/phpDebug',
+// 			      "bad guy: $requestIP, $requestURI\n",
+// 			      FILE_APPEND);
+// 	    sleep($delay);
+// 	    # No, can't do this
+// 	    # return; # bomb!
+// 	    # 'Kung', sitting on my desk in the office while I'm at home,
+// 	    # is occasionally hitting xml/getAmazonInfo.php:
+// 	    # e.g. Losing: ItemLookupErrorResponse, RequestThrottled, 129.215.197.36, /xml/getAmazonInfo.php?searchparameterdata=075154454X&locale=uk
+// 	    # repeatedly, same params
+// }
 $ext=$Aserver[$locale]['ext'];  
 $file_data=$ext;
 ksort($parameters);
 foreach ($parameters as $i=>$d) {
   $file_data.='&'.$i.'='.$d;
 }
-
-$url=aws_signed_request($ext,$parameters,$public_key,$private_key);  
-//echo $url;
+        $gotit=0;
+        $url=aws_signed_request($ext,$parameters,$public_key,$private_key);  
         $crl = curl_init();
         $timeout = 5;
         curl_setopt ($crl, CURLOPT_URL,$url);
 		curl_setopt ($crl, CURLOPT_ENCODING , "gzip"); 
         curl_setopt ($crl, CURLOPT_RETURNTRANSFER, 1);
         curl_setopt ($crl, CURLOPT_CONNECTTIMEOUT, $timeout);
-	/*	$timeDiff=((int)(microtime(true)*1000))-$lastReqTime;
-	if ($timeDiff<1000) {
-	  usleep((1000-$timeDiff)*1000); // microseconds
-	  }*/
+	$semaphore = new SyncSemaphore("Amazon");
+	$gotit = $semaphore->lock(4000);
+	if (!$gotit) {
+	  file_put_contents('/var/ywww/debug/phpDebug',
+			"Got it: ".(int)$gotit." $requestIP 1\n",FILE_APPEND);
+	  $gotit=$semaphore->lock(5000);
+	  file_put_contents('/var/ywww/debug/phpDebug',
+		        "Got it: ".(int)$gotit." $requestIP 1a\n",FILE_APPEND);
+	}
         $output = curl_exec($crl);
-	//$lastReqTime=(int)(microtime(true)*1000); // milliseconds
         curl_close($crl);
-		
-	/*$xml = new SimpleXMLElement("<?xml version=\"1.0\"?><ItemLookupErrorResponse xmlns=\"http://ecs.amazonaws.com/doc/2009-03-31/\"><Error><Code>AccountLimitExceeded</Code><Message>Account limit of 2056 requests per hour exceeded.</Message></Error><RequestID>290ed059-730c-4789-93b4-6d21e11053d3</RequestID></ItemLookupErrorResponse>");*/
+	usleep(1000000);
+	if ($gotit) {
+	  $semaphore->unlock();
+	}
+	else {
+	  file_put_contents('/var/ywww/debug/phpDebug',
+			    "W/o lock for $requestIP 1b\n",FILE_APPEND);
+	}
 	$review = "";
 	$review1 = "";
 	$review2 = "";
@@ -116,10 +140,10 @@
 	  $xml = new SimpleXMLElement($output);	
 	  $resName=$xml->getName();
 	  $code=$xml->Error->Code;
-	  file_put_contents('/var/ywww/debug/phpDebug',"Losing: $resName, $code\n",FILE_APPEND);
+	  file_put_contents('/var/ywww/debug/phpDebug',"Losing: $resName, $code, $requestIP, $requestURI\n",FILE_APPEND);
 	  if ($code=='RequestThrottled') {
-	    usleep(200000); // Try to reduce throttling until we get a 
-	    // principled solution in place
+	      sleep(5); // Try to reduce throttling until we get a 
+	      // principled solution in place
 	  }
 	  else {
 	     file_put_contents('/var/ywww/debug/phpDebug',"parms: $go\nurl: $url\n".
@@ -590,4 +614,5 @@
 
 )
 */
+	/*$xml = new SimpleXMLElement("<?xml version=\"1.0\"?><ItemLookupErrorResponse xmlns=\"http://ecs.amazonaws.com/doc/2009-03-31/\"><Error><Code>AccountLimitExceeded</Code><Message>Account limit of 2056 requests per hour exceeded.</Message></Error><RequestID>290ed059-730c-4789-93b4-6d21e11053d3</RequestID></ItemLookupErrorResponse>");*/
 ?>