Mercurial > hg > ywww
diff xml/getAmazonInfo.php @ 23:d606320ec331
post-5.7-upgrade efforts to reduce dropped connections, db insertion fails
author | Charlie Root |
---|---|
date | Sun, 30 Dec 2018 07:00:09 -0500 |
parents | 46382face560 |
children | 828895488948 |
line wrap: on
line diff
--- a/xml/getAmazonInfo.php Sun Dec 30 06:59:13 2018 -0500 +++ b/xml/getAmazonInfo.php Sun Dec 30 07:00:09 2018 -0500 @@ -113,17 +113,17 @@ curl_setopt ($crl, CURLOPT_RETURNTRANSFER, 1); curl_setopt ($crl, CURLOPT_CONNECTTIMEOUT, $timeout); $semaphore = new SyncSemaphore("Amazon"); - $gotit = $semaphore->lock(4000); + $gotit = $semaphore->lock(1000); if (!$gotit) { file_put_contents('/var/ywww/debug/phpDebug', "Got it: ".(int)$gotit." $requestIP 1\n",FILE_APPEND); - $gotit=$semaphore->lock(5000); + $gotit=$semaphore->lock(1000); file_put_contents('/var/ywww/debug/phpDebug', "Got it: ".(int)$gotit." $requestIP 1a\n",FILE_APPEND); } $output = curl_exec($crl); curl_close($crl); - usleep(1000000); + usleep(500000); if ($gotit) { $semaphore->unlock(); } @@ -136,13 +136,17 @@ $review2 = ""; $review3 = ""; // HST added this - if (preg_match("<Error>",$output)) { + $mm=array(); + if (preg_match("<Error>",$output,$mm)) { $xml = new SimpleXMLElement($output); $resName=$xml->getName(); $code=$xml->Error->Code; - file_put_contents('/var/ywww/debug/phpDebug',"Losing: $resName, $code, $requestIP, $requestURI\n",FILE_APPEND); + if (!$code) { + $code=$xml->Items->Request->Errors->Error->Code; + } + file_put_contents('/var/ywww/debug/phpDebug',"Losing: ".$mm[0].", $resName, $code, $requestIP, $requestURI\n",FILE_APPEND); if ($code=='RequestThrottled') { - sleep(5); // Try to reduce throttling until we get a + usleep(200000); // Try to reduce throttling until we get a // principled solution in place } else { @@ -178,7 +182,7 @@ $text = @file_get_contents($review . "&truncate=300"); $removeTop1 = preg_replace('~<div class="crIFrameHeaderLeftColumn">(.*?)<div class="crIFrameHeaderHistogram">~si', '', $text); $removeTop2 = preg_replace('~<div class="crIFrameHeaderHistogram">(.*?)<div class="crIframeReviewList">~si', '', $text); - if (preg_match('~<body[^>]*>(.*?)</body>~si', $removeTop2, $body)){ $getBody = trim($body[1]); } + if (preg_match('~<body[^>]*>(.*?)</body>~si', $removeTop2, $body)){ $getBody = trim($body[1]); } else { $getBody = $removeTop2;} $removeDiv = preg_replace('~<div[^>]*>(.*?)</div>~si', '', $getBody); $removeCloseDivs = preg_replace('/<\/div>/','', $removeDiv); $setBoundary = str_replace('<!-- BOUNDARY -->','BOTTOM-TOP', $removeCloseDivs); @@ -340,6 +344,10 @@ foreach ($genArr as $key => $value) { //echo "$key => $value"; + if ($key>2047) { + //HST added + break; + } $queryG = "CALL b_addBrowseNode($key,\"$value\")"; //add the name value pair for genre to new table //echo $queryG; include "../../private/db.php"; @@ -400,7 +408,7 @@ if ( mysqli_num_rows( $res ) > 0 ) { $rows=mysqli_fetch_array($res, MYSQLI_ASSOC); - + mysqli_close($link); //do not remove. reset is needed otherwise mysqli_fetch_array doesn't work after first loop $output .= "<ASIN>" . $searchparameterdata . "</ASIN>"; $output .= "<Title>" . htmlspecialchars($rows["Title"]) . "</Title>"; $output .= "<Author>" . htmlspecialchars($rows["Author"]) . "</Author>"; @@ -416,7 +424,10 @@ $output .= "<ProductGroup>Book</ProductGroup>"; $output .= "<Error>AccountLimitExceeded</Error>"; - } + } + else { + mysqli_close($link); //do not remove. reset is needed otherwise mysqli_fetch_array doesn't work after first loop + } $output .= "</Details>"; echo $output; }