Mercurial > hg > ywww

comparison xml/getAmazonInfo.php @ 23:d606320ec331

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
post-5.7-upgrade efforts to reduce dropped connections, db insertion fails
author Charlie Root
date Sun, 30 Dec 2018 07:00:09 -0500
parents 46382face560
children 828895488948
comparison
equal deleted inserted replaced
22:69c37b58d091 23:d606320ec331
111 curl_setopt ($crl, CURLOPT_URL,$url); 111 curl_setopt ($crl, CURLOPT_URL,$url);
112 curl_setopt ($crl, CURLOPT_ENCODING , "gzip"); 112 curl_setopt ($crl, CURLOPT_ENCODING , "gzip");
113 curl_setopt ($crl, CURLOPT_RETURNTRANSFER, 1); 113 curl_setopt ($crl, CURLOPT_RETURNTRANSFER, 1);
114 curl_setopt ($crl, CURLOPT_CONNECTTIMEOUT, $timeout); 114 curl_setopt ($crl, CURLOPT_CONNECTTIMEOUT, $timeout);
115 $semaphore = new SyncSemaphore("Amazon"); 115 $semaphore = new SyncSemaphore("Amazon");
116 $gotit = $semaphore->lock(4000); 116 $gotit = $semaphore->lock(1000);
117 if (!$gotit) { 117 if (!$gotit) {
118 file_put_contents('/var/ywww/debug/phpDebug', 118 file_put_contents('/var/ywww/debug/phpDebug',
119 "Got it: ".(int)$gotit." $requestIP 1\n",FILE_APPEND); 119 "Got it: ".(int)$gotit." $requestIP 1\n",FILE_APPEND);
120 $gotit=$semaphore->lock(5000); 120 $gotit=$semaphore->lock(1000);
121 file_put_contents('/var/ywww/debug/phpDebug', 121 file_put_contents('/var/ywww/debug/phpDebug',
122 "Got it: ".(int)$gotit." $requestIP 1a\n",FILE_APPEND); 122 "Got it: ".(int)$gotit." $requestIP 1a\n",FILE_APPEND);
123 } 123 }
124 $output = curl_exec($crl); 124 $output = curl_exec($crl);
125 curl_close($crl); 125 curl_close($crl);
126 usleep(1000000); 126 usleep(500000);
127 if ($gotit) { 127 if ($gotit) {
128 $semaphore->unlock(); 128 $semaphore->unlock();
129 } 129 }
130 else { 130 else {
131 file_put_contents('/var/ywww/debug/phpDebug', 131 file_put_contents('/var/ywww/debug/phpDebug',
134 $review = ""; 134 $review = "";
135 $review1 = ""; 135 $review1 = "";
136 $review2 = ""; 136 $review2 = "";
137 $review3 = ""; 137 $review3 = "";
138 // HST added this 138 // HST added this
139 if (preg_match("<Error>",$output)) { 139 $mm=array();
140 if (preg_match("<Error>",$output,$mm)) {
140 $xml = new SimpleXMLElement($output); 141 $xml = new SimpleXMLElement($output);
141 $resName=$xml->getName(); 142 $resName=$xml->getName();
142 $code=$xml->Error->Code; 143 $code=$xml->Error->Code;
143 file_put_contents('/var/ywww/debug/phpDebug',"Losing: $resName, $code, $requestIP, $requestURI\n",FILE_APPEND); 144 if (!$code) {
145 $code=$xml->Items->Request->Errors->Error->Code;
146 }
147 file_put_contents('/var/ywww/debug/phpDebug',"Losing: ".$mm[0].", $resName, $code, $requestIP, $requestURI\n",FILE_APPEND);
144 if ($code=='RequestThrottled') { 148 if ($code=='RequestThrottled') {
145 sleep(5); // Try to reduce throttling until we get a 149 usleep(200000); // Try to reduce throttling until we get a
146 // principled solution in place 150 // principled solution in place
147 } 151 }
148 else { 152 else {
149 file_put_contents('/var/ywww/debug/phpDebug',"parms: $go\nurl: $url\n". 153 file_put_contents('/var/ywww/debug/phpDebug',"parms: $go\nurl: $url\n".
150 print_r($parameters,TRUE)."\n",FILE_APPEND); 154 print_r($parameters,TRUE)."\n",FILE_APPEND);
176 if ($review != "") 180 if ($review != "")
177 { 181 {
178 $text = @file_get_contents($review . "&truncate=300"); 182 $text = @file_get_contents($review . "&truncate=300");
179 $removeTop1 = preg_replace('~<div class="crIFrameHeaderLeftColumn">(.*?)<div class="crIFrameHeaderHistogram">~si', '', $text); 183 $removeTop1 = preg_replace('~<div class="crIFrameHeaderLeftColumn">(.*?)<div class="crIFrameHeaderHistogram">~si', '', $text);
180 $removeTop2 = preg_replace('~<div class="crIFrameHeaderHistogram">(.*?)<div class="crIframeReviewList">~si', '', $text); 184 $removeTop2 = preg_replace('~<div class="crIFrameHeaderHistogram">(.*?)<div class="crIframeReviewList">~si', '', $text);
181 if (preg_match('~<body[^>]*>(.*?)</body>~si', $removeTop2, $body)){ $getBody = trim($body[1]); } 185 if (preg_match('~<body[^>]*>(.*?)</body>~si', $removeTop2, $body)){ $getBody = trim($body[1]); } else { $getBody = $removeTop2;}
182 $removeDiv = preg_replace('~<div[^>]*>(.*?)</div>~si', '', $getBody); 186 $removeDiv = preg_replace('~<div[^>]*>(.*?)</div>~si', '', $getBody);
183 $removeCloseDivs = preg_replace('/<\/div>/','', $removeDiv); 187 $removeCloseDivs = preg_replace('/<\/div>/','', $removeDiv);
184 $setBoundary = str_replace('<!-- BOUNDARY -->','BOTTOM-TOP', $removeCloseDivs); 188 $setBoundary = str_replace('<!-- BOUNDARY -->','BOTTOM-TOP', $removeCloseDivs);
185 //replace <!-- BOUNDARY --> with BOTTOM-TOP 189 //replace <!-- BOUNDARY --> with BOTTOM-TOP
186 $remove1 = '~<table cellpadding="0"(.*?)%">~s'; 190 $remove1 = '~<table cellpadding="0"(.*?)%">~s';
338 $g3 = "null"; 342 $g3 = "null";
339 $loop = 1; 343 $loop = 1;
340 344
341 foreach ($genArr as $key => $value) { 345 foreach ($genArr as $key => $value) {
342 //echo "$key => $value"; 346 //echo "$key => $value";
347 if ($key>2047) {
348 //HST added
349 break;
350 }
343 $queryG = "CALL b_addBrowseNode($key,\"$value\")"; //add the name value pair for genre to new table 351 $queryG = "CALL b_addBrowseNode($key,\"$value\")"; //add the name value pair for genre to new table
344 //echo $queryG; 352 //echo $queryG;
345 include "../../private/db.php"; 353 include "../../private/db.php";
346 $resG = mysqli_query($link, $queryG); 354 $resG = mysqli_query($link, $queryG);
347 mysqli_close($link); 355 mysqli_close($link);
398 $output .= "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"; 406 $output .= "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
399 $output .= "<Details>"; 407 $output .= "<Details>";
400 if ( mysqli_num_rows( $res ) > 0 ) 408 if ( mysqli_num_rows( $res ) > 0 )
401 { 409 {
402 $rows=mysqli_fetch_array($res, MYSQLI_ASSOC); 410 $rows=mysqli_fetch_array($res, MYSQLI_ASSOC);
403 411 mysqli_close($link); //do not remove. reset is needed otherwise mysqli_fetch_array doesn't work after first loop
404 $output .= "<ASIN>" . $searchparameterdata . "</ASIN>"; 412 $output .= "<ASIN>" . $searchparameterdata . "</ASIN>";
405 $output .= "<Title>" . htmlspecialchars($rows["Title"]) . "</Title>"; 413 $output .= "<Title>" . htmlspecialchars($rows["Title"]) . "</Title>";
406 $output .= "<Author>" . htmlspecialchars($rows["Author"]) . "</Author>"; 414 $output .= "<Author>" . htmlspecialchars($rows["Author"]) . "</Author>";
407 $output .= "<Binding>" . htmlspecialchars($rows["Binding"]) . "</Binding>"; 415 $output .= "<Binding>" . htmlspecialchars($rows["Binding"]) . "</Binding>";
408 $output .= "<Dewey>" . htmlspecialchars($rows["DeweyDecimal"]) . "</Dewey>"; 416 $output .= "<Dewey>" . htmlspecialchars($rows["DeweyDecimal"]) . "</Dewey>";
414 $output .= "<Genre2>" . htmlspecialchars($rows["Genre2"]) . "</Genre2>"; 422 $output .= "<Genre2>" . htmlspecialchars($rows["Genre2"]) . "</Genre2>";
415 $output .= "<Genre3>" . htmlspecialchars($rows["Genre3"]) . "</Genre3>"; 423 $output .= "<Genre3>" . htmlspecialchars($rows["Genre3"]) . "</Genre3>";
416 $output .= "<ProductGroup>Book</ProductGroup>"; 424 $output .= "<ProductGroup>Book</ProductGroup>";
417 $output .= "<Error>AccountLimitExceeded</Error>"; 425 $output .= "<Error>AccountLimitExceeded</Error>";
418 426
419 } 427 }
428 else {
429 mysqli_close($link); //do not remove. reset is needed otherwise mysqli_fetch_array doesn't work after first loop
430 }
420 $output .= "</Details>"; 431 $output .= "</Details>";
421 echo $output; 432 echo $output;
422 } 433 }
423 } 434 }
424 435