Tag Archives: curl

Multiple asynchronous http calls using curl

2 Replies

If you ever needed to fetch data from multiple sources on the backend, probably you already explored the benefits of using curl multi fetch.
In the examples below, I’m using curl to fetch data from two urls, which simply have

[php]
<?php
sleep(3);
echo “Content 1 …”;

[/php]

 intentionally to delay the content generation so we could see the difference between regular http call and asynchronous curl multi fetch.

http://toni-develops.com/sandbox/examples/php/curl-multi-fetch/fetched-content/fetch1.php
http://toni-develops.com/sandbox/examples/php/curl-multi-fetch/fetched-content/fetch2.php

[php]

<?php

$t1 = microtime(true);

function fetchContent($Url) {
// is cURL installed yet?
if (!function_exists(‘curl_init’)){
die(‘Sorry cURL is not installed!’);
}

// OK cool – then let’s create a new cURL resource handle
$ch = curl_init();

// Now set some options (most are optional)

// Set URL to download
curl_setopt($ch, CURLOPT_URL, $Url);

// Set a referer
curl_setopt($ch, CURLOPT_REFERER, “http://www.example.org/yay.htm”);

// User agent
curl_setopt($ch, CURLOPT_USERAGENT, “MozillaXYZ/1.0”);

// Include header in result? (0 = yes, 1 = no)
curl_setopt($ch, CURLOPT_HEADER, 0);

// Should cURL return or print out the data? (true = return, false = print)
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

// Timeout in seconds
curl_setopt($ch, CURLOPT_TIMEOUT, 10);

//Proxy if needed
//curl_setopt($ch, CURLOPT_PROXY, “64.210.197.20:80”);

curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2);

// Download the given URL, and return output
$output = curl_exec($ch);

// Close the cURL resource, and free system resources
curl_close($ch);
return $output;
}

echo “<textarea style=’width:100%;height:40%’>”;
echo fetchContent(‘http://toni-develops.com/sandbox/examples/php/curl-multi-fetch/fetched-content/fetch1.php’);
echo “</textarea>”;

echo ‘<hr>’;

echo “<textarea style=’width:100%;height:40%’>”;
echo fetchContent(‘http://toni-develops.com/sandbox/examples/php/curl-multi-fetch/fetched-content/fetch2.php’);
echo “</textarea>”;

echo ‘fetched for: ‘ . (microtime(true) – $t1) . “\n”;
[/php]

 

[php]
<?php

// is cURL installed yet?
if (!function_exists(‘curl_init’)){
die(‘Sorry cURL is not installed!’);
}

$ch = array();
$mh = curl_multi_init();
$total = 100;

$t1 = microtime(true);

$URLs = array( “http://toni-develops.com/sandbox/examples/php/curl-multi-fetch/fetched-content/fetch2.php”,
“http://toni-develops.com/sandbox/examples/php/curl-multi-fetch/fetched-content/fetch2.php”);

$i = 0;
foreach($URLs as $url) {
$ch[$i] = curl_init();
curl_setopt($ch[$i], CURLOPT_URL, $url);
curl_setopt($ch[$i], CURLOPT_HEADER, 0);
curl_setopt($ch[$i], CURLOPT_RETURNTRANSFER, true);

curl_multi_add_handle($mh, $ch[$i]);
$i ++;
}

$active = null;
do {
$mrc = curl_multi_exec($mh, $active);
//usleep(100); // Maybe needed to limit CPU load (See P.S.)
} while ($active);

$content = array();

$i = 0;
foreach ($ch AS $i => $c) {
$content[$i] = curl_multi_getcontent($c);
curl_multi_remove_handle($mh, $c);
}

curl_multi_close($mh);

echo “<textarea style=’width:100%;height:40%’>”;
echo $content[0];
echo “</textarea>”;

echo ‘<hr>’;

echo “<textarea style=’width:100%;height:40%’>”;
echo $content[1];
echo “</textarea>”;

echo ‘fetched for: ‘ . (microtime(true) – $t1) . “\n”;
[/php]

 

So this way we are reducing the amount of time to fetch data from multiple URLs to the amount needed to complete the longest transaction.