Skip to content

Commit f50d7d5

Browse files
loevgaardloevgaard
committed
Update the popular bots list and match it case-insensitively
The popular bots list is a fast path checked before the full bot regex. It is now the 20 highest-volume crawlers based on Cloudflare Radar data (search engines, AI crawlers, SEO crawlers and Meta's fetchers). Yahoo! Slurp, ia_archiver and facebookcatalog are dropped: Yahoo search is served by Bing, Alexa was retired in 2022 and facebookcatalog is low volume. The list is now matched case-insensitively. Previously the "Bingbot" entry never matched Bing's real user agent, which uses "bingbot".
1 parent 2899d69 commit f50d7d5

4 files changed

Lines changed: 76 additions & 14 deletions

File tree

‎src/BotDetector/BotDetector.php‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,25 @@ public function __construct(RequestStack $requestStack, ?array $popular = null)
2626
if (null === $popular) {
2727
$popular = [
2828
'Googlebot',
29-
'Bingbot',
30-
'Yahoo! Slurp',
31-
'DuckDuckBot',
32-
'Baiduspider',
29+
'meta-externalagent',
30+
'GPTBot',
31+
'ClaudeBot',
32+
'bingbot',
33+
'Amazonbot',
34+
'GoogleOther',
3335
'YandexBot',
36+
'Bytespider',
37+
'AhrefsBot',
38+
'Applebot',
39+
'SemrushBot',
40+
'ChatGPT-User',
41+
'OAI-SearchBot',
42+
'PerplexityBot',
43+
'meta-webindexer',
3444
'facebookexternalhit',
35-
'facebookcatalog',
36-
'ia_archiver',
45+
'Baiduspider',
46+
'PetalBot',
47+
'DuckDuckBot',
3748
];
3849
}
3950

@@ -43,7 +54,7 @@ public function __construct(RequestStack $requestStack, ?array $popular = null)
4354
public function isBot(string $userAgent): bool
4455
{
4556
if (!isset($this->cache[$userAgent])) {
46-
$minimalRegex = '#' . implode('|', $this->popular) . '#';
57+
$minimalRegex = '#' . implode('|', $this->popular) . '#i';
4758
$minimalMatch = preg_match($minimalRegex, $userAgent) === 1;
4859
$this->cache[$userAgent] = $minimalMatch || preg_match(Bots::REGEX, $userAgent) === 1;
4960
}

‎src/Resources/config/services.php‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,18 +12,29 @@
1212
return static function (ContainerConfigurator $container): void {
1313
/**
1414
* Use this collection to add popular bots to the bot detector. This will speed up detection when the bot is one of these.
15-
* These are regexes that should match the user agent of the respective bot.
15+
* These are regexes that should match the user agent of the respective bot. They are matched case-insensitively.
1616
*/
1717
$container->parameters()->set('setono_bot_detection.popular_bots', [
1818
'Googlebot',
19-
'Bingbot',
20-
'Yahoo! Slurp',
21-
'DuckDuckBot',
22-
'Baiduspider',
19+
'meta-externalagent',
20+
'GPTBot',
21+
'ClaudeBot',
22+
'bingbot',
23+
'Amazonbot',
24+
'GoogleOther',
2325
'YandexBot',
26+
'Bytespider',
27+
'AhrefsBot',
28+
'Applebot',
29+
'SemrushBot',
30+
'ChatGPT-User',
31+
'OAI-SearchBot',
32+
'PerplexityBot',
33+
'meta-webindexer',
2434
'facebookexternalhit',
25-
'facebookcatalog',
26-
'ia_archiver',
35+
'Baiduspider',
36+
'PetalBot',
37+
'DuckDuckBot',
2738
]);
2839

2940
$services = $container->services();

‎tests/BotDetector/BotDetectorTest.php‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,15 @@ public function it_returns_false_if_no_request_is_provided_and_the_request_does_
4646
self::assertFalse($botDetector->isBotRequest());
4747
}
4848

49+
/**
50+
* @test
51+
*/
52+
public function it_matches_popular_bots_case_insensitively(): void
53+
{
54+
$botDetector = new BotDetector(new RequestStack(), ['Qwertz']);
55+
self::assertTrue($botDetector->isBot('Mozilla/5.0 (compatible; qwertz/1.0)'));
56+
}
57+
4958
/**
5059
* @dataProvider getBots
5160
*

‎tests/data/bots.php‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,38 @@
1414
'Mediapartners-Google',
1515
'AdsBot-Google (+http://www.google.com/adsbot.html)',
1616
'AdsBot-Google-Mobile-Apps',
17+
'Mozilla/5.0 (compatible; GoogleOther)',
1718

1819
// bing
1920
'Mozilla/5.0 (compatible Bingbot/2.0 +http://www.bing.com/bingbot.htm)',
21+
'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',
22+
23+
// openai
24+
'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.1; +https://openai.com/gptbot)',
25+
'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot',
26+
'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot',
27+
28+
// anthropic
29+
'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)',
30+
31+
// perplexity
32+
'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)',
33+
34+
// amazon
35+
'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot) Chrome/119.0.6045.214 Safari/537.36',
36+
37+
// apple
38+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)',
39+
40+
// bytedance
41+
'Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)',
42+
43+
// huawei
44+
'Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)',
45+
46+
// seo
47+
'Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)',
48+
'Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)',
2049

2150
// yahoo
2251
'Mozilla/5.0 (compatible Yahoo! Slurp http://help.yahoo.com/help/us/ysearch/slurp)',
@@ -45,6 +74,8 @@
4574
'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
4675
'facebookexternalhit/1.1',
4776
'facebookcatalog/1.0',
77+
'meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)',
78+
'meta-webindexer/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)',
4879

4980
// alexa
5081
'ia_archiver (+http://www.alexa.com/site/help/webmasters crawler@alexa.com)',

0 commit comments

Comments
 (0)