Invision Community 5: A video walkthrough creating a custom theme and homepage By Matt Thursday at 04:02 PM
TSP Posted May 6 Posted May 6 (edited) I upgraded from version 4.7.13 to 4.7.16 on 19th April, and we have since been having trouble with both reduced performance and more downtime. Our server host has looked into this and reported back a major increase in slow queries, major increase in CPU usage for the database server overall and less usage of MySQL Sorts. It seems this increase can be tracked down to methods using IPS\Content\_Item::getItemsWithPermission:317. We have a custom controller and a frontpage widget utilizing this method. Anyone else experienced this, or have any tips on what could be going on? Hope Invision could be interested in lending a hand investigating this too Best regards, Preben Edited May 6 by TSP SeNioR- 1
Sonya* Posted May 6 Posted May 6 Just now, TSP said: Anyone else experienced this Yes. I have noticed slow performance on my test server. 1 minute ago, TSP said: have any tips No. SeNioR- 1
TSP Posted May 6 Author Posted May 6 31 minutes ago, Sonya* said: Yes. I have noticed slow performance on my test server. Thanks for the feedback 🙂 I presume you have 4.7.16 on your test server, and that your production server is still running an earlier version, which version? Between versions 4.7.13 and 4.7.16 I see a bunch of changes in system/Content/Item.php. However, I don't have a changelog for the in between versions. Do anyone know which versions these changes were introduced? I suspect some of the changes has to do with the change in database server utilization... Here are the diff for the changes in the getItemsWithPermission-method in the file system/Content/Item.php: @@ -3159,11 +3161,16 @@ abstract class _Item extends \IPS\Content { $containerWhere = array_merge( $containerWhere, $value ); unset( $where[ $key ] ); + + /* $containerWhere is used for exclusion purposes now, + so we leave this as part of the where condition as well. */ + $where = array_merge( $where, $value ); } } } /* Exclude hidden items */ + $includeAdditionalApprovalClauses = true; if( $includeHiddenItems === \IPS\Content\Hideable::FILTER_AUTOMATIC ) { $containersTheUserCanViewHiddenItemsIn = static::canViewHiddenItemsContainers( $member ); @@ -3199,6 +3206,8 @@ abstract class _Item extends \IPS\Content $col = static::$databaseTable . '.' . static::$databasePrefix . static::$databaseColumnMap['hidden']; $where[] = array( "{$col}=1" ); } + + $includeAdditionalApprovalClauses = false; } elseif ( \in_array( 'IPS\Content\Hideable', class_implements( \get_called_class() ) ) and $includeHiddenItems !== \IPS\Content\Hideable::FILTER_SHOW_HIDDEN ) { @@ -3226,6 +3235,8 @@ abstract class _Item extends \IPS\Content { $where[] = array( "{$col}=1" ); } + + $includeAdditionalApprovalClauses = false; } elseif ( isset( static::$databaseColumnMap['hidden'] ) ) { @@ -3247,6 +3258,8 @@ abstract class _Item extends \IPS\Content { $where[] = array( "{$col}=0" ); } + + $includeAdditionalApprovalClauses = false; } } else @@ -3274,17 +3287,21 @@ abstract class _Item extends \IPS\Content } } - /* No matter if we can or cannot view hidden items, we do not want these to show: -2 is queued for deletion and -3 is posted before register */ - if ( isset( static::$databaseColumnMap['hidden'] ) ) - { - $col = static::$databaseTable . '.' . static::$databasePrefix . static::$databaseColumnMap['hidden']; - $where[] = array( "{$col}!=-2 AND {$col} !=-3" ); - } - else if ( isset( static::$databaseColumnMap['approved'] ) ) - { - $col = static::$databaseTable . '.' . static::$databasePrefix . static::$databaseColumnMap['approved']; - $where[] = array( "{$col}!=-2 AND {$col}!=-3" ); - } + /* This only makes sense if we have not already filtered by a single value */ + if( $includeAdditionalApprovalClauses ) + { + /* No matter if we can or cannot view hidden items, we do not want these to show: -2 is queued for deletion and -3 is posted before register */ + if ( isset( static::$databaseColumnMap['hidden'] ) ) + { + $col = static::$databaseTable . '.' . static::$databasePrefix . static::$databaseColumnMap['hidden']; + $where[] = array( "{$col}!=-2 AND {$col} !=-3" ); + } + else if ( isset( static::$databaseColumnMap['approved'] ) ) + { + $col = static::$databaseTable . '.' . static::$databasePrefix . static::$databaseColumnMap['approved']; + $where[] = array( "{$col}!=-2 AND {$col}!=-3" ); + } + } /* Future items? */ if ( \in_array( 'IPS\Content\FuturePublishing', class_implements( \get_called_class() ) ) ) @@ -3321,18 +3338,28 @@ abstract class _Item extends \IPS\Content $categories = array(); $lookupKey = md5( $containerClass::$permApp . $containerClass::$permType . $permissionKey . json_encode( $member->groups ) ); + /* EME: Switch to use exclusion instead of inclusion */ if( !isset( static::$permissionSelect[ $lookupKey ] ) ) { static::$permissionSelect[ $lookupKey ] = array(); - $permQuery = \IPS\Db::i()->select( 'perm_type_id', 'core_permission_index', array( "core_permission_index.app='" . $containerClass::$permApp . "' AND core_permission_index.perm_type='" . $containerClass::$permType . "' AND (" . \IPS\Db::i()->findInSet( 'perm_' . $containerClass::$permissionMap[ $permissionKey ], $member->permissionArray() ) . ' OR ' . 'perm_' . $containerClass::$permissionMap[ $permissionKey ] . "='*' )" ) ); + + $permQueryJoinContainer = (bool) ( \count( $containerWhere ) ); + $clubWhere = ""; /* If we cannot access clubs, skip them */ if ( \IPS\IPS::classUsesTrait( $containerClass, 'IPS\Content\ClubContainer' ) AND !$member->canAccessModule( \IPS\Application\Module::get( 'core', 'clubs', 'front' ) ) ) { - $containerWhere[] = array( $containerClass::$databaseTable . '.' . $containerClass::$databasePrefix . $containerClass::clubIdColumn() . ' IS NULL' ); + $permQueryWhere = array( "core_permission_index.app='" . $containerClass::$permApp . "' AND core_permission_index.perm_type='" . $containerClass::$permType . "' AND (" . $containerClass::$databaseTable . '.' . $containerClass::$databasePrefix . $containerClass::clubIdColumn() . " IS NOT NULL OR !(" . \IPS\Db::i()->findInSet( 'perm_' . $containerClass::$permissionMap[ $permissionKey ], $member->permissionArray() ) . ' OR ' . 'perm_' . $containerClass::$permissionMap[ $permissionKey ] . "='*' ))" ); + $permQueryJoinContainer = true; } + else + { + $permQueryWhere = array( "core_permission_index.app='" . $containerClass::$permApp . "' AND core_permission_index.perm_type='" . $containerClass::$permType . "' AND !(" . \IPS\Db::i()->findInSet( 'perm_' . $containerClass::$permissionMap[ $permissionKey ], $member->permissionArray() ) . ' OR ' . 'perm_' . $containerClass::$permissionMap[ $permissionKey ] . "='*' )" ); + } + + $permQuery = \IPS\Db::i()->select( 'perm_type_id', 'core_permission_index', $permQueryWhere ); - if ( \count( $containerWhere ) ) + if( $permQueryJoinContainer ) { $permQuery->join( $containerClass::$databaseTable, array_merge( $containerWhere, array( 'core_permission_index.perm_type_id=' . $containerClass::$databaseTable . '.' . $containerClass::$databasePrefix . $containerClass::$databaseColumnId ) ), 'STRAIGHT_JOIN' ); } @@ -3347,11 +3374,7 @@ abstract class _Item extends \IPS\Content if( \count( $categories ) ) { - $where[] = array( static::$databaseTable . "." . static::$databasePrefix . static::$databaseColumnMap['container'] . ' IN(' . implode( ',', $categories ) . ')' ); - } - else - { - $where[] = array( static::$databaseTable . "." . static::$databasePrefix . static::$databaseColumnMap['container'] . '=0' ); + $where[] = array( static::$databaseTable . "." . static::$databasePrefix . static::$databaseColumnMap['container'] . ' NOT IN (' . implode( ',', $categories ) . ')' ); } } @@ -3363,8 +3386,10 @@ abstract class _Item extends \IPS\Content $select = \IPS\Db::i()->select( 'COUNT(*) as cnt', static::$databaseTable, $where, NULL, NULL, $groupBy, NULL, $queryFlags ); if ( $joinContainer AND isset( static::$containerNodeClass ) ) { + /* EME: Removed the $containerWhere from the join because it is now in the where clause. + We are now using $containerWhere to exclude forums that shouldn't be visible. */ $containerClass = static::$containerNodeClass; - $select->join( $containerClass::$databaseTable, array_merge( $containerWhere, array( static::$databaseTable . '.' . static::$databasePrefix . static::$databaseColumnMap['container'] . '=' . $containerClass::$databaseTable . '.' . $containerClass::$databasePrefix . $containerClass::$databaseColumnId ) ) ); + $select->join( $containerClass::$databaseTable, array( static::$databaseTable . '.' . static::$databasePrefix . static::$databaseColumnMap['container'] . '=' . $containerClass::$databaseTable . '.' . $containerClass::$databasePrefix . $containerClass::$databaseColumnId ) ); } if ( $joinComments ) { Could someone at Invision answer whether it would cause any issues to temporarily revert this method to the previous that worked for us? Or is there a lot of changes in that method dependent on other changes done elsewhere in the same or later version, so I would have to hunt down a lot of other stuff?
Sonya* Posted May 6 Posted May 6 25 minutes ago, TSP said: I presume you have 4.7.16 on your test server, and that your production server is still running an earlier version, which version? They both have the same version. But my test server is not optimized and is poor configured. I manage it myself and have no idea of how to do it properly. 😅 But I have noticed that my custom application on the test server that uses exact the same function is now slow. My live server is well configured by my server admin. So I do not notice any significant difference there. TSP 1
TSP Posted May 6 Author Posted May 6 30 minutes ago, Stuart Silvester said: What version of MySQL/MariaDB are you using? Server version: 10.5.24-MariaDB-1:10.5.24+maria~ubu2004-log mariadb.org binary distribution
NZyan Posted May 7 Posted May 7 I have an issue after the update, that might be related to this one: If the start page is configured to show the „Fluid view“ and no forums are filtered … the performance is dramatically slow. mySQL 8.0.36-28
TSP Posted May 13 Author Posted May 13 @Stuart Silvester does my reply about database version help at all? Is this a known issue that is being worked on?
Stuart Silvester Posted May 13 Posted May 13 We're working on a performance review, we've included this information in that project so we can look further into it and do some further benchmarking. We're not seeing the same slowness on our Cloud infrastructure hence my asking about MySQL/MariaDB versions. TSP and SeNioR- 2
sound Posted May 28 Posted May 28 On 5/13/2024 at 5:48 PM, Stuart Silvester said: We're working on a performance review, we've included this information in that project so we can look further into it and do some further benchmarking. We're not seeing the same slowness on our Cloud infrastructure hence my asking about MySQL/MariaDB versions. reported this a couple of years ago last time checked was still an issue wegorz23 1
Marc Posted May 28 Posted May 28 15 minutes ago, sound said: reported this a couple of years ago last time checked was still an issue There are quite a few things being reviewed in this. Im sure this will be too 🙂
Westfield Sports Car Club Posted May 28 Posted May 28 Ours is also running ridiculously slow, albeit randomly. We're seeing 100% CPU load at times. MySQL version 8.0.36-0ubuntu0.20.04.1 Invision Community v4.7.16 Appears to be related to a search query "IPS\Content\Search\Mysql\_Query::search:152*/ SELECT main.* FROM `core_search_index` AS `main` FORCE INDEX(`index_date_updated`) WHERE <long query here>... media, SeNioR- and Dll 2 1
Dll Posted May 29 Posted May 29 (edited) I'd recommend blocking crawlers from hitting profile pages for a period, to see if that resolves these issues. It appears that the user profiles are very inefficient, particularly on large communities with a lot of posts. While it's useful to allow google to spider them these days, it doesn't appear to be good for performance right now. Edited May 29 by Dll wegorz23 1
Dll Posted May 29 Posted May 29 Just as an example of how nasty the SQL for profile pages is, it's this. SELECT main.* FROM `core_search_index` AS `main` WHERE ( index_class='IPS\\forums\\Topic\\Post' OR ( index_class IN('IPS\\blog\\Entry','IPS\\blog\\Entry\\Comment') ) OR ( index_class IN('IPS\\gallery\\Image','IPS\\gallery\\Image\\Comment','IPS\\gallery\\Image\\Review') ) OR ( index_class IN('IPS\\gallery\\Album\\Item','IPS\\gallery\\Album\\Comment','IPS\\gallery\\Album\\Review') ) OR ( index_class IN('IPS\\calendar\\Event','IPS\\calendar\\Event\\Comment','IPS\\calendar\\Event\\Review') ) OR index_class='IPS\\cms\\Pages\\PageItem' OR ( index_class IN('IPS\\cms\\Records1','IPS\\cms\\Records\\Comment1','IPS\\cms\\Records\\Review1') ) OR ( index_class IN('IPS\\cms\\Records2','IPS\\cms\\Records\\Comment2','IPS\\cms\\Records\\Review2') ) ) AND index_author=94 AND ( index_container_class IS NULL OR index_container_class NOT IN('IPS\\gallery\\Album') OR index_class IN('IPS\\gallery\\Image\\Comment','IPS\\gallery\\Image\\Review') ) AND ( index_permissions = '*' OR ( FIND_IN_SET(2,index_permissions) ) ) AND index_hidden=0 ORDER BY index_date_created DESC LIMIT 0,15 Bearing in mind someone may have 100's of pages of content on their profile with google etc wanting to crawl every single one, if the crawler is allowed. Performant it isn't.
Management Matt Posted May 29 Management Posted May 29 That's a pretty benign query that won't cause any issues.
Dll Posted May 29 Posted May 29 31 minutes ago, Matt said: That's a pretty benign query that won't cause any issues. Just for some context, a user or users calling that by going to profile pages and scrolling through a few pages of content isn't a particular issue despite the SQL being inefficient. But now search engine crawlers are going to profiles as well, consuming up to hundreds of these pages a minute, going through each page of content for each user, then the inefficiency is magnified. It's scanning potentially millions of rows each time, some of which aren't indexed well or at all in amongst a fairly complex set of clauses
Management Matt Posted May 29 Management Posted May 29 The change to allow spiders to crawl profiles was approved in November 2023, and made it's way into the 4.7.15 release. What evidence is there to say that search engine crawling is "consuming hundreds of pages a minute"? It's just guessing. Most search engines do not operate at that speed and they do not try and pagination through everything. The first thing I'd suggest is to set up a CDN for guests with a 5 to 15 minute retention. That'll ensure that once a page has been generated, other guests and spiders will get the CDN version and not touch the database or PHP. That's what we do on cloud.
Management Matt Posted May 29 Management Posted May 29 19 hours ago, Westfield Sports Car Club said: Ours is also running ridiculously slow, albeit randomly. We're seeing 100% CPU load at times. MySQL version 8.0.36-0ubuntu0.20.04.1 Invision Community v4.7.16 Appears to be related to a search query "IPS\Content\Search\Mysql\_Query::search:152*/ SELECT main.* FROM `core_search_index` AS `main` FORCE INDEX(`index_date_updated`) WHERE <long query here>... Can you get the full query? I'd run it with EXPLAIN to see the output, and then remove the FORCE INDEX and run EXPLAIN again. We do not use the FORCE INDEX on cloud running MySQL 8 as it is inefficient now. I'd be interested to see if that helps or hinders. On 5/6/2024 at 3:39 PM, TSP said: Server version: 10.5.24-MariaDB-1:10.5.24+maria~ubu2004-log mariadb.org binary distribution Do you have topics with thousands of pages of posts? We did remove an old optimisation that helped with older versions of MySQL but caused issues with. MySQL 8.
Dll Posted May 29 Posted May 29 23 hours ago, Matt said: What evidence is there to say that search engine crawling is "consuming hundreds of pages a minute"? It's just guessing. Most search engines do not operate at that speed and they do not try and pagination through everything. Errm, I'm not guessing. I've been viewing it with my own eyes this morning as a combo of bing, google, yandex and a bunch of ai bots have been hitting those (and other) pages, peaking in the hundreds per minute at times on the profile pages. Since we migrated back to self-hosted from the cloud, it's been quite eye-opening to see what's going on under the hood again. This being one of the things that have come up.
Westfield Sports Car Club Posted May 29 Posted May 29 58 minutes ago, Matt said: Can you get the full query? I'd run it with EXPLAIN to see the output, and then remove the FORCE INDEX and run EXPLAIN again. We do not use the FORCE INDEX on cloud running MySQL 8 as it is inefficient now. I'd be interested to see if that helps or hinders. Removing FORCE INDEX makes a massive difference! I ran the full query manually and it just hung for over 5 mins, then without FORCE INDEX it ran in an instant - with only a few results though. EXPLAIN output attached as CSV forum_WSCC Forum_29-05-2024@12-34.csv
Management Matt Posted May 29 Management Posted May 29 17 minutes ago, Westfield Sports Car Club said: Removing FORCE INDEX makes a massive difference! I ran the full query manually and it just hung for over 5 mins, then without FORCE INDEX it ran in an instant - with only a few results though. EXPLAIN output attached as CSV forum_WSCC Forum_29-05-2024@12-34.csv 498 B · 4 downloads OK, I'll drop a plug-in for you to try shortly that removes it. Then you can see if that helps. Westfield Sports Car Club 1
Dll Posted May 29 Posted May 29 1 hour ago, Matt said: Most search engines do not operate at that speed and they do not try and pagination through everything. So, this is one very small example of a search engine spider, cycling through pages within a profile. I've redacted some things, and as I say it's a small snippet of hundreds of requests over a short space of time. [ { "action": "managed_challenge", "clientASNDescription": "HWCLOUDS-AS-AP HUAWEI CLOUDS", "clientAsn": "136907", "clientCountryName": "SG", "clientIP": "114.119.159.17", "clientRequestHTTPHost": "[redact]", "clientRequestHTTPMethodName": "GET", "clientRequestHTTPProtocol": "HTTP/1.1", "clientRequestPath": "/profile/[redact]", "clientRequestQuery": "?status=19529&type=status&page=180&dir=next", "datetime": "2024-05-29T12:05:30Z", "rayName": "88b65a5fca693dc9", "ref": "", "ruleId": "cc2a31b194ad401d9b56ad3b96181c77", "rulesetId": "14c2060af910455eb569f0e6734d7225", "source": "firewallCustom", "userAgent": "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "wafAttackScoreClass": "clean", "matchIndex": 0, "metadata": [ { "key": "ruleset_version", "value": "64" }, { "key": "version", "value": "13" }, { "key": "type", "value": "customer" } ], "sampleInterval": 1 }, { "action": "managed_challenge", "clientASNDescription": "HWCLOUDS-AS-AP HUAWEI CLOUDS", "clientAsn": "136907", "clientCountryName": "SG", "clientIP": "114.119.159.40", "clientRequestHTTPHost": "[redact]", "clientRequestHTTPMethodName": "GET", "clientRequestHTTPProtocol": "HTTP/1.1", "clientRequestPath": "/profile/[redact]", "clientRequestQuery": "?status=19625&type=status&page=4&dir=prev", "datetime": "2024-05-29T12:05:23Z", "rayName": "88b65a334c539e3e", "ref": "", "ruleId": "cc2a31b194ad401d9b56ad3b96181c77", "rulesetId": "14c2060af910455eb569f0e6734d7225", "source": "firewallCustom", "userAgent": "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "wafAttackScoreClass": "clean", "matchIndex": 0, "metadata": [ { "key": "ruleset_version", "value": "64" }, { "key": "version", "value": "13" }, { "key": "type", "value": "customer" } ], "sampleInterval": 1 }, { "action": "managed_challenge", "clientASNDescription": "HWCLOUDS-AS-AP HUAWEI CLOUDS", "clientAsn": "136907", "clientCountryName": "SG", "clientIP": "114.119.152.148", "clientRequestHTTPHost": "[redact]", "clientRequestHTTPMethodName": "GET", "clientRequestHTTPProtocol": "HTTP/1.1", "clientRequestPath": "/profile/[redact]", "clientRequestQuery": "?status=19691&type=status&page=30&dir=next", "datetime": "2024-05-29T12:05:19Z", "rayName": "88b65a1d6cbb3d37", "ref": "", "ruleId": "cc2a31b194ad401d9b56ad3b96181c77", "rulesetId": "14c2060af910455eb569f0e6734d7225", "source": "firewallCustom", "userAgent": "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "wafAttackScoreClass": "clean", "matchIndex": 0, "metadata": [ { "key": "ruleset_version", "value": "64" }, { "key": "version", "value": "13" }, { "key": "type", "value": "customer" } ], "sampleInterval": 1 }, { "action": "managed_challenge", "clientASNDescription": "HWCLOUDS-AS-AP HUAWEI CLOUDS", "clientAsn": "136907", "clientCountryName": "SG", "clientIP": "114.119.137.184", "clientRequestHTTPHost": "[redact]", "clientRequestHTTPMethodName": "GET", "clientRequestHTTPProtocol": "HTTP/1.1", "clientRequestPath": "/profile/[redact]", "clientRequestQuery": "?status=20010&type=status&page=5&dir=next", "datetime": "2024-05-29T12:05:12Z", "rayName": "88b659f29af83d80", "ref": "", "ruleId": "cc2a31b194ad401d9b56ad3b96181c77", "rulesetId": "14c2060af910455eb569f0e6734d7225", "source": "firewallCustom", "userAgent": "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "wafAttackScoreClass": "clean", "matchIndex": 0, "metadata": [ { "key": "ruleset_version", "value": "64" }, { "key": "version", "value": "13" }, { "key": "type", "value": "customer" } ], "sampleInterval": 1 }, { "action": "managed_challenge", "clientASNDescription": "HWCLOUDS-AS-AP HUAWEI CLOUDS", "clientAsn": "136907", "clientCountryName": "SG", "clientIP": "114.119.136.30", "clientRequestHTTPHost": "[redact]", "clientRequestHTTPMethodName": "GET", "clientRequestHTTPProtocol": "HTTP/1.1", "clientRequestPath": "/profile/[redact]", "clientRequestQuery": "?status=20083&type=status&page=27&dir=next", "datetime": "2024-05-29T12:05:05Z", "rayName": "88b659c59b07408f", "ref": "", "ruleId": "cc2a31b194ad401d9b56ad3b96181c77", "rulesetId": "14c2060af910455eb569f0e6734d7225", "source": "firewallCustom", "userAgent": "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "wafAttackScoreClass": "clean", "matchIndex": 0, "metadata": [ { "key": "ruleset_version", "value": "64" }, { "key": "version", "value": "13" }, { "key": "type", "value": "customer" } ], "sampleInterval": 1 }, { "action": "managed_challenge", "clientASNDescription": "HWCLOUDS-AS-AP HUAWEI CLOUDS", "clientAsn": "136907", "clientCountryName": "SG", "clientIP": "114.119.148.138", "clientRequestHTTPHost": "[redact]", "clientRequestHTTPMethodName": "GET", "clientRequestHTTPProtocol": "HTTP/1.1", "clientRequestPath": "/profile/[redact]", "clientRequestQuery": "?status=20083&type=status&page=47&dir=next", "datetime": "2024-05-29T12:04:58Z", "rayName": "88b6599b298d8bc8", "ref": "", "ruleId": "cc2a31b194ad401d9b56ad3b96181c77", "rulesetId": "14c2060af910455eb569f0e6734d7225", "source": "firewallCustom", "userAgent": "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "wafAttackScoreClass": "clean", "matchIndex": 0, "metadata": [ { "key": "ruleset_version", "value": "64" }, { "key": "version", "value": "13" }, { "key": "type", "value": "customer" } ], "sampleInterval": 1 }, { "action": "managed_challenge", "clientASNDescription": "HWCLOUDS-AS-AP HUAWEI CLOUDS", "clientAsn": "136907", "clientCountryName": "SG", "clientIP": "114.119.146.185", "clientRequestHTTPHost": "[redact]", "clientRequestHTTPMethodName": "GET", "clientRequestHTTPProtocol": "HTTP/1.1", "clientRequestPath": "/profile/[redact]", "clientRequestQuery": "?status=20127&type=status&page=44&dir=next", "datetime": "2024-05-29T12:04:55Z", "rayName": "88b65985cf2a9fd3", "ref": "", "ruleId": "cc2a31b194ad401d9b56ad3b96181c77", "rulesetId": "14c2060af910455eb569f0e6734d7225", "source": "firewallCustom", "userAgent": "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "wafAttackScoreClass": "clean", "matchIndex": 0, "metadata": [ { "key": "ruleset_version", "value": "64" }, { "key": "version", "value": "13" }, { "key": "type", "value": "customer" } ], "sampleInterval": 10 }, { "action": "managed_challenge", "clientASNDescription": "HWCLOUDS-AS-AP HUAWEI CLOUDS", "clientAsn": "136907", "clientCountryName": "SG", "clientIP": "114.119.153.118", "clientRequestHTTPHost": "[redact]", "clientRequestHTTPMethodName": "GET", "clientRequestHTTPProtocol": "HTTP/1.1", "clientRequestPath": "/profile/[redact]", "clientRequestQuery": "?status=20271&type=status&page=37&dir=next", "datetime": "2024-05-29T12:04:51Z", "rayName": "88b6596cc9dc2ee3", "ref": "", "ruleId": "cc2a31b194ad401d9b56ad3b96181c77", "rulesetId": "14c2060af910455eb569f0e6734d7225", "source": "firewallCustom", "userAgent": "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "wafAttackScoreClass": "clean", "matchIndex": 0, "metadata": [ { "key": "ruleset_version", "value": "64" }, { "key": "version", "value": "13" }, { "key": "type", "value": "customer" } ], "sampleInterval": 1 },
Management Matt Posted May 29 Management Posted May 29 Personally, I'd just block the bot at a server level (a good CDN can do this). It does not respect robots.txt, so would have ignored the disallow profiles. https://www.hypernode.com/en/blog/huawei-aspiegelbot-is-increasingly-impacting-european-online-stores/
Stuart Silvester Posted May 29 Posted May 29 Yes, I've had the same experience with Huawei on my own projects. I WAF block their entire netblocks now. Matt 1
Recommended Posts