count( $urls ), 'sources' => $sources ); } protected static function sitemaps_from_robots() { $response = wp_remote_get( home_url( '/robots.txt' ), array( 'timeout' => 10 ) ); if ( is_wp_error( $response ) || 200 !== wp_remote_retrieve_response_code( $response ) ) { return array(); } $body = wp_remote_retrieve_body( $response ); $out = array(); foreach ( preg_split( '/\r\n|\r|\n/', $body ) as $line ) { if ( 0 === stripos( trim( $line ), 'Sitemap:' ) ) { $out[] = trim( substr( trim( $line ), strlen( 'Sitemap:' ) ) ); } } return array_slice( $out, 0, 10 ); } protected static function fetch_sitemap( $url, $depth = 0 ) { $response = wp_remote_get( $url, array( 'timeout' => 15 ) ); if ( is_wp_error( $response ) || 200 !== wp_remote_retrieve_response_code( $response ) ) { return false; } $body = wp_remote_retrieve_body( $response ); if ( '' === trim( $body ) ) { return false; } $needs_legacy_guard = PHP_VERSION_ID < 80000; $previous = $needs_legacy_guard ? libxml_disable_entity_loader( true ) : null; // phpcs:ignore libxml_use_internal_errors( true ); $xml = simplexml_load_string( $body, 'SimpleXMLElement', LIBXML_NONET ); libxml_clear_errors(); if ( $needs_legacy_guard ) { libxml_disable_entity_loader( $previous ); // phpcs:ignore } if ( false === $xml ) { return false; } $urls = array(); if ( isset( $xml->sitemap ) && $depth < 1 ) { $i = 0; foreach ( $xml->sitemap as $child ) { if ( ++$i > self::MAX_SITEMAP_FILES ) { break; } $loc = (string) ( $child->loc ?? '' ); if ( $loc ) { $child_urls = self::fetch_sitemap( $loc, $depth + 1 ); if ( is_array( $child_urls ) ) { $urls = array_merge( $urls, $child_urls ); } } if ( count( $urls ) >= self::MAX_URLS ) { break; } } return $urls; } if ( isset( $xml->url ) ) { foreach ( $xml->url as $entry ) { $loc = (string) ( $entry->loc ?? '' ); if ( $loc ) { $urls[] = $loc; } if ( count( $urls ) >= self::MAX_URLS ) { break; } } return $urls; } return false; } protected static function store_discovered( array $urls, array $sources ) { global $wpdb; $table = Argus_DB::table( 'cache_discovered_urls' ); $now = current_time( 'mysql', true ); foreach ( $urls as $url ) { $url = esc_url_raw( $url ); if ( ! $url ) { continue; } $classification = Argus_Cache_Eligibility::classify_url( $url ); $source = ''; foreach ( $sources as $s ) { if ( false !== strpos( $url, wp_parse_url( home_url(), PHP_URL_HOST ) ) ) { $source = $s; break; } } $wpdb->query( $wpdb->prepare( "INSERT INTO {$table} (url, source, eligibility, eligibility_reason, discovered_at, last_checked_at) VALUES (%s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE eligibility = VALUES(eligibility), eligibility_reason = VALUES(eligibility_reason), last_checked_at = VALUES(last_checked_at)", // phpcs:ignore $url, $source ?: 'sitemap', $classification['status'], $classification['reason'], $now, $now ) ); } } public static function summary() { global $wpdb; $table = Argus_DB::table( 'cache_discovered_urls' ); $rows = $wpdb->get_results( "SELECT eligibility, COUNT(*) AS cnt FROM {$table} GROUP BY eligibility" ); // phpcs:ignore $by_status = array(); foreach ( $rows as $row ) { $by_status[ $row->eligibility ] = (int) $row->cnt; } return array( 'total' => array_sum( $by_status ), 'eligible' => $by_status[ Argus_Cache_Eligibility::SAFE_TO_CACHE ] ?? 0, 'excluded' => ( $by_status[ Argus_Cache_Eligibility::FORCE_EXCLUDED ] ?? 0 ) + ( $by_status[ Argus_Cache_Eligibility::BYPASS ] ?? 0 ), 'needs_review' => $by_status[ Argus_Cache_Eligibility::NEEDS_REVIEW ] ?? 0, 'last_run' => get_option( self::LAST_RUN_OPTION, '' ), 'last_error' => get_option( self::LAST_ERROR_OPTION, '' ), ); } public static function paginated_urls( $page = 1, $eligibility = null, $per_page = 20 ) { global $wpdb; $table = Argus_DB::table( 'cache_discovered_urls' ); $page = max( 1, (int) $page ); $offset = ( $page - 1 ) * $per_page; $where = $eligibility ? $wpdb->prepare( 'WHERE eligibility = %s', $eligibility ) : ''; // phpcs:ignore $total = (int) $wpdb->get_var( "SELECT COUNT(*) FROM {$table} {$where}" ); // phpcs:ignore $rows = $wpdb->get_results( $wpdb->prepare( "SELECT * FROM {$table} {$where} ORDER BY id DESC LIMIT %d OFFSET %d", $per_page, $offset ) // phpcs:ignore ); return array( 'rows' => $rows, 'total' => $total, 'page' => $page, 'total_pages' => max( 1, (int) ceil( $total / $per_page ) ), ); } }