argus-wp-defence/includes/class-argus-cache-discovery.php

<?php

if ( ! defined( 'ABSPATH' ) ) {
	exit;
}

class Argus_Cache_Discovery {

	const MAX_URLS          = 5000;
	const MAX_SITEMAP_FILES = 50;
	const LAST_RUN_OPTION   = 'argus_wpd_cache_discovery_last_run';
	const LAST_ERROR_OPTION = 'argus_wpd_cache_discovery_last_error';

	public static function run_discovery() {
		delete_option( self::LAST_ERROR_OPTION );
		$urls = array();
		$sources = array();

		foreach ( array( '/wp-sitemap.xml', '/sitemap_index.xml', '/sitemap.xml' ) as $path ) {
			$found = self::fetch_sitemap( home_url( $path ) );
			if ( false !== $found ) {
				$sources[] = $path;
				$urls = array_merge( $urls, $found );
				break; 
			}
		}

		foreach ( self::sitemaps_from_robots() as $sitemap_url ) {
			$found = self::fetch_sitemap( $sitemap_url );
			if ( false !== $found ) {
				$sources[] = $sitemap_url;
				$urls = array_merge( $urls, $found );
			}
		}

		$urls = array_slice( array_unique( $urls ), 0, self::MAX_URLS );

		if ( empty( $urls ) ) {
			update_option( self::LAST_ERROR_OPTION, __( 'No sitemap could be found (checked wp-sitemap.xml, sitemap_index.xml, sitemap.xml, and robots.txt).', 'argus-wordpress-defence' ), false );
		}

		self::store_discovered( $urls, $sources );
		update_option( self::LAST_RUN_OPTION, current_time( 'mysql', true ), false );

		return array( 'url_count' => count( $urls ), 'sources' => $sources );
	}

	protected static function sitemaps_from_robots() {
		$response = wp_remote_get( home_url( '/robots.txt' ), array( 'timeout' => 10 ) );
		if ( is_wp_error( $response ) || 200 !== wp_remote_retrieve_response_code( $response ) ) {
			return array();
		}
		$body = wp_remote_retrieve_body( $response );
		$out  = array();
		foreach ( preg_split( '/\r\n|\r|\n/', $body ) as $line ) {
			if ( 0 === stripos( trim( $line ), 'Sitemap:' ) ) {
				$out[] = trim( substr( trim( $line ), strlen( 'Sitemap:' ) ) );
			}
		}
		return array_slice( $out, 0, 10 );
	}

	protected static function fetch_sitemap( $url, $depth = 0 ) {
		$response = wp_remote_get( $url, array( 'timeout' => 15 ) );
		if ( is_wp_error( $response ) || 200 !== wp_remote_retrieve_response_code( $response ) ) {
			return false;
		}
		$body = wp_remote_retrieve_body( $response );
		if ( '' === trim( $body ) ) {
			return false;
		}

		$needs_legacy_guard = PHP_VERSION_ID < 80000;
		$previous = $needs_legacy_guard ? libxml_disable_entity_loader( true ) : null; // phpcs:ignore
		libxml_use_internal_errors( true );
		$xml = simplexml_load_string( $body, 'SimpleXMLElement', LIBXML_NONET );
		libxml_clear_errors();
		if ( $needs_legacy_guard ) {
			libxml_disable_entity_loader( $previous ); // phpcs:ignore
		}

		if ( false === $xml ) {
			return false;
		}

		$urls = array();

		if ( isset( $xml->sitemap ) && $depth < 1 ) {
			$i = 0;
			foreach ( $xml->sitemap as $child ) {
				if ( ++$i > self::MAX_SITEMAP_FILES ) {
					break;
				}
				$loc = (string) ( $child->loc ?? '' );
				if ( $loc ) {
					$child_urls = self::fetch_sitemap( $loc, $depth + 1 );
					if ( is_array( $child_urls ) ) {
						$urls = array_merge( $urls, $child_urls );
					}
				}
				if ( count( $urls ) >= self::MAX_URLS ) {
					break;
				}
			}
			return $urls;
		}

		if ( isset( $xml->url ) ) {
			foreach ( $xml->url as $entry ) {
				$loc = (string) ( $entry->loc ?? '' );
				if ( $loc ) {
					$urls[] = $loc;
				}
				if ( count( $urls ) >= self::MAX_URLS ) {
					break;
				}
			}
			return $urls;
		}

		return false;
	}

	protected static function store_discovered( array $urls, array $sources ) {
		global $wpdb;
		$table = Argus_DB::table( 'cache_discovered_urls' );
		$now   = current_time( 'mysql', true );

		foreach ( $urls as $url ) {
			$url = esc_url_raw( $url );
			if ( ! $url ) {
				continue;
			}
			$classification = Argus_Cache_Eligibility::classify_url( $url );
			$source = '';
			foreach ( $sources as $s ) {
				if ( false !== strpos( $url, wp_parse_url( home_url(), PHP_URL_HOST ) ) ) {
					$source = $s;
					break;
				}
			}

			$wpdb->query(
				$wpdb->prepare(
					"INSERT INTO {$table} (url, source, eligibility, eligibility_reason, discovered_at, last_checked_at)
					VALUES (%s, %s, %s, %s, %s, %s)
					ON DUPLICATE KEY UPDATE eligibility = VALUES(eligibility), eligibility_reason = VALUES(eligibility_reason), last_checked_at = VALUES(last_checked_at)", // phpcs:ignore
					$url,
					$source ?: 'sitemap',
					$classification['status'],
					$classification['reason'],
					$now,
					$now
				)
			);
		}
	}

	public static function summary() {
		global $wpdb;
		$table = Argus_DB::table( 'cache_discovered_urls' );

		$rows = $wpdb->get_results( "SELECT eligibility, COUNT(*) AS cnt FROM {$table} GROUP BY eligibility" ); // phpcs:ignore
		$by_status = array();
		foreach ( $rows as $row ) {
			$by_status[ $row->eligibility ] = (int) $row->cnt;
		}

		return array(
			'total'          => array_sum( $by_status ),
			'eligible'       => $by_status[ Argus_Cache_Eligibility::SAFE_TO_CACHE ] ?? 0,
			'excluded'       => ( $by_status[ Argus_Cache_Eligibility::FORCE_EXCLUDED ] ?? 0 ) + ( $by_status[ Argus_Cache_Eligibility::BYPASS ] ?? 0 ),
			'needs_review'   => $by_status[ Argus_Cache_Eligibility::NEEDS_REVIEW ] ?? 0,
			'last_run'       => get_option( self::LAST_RUN_OPTION, '' ),
			'last_error'     => get_option( self::LAST_ERROR_OPTION, '' ),
		);
	}

	public static function paginated_urls( $page = 1, $eligibility = null, $per_page = 20 ) {
		global $wpdb;
		$table  = Argus_DB::table( 'cache_discovered_urls' );
		$page   = max( 1, (int) $page );
		$offset = ( $page - 1 ) * $per_page;

		$where  = $eligibility ? $wpdb->prepare( 'WHERE eligibility = %s', $eligibility ) : ''; // phpcs:ignore

		$total = (int) $wpdb->get_var( "SELECT COUNT(*) FROM {$table} {$where}" ); // phpcs:ignore
		$rows  = $wpdb->get_results(
			$wpdb->prepare( "SELECT * FROM {$table} {$where} ORDER BY id DESC LIMIT %d OFFSET %d", $per_page, $offset ) // phpcs:ignore
		);

		return array(
			'rows'        => $rows,
			'total'       => $total,
			'page'        => $page,
			'total_pages' => max( 1, (int) ceil( $total / $per_page ) ),
		);
	}
}