<?php
declare(strict_types=1);

/**
 * A deliberately polite HTTP client for checking competitor pages.
 *
 * These are public marketing sites belonging to other companies, so the fetcher
 * identifies itself honestly, obeys robots.txt, waits between requests to the
 * same host, and sends conditional requests so an unchanged page costs the other
 * side almost nothing. None of that is optional politeness - a crawler that
 * hammers a defence contractor's website is a good way to get your address
 * blocked, or noticed.
 */
final class Fetcher
{
    public const MAX_BYTES     = 3_000_000;   // stop reading past this
    public const TIMEOUT       = 20;
    public const CONNECT_TIMEOUT = 8;
    public const MAX_REDIRECTS = 4;
    public const HOST_DELAY_MS = 1500;        // minimum gap between hits on one host

    /** @var array<string, float> host => unix timestamp of last request */
    private static array $lastHit = [];

    /** @var array<string, array{allow: array<string>, disallow: array<string>, delay: float}> */
    private static array $robots = [];

    public static function userAgent(): string
    {
        return 'CompetitorPortal/1.0 (+internal competitive research; contact IT)';
    }

    /**
     * Fetch a URL.
     *
     * @return array{
     *   ok: bool, status: int|null, body: string, headers: array<string,string>,
     *   final_url: string, error: string|null, not_modified: bool
     * }
     */
    public static function get(string $url, ?string $etag = null, ?string $lastModified = null): array
    {
        $result = [
            'ok' => false, 'status' => null, 'body' => '', 'headers' => [],
            'final_url' => $url, 'error' => null, 'not_modified' => false,
        ];

        if (!filter_var($url, FILTER_VALIDATE_URL) || !preg_match('#^https?://#i', $url)) {
            $result['error'] = 'Not a fetchable http(s) URL.';
            return $result;
        }

        if (!self::robotsAllows($url)) {
            $result['error'] = 'Disallowed by robots.txt';
            return $result;
        }

        self::waitForHost($url);

        $headers = [
            'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language: en;q=0.9,he;q=0.8',
        ];
        if ($etag !== null && $etag !== '') {
            $headers[] = 'If-None-Match: ' . $etag;
        }
        if ($lastModified !== null && $lastModified !== '') {
            $headers[] = 'If-Modified-Since: ' . $lastModified;
        }

        $responseHeaders = [];
        $ch = curl_init();

        curl_setopt_array($ch, [
            CURLOPT_URL            => $url,
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_MAXREDIRS      => self::MAX_REDIRECTS,
            CURLOPT_TIMEOUT        => self::TIMEOUT,
            CURLOPT_CONNECTTIMEOUT => self::CONNECT_TIMEOUT,
            CURLOPT_USERAGENT      => self::userAgent(),
            CURLOPT_HTTPHEADER     => $headers,
            CURLOPT_ENCODING       => '',            // accept gzip/deflate
            CURLOPT_SSL_VERIFYPEER => true,
            CURLOPT_SSL_VERIFYHOST => 2,
            CURLOPT_PROTOCOLS_STR  => 'http,https',  // never follow into file:// or ftp://
            CURLOPT_HEADERFUNCTION => static function ($ch, string $line) use (&$responseHeaders) {
                $parts = explode(':', $line, 2);
                if (count($parts) === 2) {
                    $responseHeaders[strtolower(trim($parts[0]))] = trim($parts[1]);
                }
                return strlen($line);
            },
            // Abort a response that turns out to be enormous.
            CURLOPT_NOPROGRESS     => false,
            CURLOPT_PROGRESSFUNCTION => static function ($res, $downloadSize, $downloaded) {
                return $downloaded > self::MAX_BYTES ? 1 : 0;
            },
        ]);

        // PROTOCOLS_STR needs curl 7.85+; fall back quietly on older builds.
        if (!defined('CURLOPT_PROTOCOLS_STR')) {
            curl_setopt($ch, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
        }

        $body   = curl_exec($ch);
        $status = (int) curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
        $final  = (string) curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
        $errNo  = curl_errno($ch);
        $errStr = curl_error($ch);
        curl_close($ch);

        self::$lastHit[self::host($url)] = microtime(true);

        if ($errNo !== 0) {
            $result['error'] = $errNo === CURLE_ABORTED_BY_CALLBACK
                ? 'Response exceeded ' . number_format(self::MAX_BYTES) . ' bytes.'
                : 'curl(' . $errNo . '): ' . $errStr;
            $result['status'] = $status ?: null;
            return $result;
        }

        $result['status']    = $status;
        $result['headers']   = $responseHeaders;
        $result['final_url'] = $final;

        if ($status === 304) {
            $result['ok'] = true;
            $result['not_modified'] = true;
            return $result;
        }

        if ($status < 200 || $status >= 400) {
            $result['error'] = 'HTTP ' . $status;
            return $result;
        }

        $result['ok']   = true;
        $result['body'] = is_string($body) ? $body : '';
        return $result;
    }

    // -----------------------------------------------------------------
    // robots.txt
    // -----------------------------------------------------------------

    public static function robotsAllows(string $url): bool
    {
        $host = self::host($url);
        if ($host === '') {
            return false;
        }

        if (!isset(self::$robots[$host])) {
            self::$robots[$host] = self::loadRobots($url);
        }

        $rules = self::$robots[$host];
        $path  = parse_url($url, PHP_URL_PATH) ?: '/';

        // Longest matching rule wins, as the standard says.
        $bestAllow = -1;
        $bestDeny  = -1;
        foreach ($rules['allow'] as $rule) {
            if ($rule !== '' && str_starts_with($path, $rule) && strlen($rule) > $bestAllow) {
                $bestAllow = strlen($rule);
            }
        }
        foreach ($rules['disallow'] as $rule) {
            if ($rule !== '' && str_starts_with($path, $rule) && strlen($rule) > $bestDeny) {
                $bestDeny = strlen($rule);
            }
        }

        return $bestDeny <= $bestAllow;
    }

    /** @return array{allow: array<string>, disallow: array<string>, delay: float} */
    private static function loadRobots(string $url): array
    {
        $empty = ['allow' => [], 'disallow' => [], 'delay' => 0.0];

        $parts = parse_url($url);
        if (!isset($parts['scheme'], $parts['host'])) {
            return $empty;
        }
        $robotsUrl = $parts['scheme'] . '://' . $parts['host']
                   . (isset($parts['port']) ? ':' . $parts['port'] : '') . '/robots.txt';

        $ch = curl_init();
        curl_setopt_array($ch, [
            CURLOPT_URL            => $robotsUrl,
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_TIMEOUT        => 10,
            CURLOPT_CONNECTTIMEOUT => 5,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_MAXREDIRS      => 2,
            CURLOPT_USERAGENT      => self::userAgent(),
            CURLOPT_SSL_VERIFYPEER => true,
        ]);
        $body   = curl_exec($ch);
        $status = (int) curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
        curl_close($ch);

        // No robots.txt, or an error fetching it, means no restrictions.
        if (!is_string($body) || $status !== 200) {
            return $empty;
        }

        $rules    = $empty;
        $applies  = false;

        foreach (preg_split('/\R/', $body) ?: [] as $line) {
            $line = trim(preg_replace('/#.*$/', '', $line) ?? '');
            if ($line === '' || !str_contains($line, ':')) {
                continue;
            }
            [$field, $value] = array_map('trim', explode(':', $line, 2));
            $field = strtolower($field);

            if ($field === 'user-agent') {
                // Only the wildcard group concerns us; we are not named anywhere.
                $applies = ($value === '*');
                continue;
            }
            if (!$applies) {
                continue;
            }
            if ($field === 'disallow' && $value !== '') {
                $rules['disallow'][] = $value;
            } elseif ($field === 'allow' && $value !== '') {
                $rules['allow'][] = $value;
            } elseif ($field === 'crawl-delay') {
                $rules['delay'] = max(0.0, (float) $value);
            }
        }

        return $rules;
    }

    // -----------------------------------------------------------------
    // rate limiting
    // -----------------------------------------------------------------

    private static function waitForHost(string $url): void
    {
        $host = self::host($url);
        $delayMs = self::HOST_DELAY_MS;

        if (isset(self::$robots[$host]) && self::$robots[$host]['delay'] > 0) {
            $delayMs = max($delayMs, (int) (self::$robots[$host]['delay'] * 1000));
        }

        if (!isset(self::$lastHit[$host])) {
            return;
        }

        $elapsedMs = (microtime(true) - self::$lastHit[$host]) * 1000;
        if ($elapsedMs < $delayMs) {
            usleep((int) (($delayMs - $elapsedMs) * 1000));
        }
    }

    private static function host(string $url): string
    {
        return strtolower((string) (parse_url($url, PHP_URL_HOST) ?: ''));
    }
}
