<?php
declare(strict_types=1);

/**
 * The crawl itself, shared by the cron script and the "Run now" button.
 *
 * A web request cannot run for minutes, so the work is done in bounded batches:
 * the caller gives a time budget and a link cap, and the crawler stops cleanly
 * at whichever comes first, leaving the rest of the queue for the next call.
 */
final class Crawler
{
    /** How much of a page must move before it is worth an analyst's attention. */
    public const MIN_CHANGE_RATIO  = 0.005;   // half a percent of the page
    public const MIN_CHANGED_WORDS = 6;
    /** Snapshots kept per link, so you can see what a page looked like before. */
    public const KEEP_SNAPSHOTS    = 6;
    /** Give up on a link after this many consecutive failures. */
    public const MAX_FAILURES      = 8;

    private static $lock = null;

    /**
     * @param array{
     *   limit?: int, company?: int|null, min_age_hours?: int, dry_run?: bool,
     *   time_budget?: int|null, on_line?: callable|null
     * } $options
     *
     * @return array{
     *   checked: int, changed: int, errors: int, feed_items: int,
     *   remaining: int, stopped_early: bool, run_id: int|null, skipped: string|null
     * }
     */
    public static function run(array $options = []): array
    {
        $limit      = max(1, (int) ($options['limit'] ?? 500));
        $company    = $options['company'] ?? null;
        $minAge     = max(0, (int) ($options['min_age_hours'] ?? 20));
        $dryRun     = (bool) ($options['dry_run'] ?? false);
        $budget     = $options['time_budget'] ?? null;      // seconds, null = no limit
        $onLine     = $options['on_line'] ?? null;

        $summary = [
            'checked' => 0, 'changed' => 0, 'errors' => 0, 'feed_items' => 0,
            'remaining' => 0, 'stopped_early' => false, 'run_id' => null, 'skipped' => null,
        ];

        $say = static function (string $msg, string $level = 'info') use ($onLine): void {
            if ($onLine !== null) {
                $onLine($msg, $level);
            }
        };

        // Only one crawl at a time: a manual run must not race the nightly cron.
        if (!self::acquireLock()) {
            $summary['skipped'] = 'Another crawl is already running.';
            return $summary;
        }

        try {
            $queue = self::queue($limit, $company, $minAge);

            if ($queue === []) {
                $summary['skipped'] = 'Nothing is due.';
                return $summary;
            }

            $started = microtime(true);

            if (!$dryRun) {
                Database::run('INSERT INTO fetch_runs () VALUES ()');
                $summary['run_id'] = Database::lastId();
            }

            foreach ($queue as $i => $row) {
                // Stop before starting a fetch we do not have time to finish.
                if ($budget !== null && (microtime(true) - $started) > ($budget - Fetcher::TIMEOUT)) {
                    $summary['stopped_early'] = true;
                    $summary['remaining'] = count($queue) - $i;
                    break;
                }

                self::checkOne($row, $dryRun, $summary, $say);
            }

            if (!$dryRun && $summary['run_id'] !== null) {
                Database::run(
                    'UPDATE fetch_runs SET finished_at = NOW(), links_checked = :c,
                            changes_found = :ch, errors = :e, feed_items = :f, notes = :n
                      WHERE id = :id',
                    [
                        'c'  => $summary['checked'],
                        'ch' => $summary['changed'],
                        'e'  => $summary['errors'],
                        'f'  => $summary['feed_items'],
                        'n'  => $summary['stopped_early']
                                ? 'Stopped on time budget; ' . $summary['remaining'] . ' left in this batch.'
                                : null,
                        'id' => $summary['run_id'],
                    ]
                );
            }

            // How much is still waiting overall, not just in this batch.
            $summary['remaining'] = self::dueCount($company, $minAge);

        } finally {
            self::releaseLock();
        }

        return $summary;
    }

    /** How many links are due right now. */
    public static function dueCount(?int $company = null, int $minAge = 20): int
    {
        [$where, $params] = self::queueFilter($company, $minAge);
        return (int) Database::scalar(
            'SELECT COUNT(*) FROM v_fetch_queue WHERE ' . $where, $params
        );
    }

    // -----------------------------------------------------------------
    // one link
    // -----------------------------------------------------------------

    private static function checkOne(array $row, bool $dryRun, array &$summary, callable $say): void
    {
        $linkId = (int) $row['link_id'];
        $url    = (string) $row['url'];
        $label  = $row['company_name'] . ' · ' . (parse_url($url, PHP_URL_HOST) ?: $url);

        $response = Fetcher::get($url, $row['etag'], $row['last_modified_header']);
        $summary['checked']++;

        // --- failure ---------------------------------------------------
        if (!$response['ok']) {
            $summary['errors']++;
            $say(sprintf('%s — %s', $label, $response['error']), 'error');

            if (!$dryRun) {
                Database::run(
                    'UPDATE links
                        SET last_checked_at = NOW(), http_status = :st, last_error = :err,
                            consecutive_failures = consecutive_failures + 1
                      WHERE id = :id',
                    [
                        'st'  => $response['status'],
                        'err' => mb_substr((string) $response['error'], 0, 255),
                        'id'  => $linkId,
                    ]
                );

                // A page that was fine and is now a hard 404 is itself news.
                if ($response['status'] === 404 && (int) $row['consecutive_failures'] === 0) {
                    self::recordChange($linkId, (int) $row['company_id'], 'gone', null, null,
                        0, 0, 0.0, null, 'The page returned 404. It may have moved or been withdrawn.');
                    $summary['changed']++;
                }
            }
            return;
        }

        // --- server says unchanged -------------------------------------
        if ($response['not_modified']) {
            $say($label . ' — not modified', 'quiet');
            if (!$dryRun) {
                Database::run(
                    'UPDATE links SET last_checked_at = NOW(), http_status = 304,
                            last_error = NULL, consecutive_failures = 0
                      WHERE id = :id',
                    ['id' => $linkId]
                );
            }
            return;
        }

        $body = $response['body'];

        // --- feed -------------------------------------------------------
        if ((int) $row['is_feed'] === 1 || preg_match('#<(rss|feed)\b#i', mb_substr($body, 0, 2000))) {
            $new = 0;
            foreach (Extractor::parseFeed($body) as $item) {
                if ($dryRun) {
                    $new++;
                    continue;
                }
                $new += Database::run(
                    'INSERT IGNORE INTO feed_items
                        (company_id, link_id, guid_hash, title, url, summary, published_at)
                     VALUES (:c, :l, :g, :t, :u, :s, :p)',
                    [
                        'c' => (int) $row['company_id'],
                        'l' => $linkId,
                        'g' => hash('sha256', $item['guid']),
                        't' => $item['title'],
                        'u' => $item['url'],
                        's' => $item['summary'],
                        'p' => $item['published'],
                    ]
                );
            }

            $summary['feed_items'] += $new;
            $say(sprintf('%s — %d new feed item(s)', $label, $new), $new > 0 ? 'change' : 'quiet');

            if (!$dryRun) {
                self::updateLinkMeta($linkId, $response, null);
            }
            return;
        }

        // --- page content -----------------------------------------------
        $extract = Extractor::fromHtml($body, $response['final_url']);

        if ($extract['words'] < 20) {
            // Almost certainly a JavaScript-rendered shell; diffing it is pointless.
            $say(sprintf('%s — only %d words of text, skipped', $label, $extract['words']), 'quiet');
            if (!$dryRun) {
                self::updateLinkMeta($linkId, $response, $extract['hash']);
            }
            return;
        }

        $previousHash = (string) ($row['content_hash'] ?? '');

        if ($previousHash === '') {
            $say(sprintf('%s — baseline recorded (%d words)', $label, $extract['words']), 'info');
            if (!$dryRun) {
                self::saveSnapshot($linkId, $response['status'], $extract);
                self::updateLinkMeta($linkId, $response, $extract['hash']);
                self::pruneSnapshots($linkId);
            }
            return;
        }

        if ($previousHash === $extract['hash']) {
            $say($label . ' — unchanged', 'quiet');
            if (!$dryRun) {
                self::updateLinkMeta($linkId, $response, $extract['hash']);
            }
            return;
        }

        // Something moved. Is it worth reporting?
        $oldSnap = Database::one(
            'SELECT id, text_content FROM link_snapshots
              WHERE link_id = :id ORDER BY fetched_at DESC LIMIT 1',
            ['id' => $linkId]
        );

        $diff  = Extractor::diff((string) ($oldSnap['text_content'] ?? ''), $extract['text']);
        $moved = $diff['words_added'] + $diff['words_removed'];

        if ($diff['ratio'] < self::MIN_CHANGE_RATIO || $moved < self::MIN_CHANGED_WORDS) {
            $say(sprintf('%s — %d words moved, below threshold', $label, $moved), 'quiet');
            if (!$dryRun) {
                self::updateLinkMeta($linkId, $response, $extract['hash']);
            }
            return;
        }

        $summary['changed']++;
        $say(sprintf('%s — changed: +%d / −%d words (%.1f%%)',
            $label, $diff['words_added'], $diff['words_removed'], $diff['ratio'] * 100), 'change');

        if (!$dryRun) {
            $newSnap = self::saveSnapshot($linkId, $response['status'], $extract);

            self::recordChange(
                $linkId, (int) $row['company_id'], 'content',
                $oldSnap['id'] ?? null, $newSnap,
                $diff['words_added'], $diff['words_removed'], $diff['ratio'],
                Extractor::excerpt($diff['added']),
                Extractor::excerpt($diff['removed'])
            );

            Database::run('UPDATE links SET last_changed_at = NOW() WHERE id = :id', ['id' => $linkId]);
            self::updateLinkMeta($linkId, $response, $extract['hash']);
            self::pruneSnapshots($linkId);
        }
    }

    // -----------------------------------------------------------------
    // queue
    // -----------------------------------------------------------------

    /** @return array{0: string, 1: array<string, mixed>} */
    private static function queueFilter(?int $company, int $minAge): array
    {
        $where  = ['1 = 1'];
        $params = [];

        if ($company !== null) {
            $where[] = 'company_id = :cid';
            $params['cid'] = $company;
        } elseif ($minAge > 0) {
            $where[] = '(last_checked_at IS NULL OR last_checked_at < (NOW() - INTERVAL '
                     . (int) $minAge . ' HOUR))';
        }

        return [implode(' AND ', $where), $params];
    }

    private static function queue(int $limit, ?int $company, int $minAge): array
    {
        [$where, $params] = self::queueFilter($company, $minAge);

        return Database::all(
            'SELECT * FROM v_fetch_queue WHERE ' . $where
          . ' ORDER BY (last_checked_at IS NULL) DESC, last_checked_at ASC LIMIT ' . $limit,
            $params
        );
    }

    // -----------------------------------------------------------------
    // persistence
    // -----------------------------------------------------------------

    private static function updateLinkMeta(int $linkId, array $response, ?string $hash): void
    {
        Database::run(
            'UPDATE links
                SET last_checked_at = NOW(), http_status = :st, etag = :etag,
                    last_modified_header = :lm,
                    content_hash = COALESCE(:hash, content_hash),
                    last_error = NULL, consecutive_failures = 0
              WHERE id = :id',
            [
                'st'   => $response['status'],
                'etag' => isset($response['headers']['etag'])
                          ? mb_substr($response['headers']['etag'], 0, 190) : null,
                'lm'   => isset($response['headers']['last-modified'])
                          ? mb_substr($response['headers']['last-modified'], 0, 64) : null,
                'hash' => $hash,
                'id'   => $linkId,
            ]
        );
    }

    private static function saveSnapshot(int $linkId, ?int $status, array $extract): int
    {
        Database::run(
            'INSERT INTO link_snapshots
                (link_id, http_status, content_hash, page_title, text_content, word_count)
             VALUES (:l, :s, :h, :t, :c, :w)',
            [
                'l' => $linkId, 's' => $status, 'h' => $extract['hash'],
                't' => $extract['title'], 'c' => $extract['text'], 'w' => $extract['words'],
            ]
        );
        return Database::lastId();
    }

    private static function pruneSnapshots(int $linkId): void
    {
        $keep = Database::all(
            'SELECT id FROM link_snapshots WHERE link_id = :l
              ORDER BY fetched_at DESC LIMIT ' . self::KEEP_SNAPSHOTS,
            ['l' => $linkId]
        );
        if (count($keep) < self::KEEP_SNAPSHOTS) {
            return;
        }
        $ids = implode(',', array_map('intval', array_column($keep, 'id')));
        Database::run(
            "DELETE FROM link_snapshots WHERE link_id = :l AND id NOT IN ({$ids})",
            ['l' => $linkId]
        );
    }

    private static function recordChange(int $linkId, int $companyId, string $kind,
                                         ?int $oldSnap, ?int $newSnap, int $added, int $removed,
                                         float $ratio, ?string $addedText, ?string $removedText): void
    {
        Database::run(
            'INSERT INTO link_changes
                (link_id, company_id, change_kind, old_snapshot_id, new_snapshot_id,
                 words_added, words_removed, change_ratio, added_excerpt, removed_excerpt)
             VALUES (:l, :c, :k, :os, :ns, :wa, :wr, :r, :ae, :re)',
            [
                'l' => $linkId, 'c' => $companyId, 'k' => $kind,
                'os' => $oldSnap, 'ns' => $newSnap,
                'wa' => $added, 'wr' => $removed, 'r' => round($ratio, 4),
                'ae' => $addedText, 're' => $removedText,
            ]
        );
    }

    // -----------------------------------------------------------------
    // lock
    // -----------------------------------------------------------------

    private static function acquireLock(): bool
    {
        $path = LOG_DIR . '/crawler.lock';
        $fh = @fopen($path, 'c');
        if ($fh === false) {
            // Cannot lock, so cannot guarantee exclusivity - refuse rather than race.
            error_log('[crawler] cannot open lock file ' . $path);
            return false;
        }
        if (!flock($fh, LOCK_EX | LOCK_NB)) {
            fclose($fh);
            return false;
        }
        self::$lock = $fh;
        return true;
    }

    private static function releaseLock(): void
    {
        if (self::$lock !== null) {
            flock(self::$lock, LOCK_UN);
            fclose(self::$lock);
            self::$lock = null;
        }
    }
}
