<?php
declare(strict_types=1);

/**
 * Turns a fetched page into something worth comparing.
 *
 * Naive change detection on a modern marketing site is useless: a rotating
 * banner, a cookie nonce or a "12 people viewing" counter will report a change
 * every single night and the queue becomes noise an analyst learns to ignore.
 * So the text is stripped down to prose, obvious volatile fragments are removed,
 * and only then is it hashed.
 */
final class Extractor
{
    /** Elements that never carry meaning for our purposes. */
    private const DROP_TAGS = ['script', 'style', 'noscript', 'svg', 'canvas',
                               'iframe', 'template', 'form', 'nav', 'footer'];

    /**
     * @return array{title: string|null, text: string, hash: string, words: int}
     */
    public static function fromHtml(string $html, string $url): array
    {
        $title = self::title($html);
        $text  = self::plainText($html);
        $text  = self::removeVolatile($text);

        return [
            'title' => $title,
            'text'  => $text,
            'hash'  => hash('sha256', $text),
            'words' => $text === '' ? 0 : count(preg_split('/\s+/', $text) ?: []),
        ];
    }

    public static function title(string $html): ?string
    {
        if (preg_match('#<title[^>]*>(.*?)</title>#is', $html, $m)) {
            $title = self::decode(strip_tags($m[1]));
            $title = trim(preg_replace('/\s+/u', ' ', $title) ?? '');
            return $title === '' ? null : mb_substr($title, 0, 255);
        }
        return null;
    }

    /** HTML to normalised prose, one sentence-ish fragment per line. */
    public static function plainText(string $html): string
    {
        // Comments first, so a commented-out block cannot leak through.
        $html = preg_replace('/<!--.*?-->/s', ' ', $html) ?? $html;

        foreach (self::DROP_TAGS as $tag) {
            $html = preg_replace('#<' . $tag . '\b[^>]*>.*?</' . $tag . '>#is', ' ', $html) ?? $html;
            $html = preg_replace('#<' . $tag . '\b[^>]*/?>#is', ' ', $html) ?? $html;
        }

        // Keep block boundaries as line breaks so the diff lands on real units.
        $html = preg_replace('#</(p|div|li|h[1-6]|tr|section|article|br)\s*>#i', "\n", $html) ?? $html;
        $html = preg_replace('#<br\s*/?>#i', "\n", $html) ?? $html;

        $text = strip_tags($html);
        $text = self::decode($text);

        // Normalise whitespace, drop empty and single-word navigation crumbs.
        $lines = [];
        foreach (preg_split('/\R/', $text) ?: [] as $line) {
            $line = trim(preg_replace('/[ \t\x{00A0}]+/u', ' ', $line) ?? '');
            if ($line === '') {
                continue;
            }
            // A line of one or two words is almost always a menu item or a label.
            if (mb_strlen($line) < 12) {
                continue;
            }
            $lines[] = $line;
        }

        // Collapse repeats - the same footer address appearing four times adds nothing.
        $lines = array_values(array_unique($lines));

        return implode("\n", $lines);
    }

    /**
     * Remove the fragments that legitimately change on every request, so they
     * do not each raise a false alarm.
     */
    public static function removeVolatile(string $text): string
    {
        $patterns = [
            // cache-busting ids, nonces, session tokens
            '/\b[a-f0-9]{16,}\b/i',
            // absolute timestamps and "last updated" stamps
            '/\b\d{1,2}:\d{2}(:\d{2})?\s*(am|pm)?\b/i',
            '/\b(19|20)\d{2}-\d{2}-\d{2}T[\d:.+\-Z]+/i',
            // visitor and share counters
            '/\b\d[\d,\.]*\s+(views?|shares?|comments?|likes?|people|online|visitors?)\b/i',
            // copyright year, which flips every January
            '/(©|copyright)\s*(19|20)\d{2}(\s*[-\x{2013}]\s*(19|20)\d{2})?/iu',
            // cookie banners, which vary by consent state
            '/\bcookies?\b[^\n]{0,120}\b(consent|policy|accept|preferences)\b/i',
        ];

        $text = preg_replace($patterns, ' ', $text) ?? $text;
        $text = preg_replace('/[ \t]{2,}/', ' ', $text) ?? $text;

        return trim($text);
    }

    private static function decode(string $s): string
    {
        return html_entity_decode($s, ENT_QUOTES | ENT_HTML5, 'UTF-8');
    }

    // -----------------------------------------------------------------
    // Change comparison
    // -----------------------------------------------------------------

    /**
     * Compare two extracted texts.
     *
     * @return array{
     *   added: array<string>, removed: array<string>,
     *   words_added: int, words_removed: int, ratio: float
     * }
     */
    public static function diff(string $old, string $new): array
    {
        $oldLines = $old === '' ? [] : explode("\n", $old);
        $newLines = $new === '' ? [] : explode("\n", $new);

        // Line-set comparison. Order changes on a page are usually cosmetic
        // reordering of cards, so treating it as a set keeps the noise down.
        $oldSet = array_flip($oldLines);
        $newSet = array_flip($newLines);

        $added   = array_keys(array_diff_key($newSet, $oldSet));
        $removed = array_keys(array_diff_key($oldSet, $newSet));

        $wordsIn  = self::wordCount($added);
        $wordsOut = self::wordCount($removed);
        $baseline = max(1, self::wordCount($oldLines));

        return [
            'added'        => $added,
            'removed'      => $removed,
            'words_added'  => $wordsIn,
            'words_removed'=> $wordsOut,
            'ratio'        => min(1.0, ($wordsIn + $wordsOut) / ($baseline * 2)),
        ];
    }

    /** @param array<string> $lines */
    private static function wordCount(array $lines): int
    {
        $n = 0;
        foreach ($lines as $line) {
            $n += count(preg_split('/\s+/', trim($line)) ?: []);
        }
        return $n;
    }

    /** @param array<string> $lines */
    public static function excerpt(array $lines, int $maxChars = 1200): string
    {
        $out = '';
        foreach ($lines as $line) {
            if (mb_strlen($out) + mb_strlen($line) > $maxChars) {
                $out .= '…';
                break;
            }
            $out .= ($out === '' ? '' : "\n") . $line;
        }
        return $out;
    }

    // -----------------------------------------------------------------
    // Feeds
    // -----------------------------------------------------------------

    /** Find an RSS or Atom feed advertised in a page's head. */
    public static function discoverFeed(string $html, string $baseUrl): ?string
    {
        if (!preg_match_all('#<link\b[^>]*>#i', $html, $tags)) {
            return null;
        }

        foreach ($tags[0] as $tag) {
            if (!preg_match('#type=["\']?application/(rss|atom)\+xml#i', $tag)) {
                continue;
            }
            if (preg_match('#href=["\']([^"\']+)["\']#i', $tag, $m)) {
                return self::absolute(html_entity_decode($m[1]), $baseUrl);
            }
        }
        return null;
    }

    /**
     * Parse RSS 2.0 or Atom into a flat list.
     *
     * @return array<int, array{title: string, url: string|null, summary: string|null, published: string|null, guid: string}>
     */
    public static function parseFeed(string $xml): array
    {
        $items = [];

        $previous = libxml_use_internal_errors(true);
        // LIBXML_NONET blocks external entity fetches; no DTD loading is enabled.
        $doc = simplexml_load_string($xml, SimpleXMLElement::class, LIBXML_NONET | LIBXML_NOCDATA);
        libxml_clear_errors();
        libxml_use_internal_errors($previous);

        if ($doc === false) {
            return $items;
        }

        // RSS
        if (isset($doc->channel->item)) {
            foreach ($doc->channel->item as $item) {
                $url   = trim((string) $item->link);
                $guid  = trim((string) $item->guid) ?: $url;
                $title = trim((string) $item->title);
                if ($title === '' && $url === '') {
                    continue;
                }
                $items[] = [
                    'title'     => mb_substr($title ?: $url, 0, 500),
                    'url'       => $url ?: null,
                    'summary'   => self::cleanSummary((string) $item->description),
                    'published' => self::parseDate((string) $item->pubDate),
                    'guid'      => $guid ?: $title,
                ];
            }
            return $items;
        }

        // Atom
        if (isset($doc->entry)) {
            foreach ($doc->entry as $entry) {
                $url = '';
                foreach ($entry->link as $link) {
                    $rel = (string) $link['rel'];
                    if ($rel === '' || $rel === 'alternate') {
                        $url = trim((string) $link['href']);
                        break;
                    }
                }
                $title = trim((string) $entry->title);
                $guid  = trim((string) $entry->id) ?: $url;
                if ($title === '' && $url === '') {
                    continue;
                }
                $items[] = [
                    'title'     => mb_substr($title ?: $url, 0, 500),
                    'url'       => $url ?: null,
                    'summary'   => self::cleanSummary((string) ($entry->summary ?: $entry->content)),
                    'published' => self::parseDate((string) ($entry->published ?: $entry->updated)),
                    'guid'      => $guid ?: $title,
                ];
            }
        }

        return $items;
    }

    private static function cleanSummary(string $html): ?string
    {
        $text = trim(preg_replace('/\s+/u', ' ', self::decode(strip_tags($html))) ?? '');
        return $text === '' ? null : mb_substr($text, 0, 1000);
    }

    private static function parseDate(string $raw): ?string
    {
        $raw = trim($raw);
        if ($raw === '') {
            return null;
        }
        try {
            return (new DateTimeImmutable($raw))->format('Y-m-d H:i:s');
        } catch (Exception) {
            return null;
        }
    }

    /** Resolve a possibly relative URL against the page it came from. */
    public static function absolute(string $href, string $baseUrl): string
    {
        if (preg_match('#^https?://#i', $href)) {
            return $href;
        }

        $base = parse_url($baseUrl);
        if (!isset($base['scheme'], $base['host'])) {
            return $href;
        }
        $root = $base['scheme'] . '://' . $base['host']
              . (isset($base['port']) ? ':' . $base['port'] : '');

        if (str_starts_with($href, '//')) {
            return $base['scheme'] . ':' . $href;
        }
        if (str_starts_with($href, '/')) {
            return $root . $href;
        }

        $dir = rtrim(dirname($base['path'] ?? '/'), '/');
        return $root . $dir . '/' . ltrim($href, '/');
    }
}
