#!/usr/bin/env python3
"""Lint robots.txt User-agent values against Google's reference parser semantics.

Google's open-source robots.txt parser (github.com/google/robotstxt, robots.cc)
truncates every User-agent value before comparing it:

    /*static*/ absl::string_view RobotsMatcher::ExtractUserAgent(
        absl::string_view user_agent) {
      // Allowed characters in user-agent are [a-zA-Z_-].
      const char* end = user_agent.data();
      while (absl::ascii_isalpha(*end) || *end == '-' || *end == '_') ++end;
      return user_agent.substr(0, end - user_agent.data());
    }

and then compares the truncated value for case-insensitive *equality* against
the crawler's own product token (HandleUserAgent -> absl::EqualsIgnoreCase).

So a line like `User-agent: Brightbot 1.0` is read as the token `Brightbot`,
and `User-agent: bigsur.ai` is read as `bigsur`. This lints for values that do
not survive that truncation intact.

Usage:
    python tools/robots_agent_lint.py <path-or-url> [more...]
"""
import re
import sys
import urllib.request

ALLOWED = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_")


_LINES = re.compile(r"\r\n|\r|\n")  # robots.cc line terminators: LF, CR, CRLF

def extract_user_agent(value):
    """Port of RobotsMatcher::ExtractUserAgent from Google's robots.cc."""
    end = 0
    while end < len(value) and value[end] in ALLOWED:
        end += 1
    return value[:end]


def is_global(value):
    """Port of the '*' carve-out in RobotsMatcher::HandleUserAgent. Google treats a
    value starting with '*' followed by end-of-value or whitespace as the global
    group and never truncates it, so `User-agent: * v2` is still the catch-all and
    is not an error."""
    return len(value) >= 1 and value[0] == "*" and (len(value) == 1 or value[1].isspace())


def load(src):
    if src.startswith(("http://", "https://")):
        req = urllib.request.Request(src, headers={"User-Agent": "robots-agent-lint/1.0"})
        with urllib.request.urlopen(req, timeout=30) as r:
            return r.read().decode("utf-8", "replace")
    with open(src, encoding="utf-8", errors="replace") as f:
        return f.read()


def lint(src):
    text = load(src)
    values = []
    # robots.cc treats BOTH 0x0A and 0x0D as line terminators (CRLF counts as
    # one), so a file with bare CR endings is many lines to Google and ONE line
    # to a split("\n"), which swallows every Disallow and fails open (alrosa.ru).
    for i, line in enumerate(_LINES.split(text), 1):
        stripped = line.split("#", 1)[0].strip()
        if ":" not in stripped:
            continue
        key, _, val = stripped.partition(":")
        if key.strip().lower() == "user-agent":
            values.append((i, val.strip()))

    tokens_declared = {v.lower() for _, v in values}
    truncated = []
    for lineno, val in values:
        if is_global(val):
            continue
        tok = extract_user_agent(val)
        if tok != val:
            truncated.append((lineno, val, tok))

    print(f"=== {src}")
    print(f"User-agent lines: {len(values)}")
    print(f"Values truncated by Google's parser: {len(truncated)}")
    if truncated:
        print()
        print(f"  {'line':>5}  {'declared value':<32} {'Google reads as':<24} note")
        print(f"  {'-'*5}  {'-'*32} {'-'*24} {'-'*40}")
        for lineno, val, tok in truncated:
            if not tok:
                note = "reads as EMPTY: line matches nothing"
            elif tok.lower() in tokens_declared:
                note = "collapses onto another declared agent (redundant)"
            else:
                note = "becomes a token that is NOT declared anywhere"
            print(f"  {lineno:>5}  {val:<32} {tok or '(empty)':<24} {note}")
    return truncated


if __name__ == "__main__":
    args = sys.argv[1:]
    if not args:
        print(__doc__)
        sys.exit(2)
    total = 0
    for src in args:
        total += len(lint(src))
        print()
    sys.exit(1 if total else 0)