#!/usr/bin/env python3
"""
Wasabi Bucket Log -> Reporter
Created by Francisco Santos

Cross-platform:
  Linux:   python3 wasabi_log_report.py L
  Windows: python  wasabi_log_report.py W

The only runtime argument is:
  L = use the Linux/macOS root defined in get_paths()
  W = use C:\\Wasabi

Edit the CONFIGURATION section below for the Wasabi logging bucket,
optional prefix, endpoint URL, AWS CLI profile, and report behavior.

Workflow:
  1. Create the platform folder structure.
  2. Mirror the Wasabi logging bucket/prefix locally with:
       aws s3 sync ... --delete --endpoint-url ...
  3. Parse all current local bucket-log files.
  4. Ignore Wasabi header/separator lines ("Record..." and "====...").
  5. Preserve every documented field.
  6. Generate detailed CSV and summary CSV files.
  7. Classify REST.PUT.OBJECT as upload traffic using ObjectSize.
  8. Classify REST.GET.OBJECT as download traffic using BytesSent.
  9. Write malformed/unrecognized lines to parse_errors.csv.
 10. Build an Excel workbook (wasabi_report.xlsx) with native charts and a
     PDF report (wasabi_report.pdf) covering GET/PUT/DELETE volumes and
     403 / 404 / 5XX error counts.

Optional dependencies:
  Excel report : openpyxl      (pip install openpyxl)
  PDF report   : matplotlib    (pip install matplotlib)
Missing libraries are reported and skipped; CSV output always runs.

Important:
  --delete only removes files from the LOCAL mirror when they no longer
  exist in the Wasabi source. Configure a lifecycle rule on the logging
  bucket/prefix to expire old log objects. The next sync then removes the
  corresponding local copies automatically.
"""

from __future__ import annotations

import argparse
import csv
import os
import re
import shlex
import shutil
import subprocess
import sys
from collections import Counter, OrderedDict, defaultdict
from datetime import datetime, timezone
from pathlib import Path
from typing import Dict, Iterable, List, Optional
from urllib.parse import unquote


# ============================================================
# CONFIGURATION
# ============================================================

# Bucket containing the Wasabi bucket-log objects.
LOG_BUCKET = "fcsbr-log"

# Optional prefix inside LOG_BUCKET.
#
# Root of bucket:
#   LOG_PREFIX = ""
#
# Prefix:
#   LOG_PREFIX = "bucket-logs/"
#
# Do not include s3://bucket-name here.
LOG_PREFIX = "logs-bucket/"

# Wasabi service endpoint for the region containing LOG_BUCKET.
# Examples:
#   https://s3.us-east-1.wasabisys.com
#   https://s3.eu-central-1.wasabisys.com
ENDPOINT_URL = "https://s3.us-east-1.wasabisys.com"

# Optional AWS CLI named profile.
# Set to "" to use the normal/default AWS CLI credential resolution.
AWS_PROFILE = ""

# Pass an explicit region to AWS CLI if desired.
# Set to "" to omit --region.
AWS_REGION = "us-east-1"

# Keep AWS CLI output quiet except for errors.
AWS_ONLY_SHOW_ERRORS = True

# The local mirror is intentionally retained between runs.
# This makes aws s3 sync incremental and allows --delete to mirror
# lifecycle deletions from Wasabi.
#
# CSV files are rebuilt from the current local mirror on every run.
#
# Upload/download operation definitions can be extended if your
# environment uses additional operation names.
UPLOAD_OPERATIONS = {
    "REST.PUT.OBJECT",
}

DOWNLOAD_OPERATIONS = {
    "REST.GET.OBJECT",
}

# Graphical reports written next to the CSV files.
# Set either to False to skip that format.
WRITE_EXCEL_REPORT = True
WRITE_PDF_REPORT = True

# Cap on how many individual failing requests are listed on the
# "Error Records" worksheet. Aggregates always cover every record.
MAX_ERROR_RECORDS = 5000


# ============================================================
# PLATFORM PATHS
# ============================================================

def get_paths(mode: str) -> Dict[str, Path]:
    mode = mode.upper()

    if mode == "L":
        root = Path("/Users/fsantos/Desktop/Wasabi")
    elif mode == "W":
        root = Path(r"C:\Wasabi")
    else:
        raise ValueError("Mode must be L (Linux) or W (Windows).")

    return {
        "root": root,
        "raw": root / "logs" / "raw",
        "output": root / "output",
        "runtime": root / "runtime",
    }


def create_directories(paths: Dict[str, Path]) -> None:
    for path in paths.values():
        path.mkdir(parents=True, exist_ok=True)


# ============================================================
# AWS CLI DOWNLOAD / MIRROR
# ============================================================

def find_aws_cli() -> str:
    aws = shutil.which("aws")
    if not aws:
        raise RuntimeError(
            "AWS CLI was not found in PATH. Install AWS CLI v2 and verify "
            "that 'aws --version' works before running this script."
        )
    return aws


def build_s3_source() -> str:
    prefix = LOG_PREFIX.strip("/")

    if prefix:
        return f"s3://{LOG_BUCKET}/{prefix}/"

    return f"s3://{LOG_BUCKET}/"


def sync_logs(raw_dir: Path) -> None:
    if not LOG_BUCKET or LOG_BUCKET == "CHANGE-ME-LOG-BUCKET":
        raise RuntimeError(
            "Set LOG_BUCKET in the CONFIGURATION section first.")

    if not ENDPOINT_URL:
        raise RuntimeError(
            "Set ENDPOINT_URL in the CONFIGURATION section first.")

    aws = find_aws_cli()
    source = build_s3_source()

    cmd = [
        aws,
        "s3",
        "sync",
        source,
        str(raw_dir),
        "--delete",
        "--endpoint-url",
        ENDPOINT_URL,
    ]

    if AWS_PROFILE:
        cmd.extend(["--profile", AWS_PROFILE])

    if AWS_REGION:
        cmd.extend(["--region", AWS_REGION])

    if AWS_ONLY_SHOW_ERRORS:
        cmd.append("--only-show-errors")

    print(f"Synchronizing: {source}")
    print(f"Local mirror : {raw_dir}")

    result = subprocess.run(cmd, check=False)

    if result.returncode != 0:
        raise RuntimeError(
            f"AWS CLI sync failed with return code {result.returncode}. "
            "CSV generation was not started."
        )


# ============================================================
# WASABI LOG PARSER
# ============================================================

# Parse all fields through User-Agent, then handle the tail separately.
#
# Documented format:
# BucketOwner Bucket Time RemoteIP Requester RequestId Operation Key
# Request-URI HttpStatus ErrorCode BytesSent ObjectSize TotalTime
# Turn-AroundTime Referrer User-Agent VersionId
#
# Some newer examples may contain an additional quoted TLS value before
# VersionId. The parser preserves it in TLSVersion when present.
LOG_RE = re.compile(
    r'^'
    r'(?P<BucketOwner>\S+)\s+'
    r'(?P<Bucket>\S+)\s+'
    r'\[(?P<Time>[^\]]+)\]\s+'
    r'(?P<RemoteIP>\S+)\s+'
    r'(?P<Requester>\S+)\s+'
    r'(?P<RequestId>\S+)\s+'
    r'(?P<Operation>\S+)\s+'
    r'(?P<Key>\S+)\s+'
    r'"(?P<RequestURI>[^"]*)"\s+'
    r'(?P<HttpStatus>\S+)\s+'
    r'(?P<ErrorCode>\S+)\s+'
    r'(?P<BytesSent>\S+)\s+'
    r'(?P<ObjectSize>\S+)\s+'
    r'(?P<TotalTime>\S+)\s+'
    r'(?P<TurnAroundTime>\S+)\s+'
    r'"(?P<Referrer>[^"]*)"\s+'
    r'"(?P<UserAgent>[^"]*)"\s+'
    r'(?P<Tail>.+?)'
    r'\s*$'
)


CSV_FIELDS = [
    "BucketOwner",
    "Bucket",
    "TimeRaw",
    "TimestampISO8601",
    "TimestampUTC",
    "RemoteIP",
    "Requester",
    "RequestId",
    "Operation",
    "KeyRaw",
    "KeyDecoded",
    "RequestURI",
    "RequestURIDecoded",
    "HttpMethod",
    "HttpStatus",
    "ErrorCode",
    "BytesSent",
    "ObjectSize",
    "TotalTimeMs",
    "TurnAroundTimeMs",
    "Referrer",
    "UserAgent",
    "TLSVersion",
    "VersionId",
    "TrafficDirection",
    "TrafficBytes",
    "TrafficMiB",
    "TrafficGiB",
    "TrafficTiB",
    "Successful",
]


def dash_to_blank(value: str) -> str:
    return "" if value == "-" else value


def parse_optional_int(value: str) -> Optional[int]:
    if value in ("", "-"):
        return None

    try:
        return int(value)
    except ValueError:
        return None


def parse_time(raw: str):
    try:
        dt = datetime.strptime(raw, "%d/%b/%Y:%H:%M:%S %z")
        return dt
    except ValueError:
        return None


def extract_http_method(request_uri: str) -> str:
    if not request_uri or request_uri == "-":
        return ""

    first = request_uri.split(" ", 1)[0].strip()
    if re.fullmatch(r"[A-Z]+", first):
        return first

    return ""


def parse_tail(tail: str):
    """
    Supported tails:
      -
      version-id
      "version-id"
      "TLSv1.3" version-id

    If extra tokens ever appear, preserve them in TLSVersion and use the
    final token as VersionId rather than dropping the record.
    """
    try:
        tokens = shlex.split(tail, posix=True)
    except ValueError:
        tokens = tail.split()

    if not tokens:
        return "", ""

    if len(tokens) == 1:
        return "", dash_to_blank(tokens[0])

    tls_version = " ".join(tokens[:-1])
    version_id = dash_to_blank(tokens[-1])
    return dash_to_blank(tls_version), version_id


def classify_traffic(operation: str, bytes_sent: Optional[int], object_size: Optional[int]):
    if operation in UPLOAD_OPERATIONS:
        # For PUT object traffic, ObjectSize represents the uploaded object.
        traffic_bytes = object_size or 0
        return "UPLOAD", traffic_bytes

    if operation in DOWNLOAD_OPERATIONS:
        # For GET object traffic, BytesSent is the actual response payload.
        # This correctly represents byte-range GETs (HTTP 206) as well.
        traffic_bytes = bytes_sent or 0
        return "DOWNLOAD", traffic_bytes

    return "OTHER", 0


def is_success(status: str) -> str:
    if not status.isdigit():
        return ""

    code = int(status)
    return "YES" if 200 <= code <= 299 else "NO"


def parse_record(line: str) -> Optional[Dict[str, object]]:
    match = LOG_RE.match(line)
    if not match:
        return None

    data = match.groupdict()

    dt = parse_time(data["Time"])

    if dt:
        timestamp_iso = dt.isoformat()
        timestamp_utc = dt.astimezone(timezone.utc).isoformat()
    else:
        timestamp_iso = ""
        timestamp_utc = ""

    bytes_sent = parse_optional_int(data["BytesSent"])
    object_size = parse_optional_int(data["ObjectSize"])
    total_time = parse_optional_int(data["TotalTime"])
    turnaround = parse_optional_int(data["TurnAroundTime"])

    tls_version, version_id = parse_tail(data["Tail"])

    direction, traffic_bytes = classify_traffic(
        data["Operation"],
        bytes_sent,
        object_size,
    )

    key_raw = dash_to_blank(data["Key"])
    request_uri = dash_to_blank(data["RequestURI"])

    row = {
        "BucketOwner": dash_to_blank(data["BucketOwner"]),
        "Bucket": dash_to_blank(data["Bucket"]),
        "TimeRaw": data["Time"],
        "TimestampISO8601": timestamp_iso,
        "TimestampUTC": timestamp_utc,
        "RemoteIP": dash_to_blank(data["RemoteIP"]),
        "Requester": dash_to_blank(data["Requester"]),
        "RequestId": dash_to_blank(data["RequestId"]),
        "Operation": dash_to_blank(data["Operation"]),
        "KeyRaw": key_raw,
        "KeyDecoded": unquote(key_raw) if key_raw else "",
        "RequestURI": request_uri,
        "RequestURIDecoded": unquote(request_uri) if request_uri else "",
        "HttpMethod": extract_http_method(data["RequestURI"]),
        "HttpStatus": dash_to_blank(data["HttpStatus"]),
        "ErrorCode": dash_to_blank(data["ErrorCode"]),
        "BytesSent": "" if bytes_sent is None else bytes_sent,
        "ObjectSize": "" if object_size is None else object_size,
        "TotalTimeMs": "" if total_time is None else total_time,
        "TurnAroundTimeMs": "" if turnaround is None else turnaround,
        "Referrer": data["Referrer"],
        "UserAgent": data["UserAgent"],
        "TLSVersion": tls_version,
        "VersionId": version_id,
        "TrafficDirection": direction,
        "TrafficBytes": traffic_bytes,
        "TrafficMiB": f"{traffic_bytes / (1024 ** 2):.6f}",
        "TrafficGiB": f"{traffic_bytes / (1024 ** 3):.6f}",
        "TrafficTiB": f"{traffic_bytes / (1024 ** 4):.6f}",
        "Successful": is_success(data["HttpStatus"]),
    }

    return row


def should_skip_line(line: str) -> bool:
    stripped = line.strip()

    if not stripped:
        return True

    # Wasabi log files may contain:
    # Record format: [...]
    if stripped.lower().startswith("record"):
        return True

    # Separator line:
    # ========================================================
    if stripped.startswith("="):
        return True

    return False


def iter_log_files(raw_dir: Path) -> Iterable[Path]:
    """
    Recursively process every file in the mirror.

    This supports:
      - logging objects stored at the bucket root;
      - logging objects stored under a prefix;
      - nested paths below the configured prefix.

    Output/runtime files live outside raw_dir and therefore cannot be
    accidentally parsed as source logs.
    """
    for path in sorted(raw_dir.rglob("*")):
        if path.is_file():
            yield path


# ============================================================
# CSV GENERATION
# ============================================================

def write_reports(raw_dir: Path, output_dir: Path) -> Dict[str, object]:
    detail_csv = output_dir / "wasabi_log_details.csv"
    traffic_csv = output_dir / "wasabi_traffic_summary.csv"
    operation_csv = output_dir / "wasabi_operation_summary.csv"
    error_csv = output_dir / "parse_errors.csv"

    parsed_rows: List[Dict[str, object]] = []
    parse_errors = []

    input_files = 0
    input_lines = 0
    skipped_lines = 0

    for file_path in iter_log_files(raw_dir):
        input_files += 1

        try:
            relative_name = str(file_path.relative_to(raw_dir))
        except ValueError:
            relative_name = str(file_path)

        try:
            with file_path.open(
                "r",
                encoding="utf-8",
                errors="replace",
                newline="",
            ) as handle:
                for line_number, raw_line in enumerate(handle, start=1):
                    input_lines += 1
                    line = raw_line.rstrip("\r\n")

                    if should_skip_line(line):
                        skipped_lines += 1
                        continue

                    row = parse_record(line)

                    if row is None:
                        parse_errors.append(
                            {
                                "Location": f"{relative_name}:{line_number}",
                                "Reason": "Record did not match the Wasabi log format",
                                "RawRecord": line,
                            }
                        )
                        continue

                    parsed_rows.append(row)

        except OSError as exc:
            parse_errors.append(
                {
                    "Location": relative_name,
                    "Reason": f"Unable to read file: {exc}",
                    "RawRecord": "",
                }
            )

    # Sort by parsed timestamp; records with bad/missing timestamps are last.
    parsed_rows.sort(
        key=lambda row: (
            row["TimestampISO8601"] == "",
            row["TimestampISO8601"],
            row["RequestId"],
        )
    )

    with detail_csv.open("w", encoding="utf-8-sig", newline="") as handle:
        writer = csv.DictWriter(handle, fieldnames=CSV_FIELDS)
        writer.writeheader()
        writer.writerows(parsed_rows)

    with error_csv.open("w", encoding="utf-8-sig", newline="") as handle:
        fields = ["Location", "Reason", "RawRecord"]
        writer = csv.DictWriter(handle, fieldnames=fields)
        writer.writeheader()
        writer.writerows(parse_errors)

    # --------------------------------------------------------
    # Traffic summary
    # --------------------------------------------------------

    traffic_stats = {
        "UPLOAD": {"requests": 0, "bytes": 0},
        "DOWNLOAD": {"requests": 0, "bytes": 0},
        "OTHER": {"requests": 0, "bytes": 0},
    }

    for row in parsed_rows:
        direction = str(row["TrafficDirection"])
        traffic_stats[direction]["requests"] += 1
        traffic_stats[direction]["bytes"] += int(row["TrafficBytes"])

    with traffic_csv.open("w", encoding="utf-8-sig", newline="") as handle:
        fields = [
            "TrafficDirection",
            "Requests",
            "TrafficBytes",
            "TrafficMiB",
            "TrafficGiB",
            "TrafficTiB",
        ]
        writer = csv.DictWriter(handle, fieldnames=fields)
        writer.writeheader()

        for direction in ("UPLOAD", "DOWNLOAD", "OTHER"):
            count = traffic_stats[direction]["requests"]
            size = traffic_stats[direction]["bytes"]

            writer.writerow(
                {
                    "TrafficDirection": direction,
                    "Requests": count,
                    "TrafficBytes": size,
                    "TrafficMiB": f"{size / (1024 ** 2):.6f}",
                    "TrafficGiB": f"{size / (1024 ** 3):.6f}",
                    "TrafficTiB": f"{size / (1024 ** 4):.6f}",
                }
            )

        total_bytes = (
            traffic_stats["UPLOAD"]["bytes"]
            + traffic_stats["DOWNLOAD"]["bytes"]
        )

        writer.writerow(
            {
                "TrafficDirection": "UPLOAD+DOWNLOAD",
                "Requests": (
                    traffic_stats["UPLOAD"]["requests"]
                    + traffic_stats["DOWNLOAD"]["requests"]
                ),
                "TrafficBytes": total_bytes,
                "TrafficMiB": f"{total_bytes / (1024 ** 2):.6f}",
                "TrafficGiB": f"{total_bytes / (1024 ** 3):.6f}",
                "TrafficTiB": f"{total_bytes / (1024 ** 4):.6f}",
            }
        )

    # --------------------------------------------------------
    # Operation summary
    # --------------------------------------------------------

    operation_stats = defaultdict(
        lambda: {
            "requests": 0,
            "traffic_bytes": 0,
            "success": 0,
            "failure": 0,
            "unknown_status": 0,
            "errors": Counter(),
        }
    )

    for row in parsed_rows:
        op = str(row["Operation"])
        stats = operation_stats[op]
        stats["requests"] += 1
        stats["traffic_bytes"] += int(row["TrafficBytes"])

        if row["Successful"] == "YES":
            stats["success"] += 1
        elif row["Successful"] == "NO":
            stats["failure"] += 1
        else:
            stats["unknown_status"] += 1

        if row["ErrorCode"]:
            stats["errors"][str(row["ErrorCode"])] += 1

    with operation_csv.open("w", encoding="utf-8-sig", newline="") as handle:
        fields = [
            "Operation",
            "Requests",
            "SuccessfulRequests",
            "FailedRequests",
            "UnknownStatusRequests",
            "TrafficBytes",
            "TrafficMiB",
            "TrafficGiB",
            "TrafficTiB",
            "TopErrorCodes",
        ]

        writer = csv.DictWriter(handle, fieldnames=fields)
        writer.writeheader()

        for operation in sorted(operation_stats):
            stats = operation_stats[operation]
            size = stats["traffic_bytes"]
            errors = "; ".join(
                f"{name}={count}"
                for name, count in stats["errors"].most_common(10)
            )

            writer.writerow(
                {
                    "Operation": operation,
                    "Requests": stats["requests"],
                    "SuccessfulRequests": stats["success"],
                    "FailedRequests": stats["failure"],
                    "UnknownStatusRequests": stats["unknown_status"],
                    "TrafficBytes": size,
                    "TrafficMiB": f"{size / (1024 ** 2):.6f}",
                    "TrafficGiB": f"{size / (1024 ** 3):.6f}",
                    "TrafficTiB": f"{size / (1024 ** 4):.6f}",
                    "TopErrorCodes": errors,
                }
            )

    return {
        "rows": parsed_rows,
        "input_files": input_files,
        "input_lines": input_lines,
        "skipped_lines": skipped_lines,
        "parsed_records": len(parsed_rows),
        "parse_errors": len(parse_errors),
        "detail_csv": detail_csv,
        "traffic_csv": traffic_csv,
        "operation_csv": operation_csv,
        "error_csv": error_csv,
        "upload_requests": traffic_stats["UPLOAD"]["requests"],
        "upload_bytes": traffic_stats["UPLOAD"]["bytes"],
        "download_requests": traffic_stats["DOWNLOAD"]["requests"],
        "download_bytes": traffic_stats["DOWNLOAD"]["bytes"],
    }


# ============================================================
# ANALYTICS FOR THE GRAPHICAL REPORTS
# ============================================================

# Verbs charted individually; anything else is folded into OTHER.
REPORT_VERBS = ["GET", "PUT", "DELETE", "HEAD", "POST", "OTHER"]

KNOWN_VERBS = {"GET", "PUT", "POST", "DELETE", "HEAD", "COPY", "OPTIONS"}

# Error buckets requested for the report.
ERROR_CATEGORIES = ["403", "404", "5XX", "Other 4XX"]


def operation_verb(operation: str, http_method: str) -> str:
    """
    Wasabi operations look like REST.PUT.OBJECT or S3.DELETE.UPLOAD, so the
    second component is the verb. Fall back to the method parsed out of the
    Request-URI, then to OTHER.
    """
    parts = operation.split(".")

    if len(parts) >= 2 and parts[1] in KNOWN_VERBS:
        verb = parts[1]
    elif http_method in KNOWN_VERBS:
        verb = http_method
    else:
        return "OTHER"

    return verb if verb in REPORT_VERBS else "OTHER"


def status_class(status: str) -> str:
    if not status.isdigit():
        return "UNKNOWN"

    return f"{int(status) // 100}XX"


def error_category(status: str) -> Optional[str]:
    """403, 404, 5XX or Other 4XX. None for anything that is not an error."""
    if not status.isdigit():
        return None

    code = int(status)

    if code == 403:
        return "403"
    if code == 404:
        return "404"
    if 500 <= code <= 599:
        return "5XX"
    if 400 <= code <= 499:
        return "Other 4XX"

    return None


def new_verb_stats() -> Dict[str, object]:
    stats: Dict[str, object] = {
        "requests": 0,
        "success": 0,
        "failed": 0,
        "unknown": 0,
        "traffic_bytes": 0,
    }

    for category in ERROR_CATEGORIES:
        stats[category] = 0

    return stats


def build_analytics(rows: List[Dict[str, object]]) -> Dict[str, object]:
    verbs = OrderedDict((verb, new_verb_stats()) for verb in REPORT_VERBS)

    status_counts: Counter = Counter()
    class_counts: Counter = Counter()
    error_totals: Counter = Counter()
    error_codes: Counter = Counter()
    error_by_operation: Dict[str, Counter] = defaultdict(Counter)

    daily: Dict[str, Dict[str, int]] = OrderedDict()
    error_records: List[Dict[str, object]] = []
    error_records_total = 0

    for row in rows:
        status = str(row["HttpStatus"])
        verb = operation_verb(str(row["Operation"]), str(row["HttpMethod"]))
        category = error_category(status)

        stats = verbs[verb]
        stats["requests"] += 1
        stats["traffic_bytes"] += int(row["TrafficBytes"])

        if row["Successful"] == "YES":
            stats["success"] += 1
        elif row["Successful"] == "NO":
            stats["failed"] += 1
        else:
            stats["unknown"] += 1

        status_counts[status or "-"] += 1
        class_counts[status_class(status)] += 1

        if category:
            stats[category] += 1
            error_totals[category] += 1
            error_by_operation[str(row["Operation"])][category] += 1
            error_records_total += 1

            if len(error_records) < MAX_ERROR_RECORDS:
                error_records.append(
                    {
                        "TimestampUTC": row["TimestampUTC"],
                        "Verb": verb,
                        "Operation": row["Operation"],
                        "HttpStatus": status,
                        "ErrorCategory": category,
                        "ErrorCode": row["ErrorCode"],
                        "Bucket": row["Bucket"],
                        "Key": row["KeyDecoded"],
                        "RemoteIP": row["RemoteIP"],
                        "Requester": row["Requester"],
                        "RequestId": row["RequestId"],
                    }
                )

        if row["ErrorCode"]:
            error_codes[str(row["ErrorCode"])] += 1

        day = str(row["TimestampUTC"])[:10]

        if day:
            bucket = daily.get(day)

            if bucket is None:
                bucket = {verb_name: 0 for verb_name in REPORT_VERBS}
                bucket.update(
                    {
                        "requests": 0,
                        "errors": 0,
                        "upload_bytes": 0,
                        "download_bytes": 0,
                    }
                )
                daily[day] = bucket

            bucket[verb] += 1
            bucket["requests"] += 1

            if category:
                bucket["errors"] += 1

            if row["TrafficDirection"] == "UPLOAD":
                bucket["upload_bytes"] += int(row["TrafficBytes"])
            elif row["TrafficDirection"] == "DOWNLOAD":
                bucket["download_bytes"] += int(row["TrafficBytes"])

    ordered_daily = OrderedDict(sorted(daily.items()))

    return {
        "total_requests": len(rows),
        "verbs": verbs,
        "status_counts": status_counts,
        "class_counts": class_counts,
        "error_totals": error_totals,
        "error_codes": error_codes,
        "error_by_operation": error_by_operation,
        "error_records": error_records,
        "error_records_total": error_records_total,
        "daily": ordered_daily,
    }


def verb_rows(analytics: Dict[str, object]) -> List[List[object]]:
    """verb, requests, successful, 403, 404, 5XX, Other 4XX, traffic GiB."""
    table = []

    for verb, stats in analytics["verbs"].items():
        table.append(
            [
                verb,
                stats["requests"],
                stats["success"],
                stats["403"],
                stats["404"],
                stats["5XX"],
                stats["Other 4XX"],
                round(stats["traffic_bytes"] / (1024 ** 3), 6),
            ]
        )

    return table


# ============================================================
# EXCEL REPORT
# ============================================================

def write_excel_report(analytics: Dict[str, object], output_dir: Path) -> Optional[Path]:
    try:
        from openpyxl import Workbook
        from openpyxl.chart import BarChart, LineChart, Reference
        from openpyxl.styles import Alignment, Font, PatternFill
        from openpyxl.utils import get_column_letter
    except ImportError:
        print(
            "Excel report skipped: openpyxl is not installed "
            "(pip install openpyxl)."
        )
        return None

    target = output_dir / "wasabi_report.xlsx"

    header_font = Font(bold=True, color="FFFFFF")
    header_fill = PatternFill("solid", fgColor="1F4E79")
    title_font = Font(bold=True, size=14)

    workbook = Workbook()

    def add_table(sheet, headers, table, start_row: int = 1) -> int:
        for column, name in enumerate(headers, start=1):
            cell = sheet.cell(row=start_row, column=column, value=name)
            cell.font = header_font
            cell.fill = header_fill
            cell.alignment = Alignment(horizontal="center")

        for offset, record in enumerate(table, start=1):
            for column, value in enumerate(record, start=1):
                sheet.cell(row=start_row + offset, column=column, value=value)

        for column, name in enumerate(headers, start=1):
            widest = len(str(name))

            for record in table:
                if column <= len(record):
                    widest = max(widest, len(str(record[column - 1])))

            sheet.column_dimensions[get_column_letter(column)].width = min(
                widest + 3, 60
            )

        return start_row + len(table)

    # --------------------------------------------------------
    # Summary
    # --------------------------------------------------------
    summary = workbook.active
    summary.title = "Summary"

    errors = analytics["error_totals"]
    verbs = analytics["verbs"]
    upload_bytes = sum(
        day["upload_bytes"] for day in analytics["daily"].values()
    )
    download_bytes = sum(
        day["download_bytes"] for day in analytics["daily"].values()
    )

    summary["A1"] = "Wasabi Bucket Log Report"
    summary["A1"].font = title_font
    summary["A2"] = f"Bucket logs: s3://{LOG_BUCKET}/{LOG_PREFIX.strip('/')}"
    summary["A3"] = f"Generated (UTC): {datetime.now(timezone.utc).isoformat(timespec='seconds')}"

    days = list(analytics["daily"])
    coverage = f"{days[0]} to {days[-1]}" if days else "no dated records"

    summary_table = [
        ["Total parsed requests", analytics["total_requests"]],
        ["Date coverage (UTC)", coverage],
        ["GET requests", verbs["GET"]["requests"]],
        ["PUT requests", verbs["PUT"]["requests"]],
        ["DELETE requests", verbs["DELETE"]["requests"]],
        ["HEAD requests", verbs["HEAD"]["requests"]],
        ["POST requests", verbs["POST"]["requests"]],
        ["Other requests", verbs["OTHER"]["requests"]],
        ["403 Forbidden", errors["403"]],
        ["404 Not Found", errors["404"]],
        ["5XX Server errors", errors["5XX"]],
        ["Other 4XX errors", errors["Other 4XX"]],
        ["Total errors", sum(errors.values())],
        ["Upload traffic (GiB)", round(upload_bytes / (1024 ** 3), 6)],
        ["Download traffic (GiB)", round(download_bytes / (1024 ** 3), 6)],
    ]

    add_table(summary, ["Metric", "Value"], summary_table, start_row=5)

    # --------------------------------------------------------
    # Requests by verb
    # --------------------------------------------------------
    operations = workbook.create_sheet("Requests by Verb")
    headers = [
        "Verb",
        "Requests",
        "Successful",
        "403",
        "404",
        "5XX",
        "Other 4XX",
        "TrafficGiB",
    ]
    table = verb_rows(analytics)
    last_row = add_table(operations, headers, table, start_row=1)

    volume = BarChart()
    volume.type = "col"
    volume.title = "Requests by operation verb"
    volume.y_axis.title = "Requests"
    volume.x_axis.title = "Verb"
    volume.add_data(
        Reference(operations, min_col=2, min_row=1, max_row=last_row),
        titles_from_data=True,
    )
    volume.set_categories(
        Reference(operations, min_col=1, min_row=2, max_row=last_row)
    )
    volume.height = 9
    volume.width = 18
    operations.add_chart(volume, "J2")

    breakdown = BarChart()
    breakdown.type = "col"
    breakdown.grouping = "stacked"
    breakdown.overlap = 100
    breakdown.title = "Errors by verb (403 / 404 / 5XX / other 4XX)"
    breakdown.y_axis.title = "Failed requests"
    breakdown.x_axis.title = "Verb"
    breakdown.add_data(
        Reference(operations, min_col=4, max_col=7, min_row=1, max_row=last_row),
        titles_from_data=True,
    )
    breakdown.set_categories(
        Reference(operations, min_col=1, min_row=2, max_row=last_row)
    )
    breakdown.height = 9
    breakdown.width = 18
    operations.add_chart(breakdown, "J21")

    # --------------------------------------------------------
    # Errors
    # --------------------------------------------------------
    error_sheet = workbook.create_sheet("Errors")

    category_table = [
        [category, errors[category]] for category in ERROR_CATEGORIES
    ]
    category_last = add_table(
        error_sheet, ["ErrorCategory", "Requests"], category_table, start_row=1
    )

    category_chart = BarChart()
    category_chart.type = "col"
    category_chart.title = "403 / 404 / 5XX totals"
    category_chart.y_axis.title = "Requests"
    category_chart.x_axis.title = "Category"
    category_chart.add_data(
        Reference(error_sheet, min_col=2, min_row=1, max_row=category_last),
        titles_from_data=True,
    )
    category_chart.set_categories(
        Reference(error_sheet, min_col=1, min_row=2, max_row=category_last)
    )
    category_chart.height = 9
    category_chart.width = 16
    error_sheet.add_chart(category_chart, "F2")

    status_start = category_last + 3
    status_table = [
        [code, count]
        for code, count in sorted(analytics["status_counts"].items())
    ]
    status_last = add_table(
        error_sheet,
        ["HttpStatus", "Requests"],
        status_table,
        start_row=status_start,
    )

    status_chart = BarChart()
    status_chart.type = "col"
    status_chart.title = "Requests by HTTP status code"
    status_chart.y_axis.title = "Requests"
    status_chart.x_axis.title = "Status"
    status_chart.add_data(
        Reference(
            error_sheet, min_col=2, min_row=status_start, max_row=status_last
        ),
        titles_from_data=True,
    )
    status_chart.set_categories(
        Reference(
            error_sheet,
            min_col=1,
            min_row=status_start + 1,
            max_row=status_last,
        )
    )
    status_chart.height = 9
    status_chart.width = 16
    error_sheet.add_chart(status_chart, "F21")

    code_start = status_last + 3
    code_table = [
        [name, count] for name, count in analytics["error_codes"].most_common(25)
    ]

    if code_table:
        code_last = add_table(
            error_sheet,
            ["S3 ErrorCode", "Occurrences"],
            code_table,
            start_row=code_start,
        )

        code_chart = BarChart()
        code_chart.type = "bar"
        code_chart.title = "Top S3 error codes"
        code_chart.y_axis.title = "Occurrences"
        code_chart.add_data(
            Reference(
                error_sheet, min_col=2, min_row=code_start, max_row=code_last
            ),
            titles_from_data=True,
        )
        code_chart.set_categories(
            Reference(
                error_sheet,
                min_col=1,
                min_row=code_start + 1,
                max_row=code_last,
            )
        )
        code_chart.height = 10
        code_chart.width = 18
        error_sheet.add_chart(code_chart, "F40")

    # --------------------------------------------------------
    # Errors by operation
    # --------------------------------------------------------
    by_operation = workbook.create_sheet("Errors by Operation")
    operation_table = []

    for operation in sorted(analytics["error_by_operation"]):
        counts = analytics["error_by_operation"][operation]
        operation_table.append(
            [operation]
            + [counts[category] for category in ERROR_CATEGORIES]
            + [sum(counts.values())]
        )

    operation_table.sort(key=lambda record: record[-1], reverse=True)

    if operation_table:
        operation_last = add_table(
            by_operation,
            ["Operation"] + ERROR_CATEGORIES + ["TotalErrors"],
            operation_table,
            start_row=1,
        )

        operation_chart = BarChart()
        operation_chart.type = "bar"
        operation_chart.grouping = "stacked"
        operation_chart.overlap = 100
        operation_chart.title = "Errors by operation"
        operation_chart.y_axis.title = "Failed requests"
        operation_chart.add_data(
            Reference(
                by_operation,
                min_col=2,
                max_col=1 + len(ERROR_CATEGORIES),
                min_row=1,
                max_row=operation_last,
            ),
            titles_from_data=True,
        )
        operation_chart.set_categories(
            Reference(by_operation, min_col=1, min_row=2, max_row=operation_last)
        )
        operation_chart.height = 12
        operation_chart.width = 20
        by_operation.add_chart(operation_chart, "H2")
    else:
        by_operation["A1"] = "No 4XX or 5XX responses were recorded."

    # --------------------------------------------------------
    # Daily activity
    # --------------------------------------------------------
    daily_sheet = workbook.create_sheet("Daily Activity")
    daily_headers = [
        "Date (UTC)",
        "GET",
        "PUT",
        "DELETE",
        "Errors",
        "UploadGiB",
        "DownloadGiB",
    ]
    daily_table = [
        [
            day,
            values["GET"],
            values["PUT"],
            values["DELETE"],
            values["errors"],
            round(values["upload_bytes"] / (1024 ** 3), 6),
            round(values["download_bytes"] / (1024 ** 3), 6),
        ]
        for day, values in analytics["daily"].items()
    ]

    if daily_table:
        daily_last = add_table(
            daily_sheet, daily_headers, daily_table, start_row=1
        )

        trend = LineChart()
        trend.title = "Daily GET / PUT / DELETE and errors"
        trend.y_axis.title = "Requests"
        trend.x_axis.title = "Date (UTC)"
        trend.add_data(
            Reference(
                daily_sheet, min_col=2, max_col=5, min_row=1, max_row=daily_last
            ),
            titles_from_data=True,
        )
        trend.set_categories(
            Reference(daily_sheet, min_col=1, min_row=2, max_row=daily_last)
        )
        trend.height = 10
        trend.width = 24
        daily_sheet.add_chart(trend, "I2")

        traffic = BarChart()
        traffic.type = "col"
        traffic.grouping = "stacked"
        traffic.overlap = 100
        traffic.title = "Daily traffic (GiB)"
        traffic.y_axis.title = "GiB"
        traffic.x_axis.title = "Date (UTC)"
        traffic.add_data(
            Reference(
                daily_sheet, min_col=6, max_col=7, min_row=1, max_row=daily_last
            ),
            titles_from_data=True,
        )
        traffic.set_categories(
            Reference(daily_sheet, min_col=1, min_row=2, max_row=daily_last)
        )
        traffic.height = 10
        traffic.width = 24
        daily_sheet.add_chart(traffic, "I23")
    else:
        daily_sheet["A1"] = "No records carried a parseable timestamp."

    # --------------------------------------------------------
    # Error records
    # --------------------------------------------------------
    records_sheet = workbook.create_sheet("Error Records")
    record_fields = [
        "TimestampUTC",
        "Verb",
        "Operation",
        "HttpStatus",
        "ErrorCategory",
        "ErrorCode",
        "Bucket",
        "Key",
        "RemoteIP",
        "Requester",
        "RequestId",
    ]

    if analytics["error_records"]:
        add_table(
            records_sheet,
            record_fields,
            [
                [record[field] for field in record_fields]
                for record in analytics["error_records"]
            ],
            start_row=1,
        )
        records_sheet.freeze_panes = "A2"

        if analytics["error_records_total"] > len(analytics["error_records"]):
            records_sheet.cell(
                row=len(analytics["error_records"]) + 3,
                column=1,
                value=(
                    f"Showing the first {len(analytics['error_records'])} of "
                    f"{analytics['error_records_total']} failed requests. "
                    "See wasabi_log_details.csv for the complete set."
                ),
            )
    else:
        records_sheet["A1"] = "No 4XX or 5XX responses were recorded."

    workbook.save(target)
    return target


# ============================================================
# PDF REPORT
# ============================================================

def write_pdf_report(analytics: Dict[str, object], output_dir: Path) -> Optional[Path]:
    try:
        import matplotlib

        matplotlib.use("Agg")

        import matplotlib.pyplot as plt
        from matplotlib.backends.backend_pdf import PdfPages
    except ImportError:
        print(
            "PDF report skipped: matplotlib is not installed "
            "(pip install matplotlib)."
        )
        return None

    target = output_dir / "wasabi_report.pdf"

    verbs = analytics["verbs"]
    errors = analytics["error_totals"]
    daily = analytics["daily"]
    days = list(daily)

    palette = {
        "403": "#c0392b",
        "404": "#e67e22",
        "5XX": "#8e44ad",
        "Other 4XX": "#7f8c8d",
    }

    def annotate(axis, bars) -> None:
        for bar in bars:
            height = bar.get_height()
            axis.annotate(
                f"{int(height):,}",
                (bar.get_x() + bar.get_width() / 2, height),
                ha="center",
                va="bottom",
                fontsize=8,
            )

    with PdfPages(target) as pdf:
        # Cover page with the headline numbers.
        figure = plt.figure(figsize=(11.69, 8.27))
        figure.suptitle("Wasabi Bucket Log Report", fontsize=20, y=0.94)

        coverage = f"{days[0]} to {days[-1]}" if days else "no dated records"
        upload_bytes = sum(day["upload_bytes"] for day in daily.values())
        download_bytes = sum(day["download_bytes"] for day in daily.values())

        lines = [
            f"Source            : s3://{LOG_BUCKET}/{LOG_PREFIX.strip('/')}",
            f"Generated (UTC)   : "
            f"{datetime.now(timezone.utc).isoformat(timespec='seconds')}",
            f"Date coverage     : {coverage}",
            "",
            f"Total requests    : {analytics['total_requests']:,}",
            f"GET               : {verbs['GET']['requests']:,}",
            f"PUT               : {verbs['PUT']['requests']:,}",
            f"DELETE            : {verbs['DELETE']['requests']:,}",
            f"HEAD              : {verbs['HEAD']['requests']:,}",
            f"POST              : {verbs['POST']['requests']:,}",
            f"Other             : {verbs['OTHER']['requests']:,}",
            "",
            f"403 Forbidden     : {errors['403']:,}",
            f"404 Not Found     : {errors['404']:,}",
            f"5XX Server errors : {errors['5XX']:,}",
            f"Other 4XX errors  : {errors['Other 4XX']:,}",
            "",
            f"Upload traffic    : {format_bytes(upload_bytes)}",
            f"Download traffic  : {format_bytes(download_bytes)}",
        ]

        figure.text(
            0.08,
            0.82,
            "\n".join(lines),
            fontsize=12,
            family="monospace",
            va="top",
        )
        pdf.savefig(figure)
        plt.close(figure)

        # Requests per verb and the requested error totals.
        figure, (left, right) = plt.subplots(1, 2, figsize=(11.69, 8.27))

        names = REPORT_VERBS
        counts = [verbs[verb]["requests"] for verb in names]
        annotate(left, left.bar(names, counts, color="#1f4e79"))
        left.set_title("Requests by operation verb")
        left.set_ylabel("Requests")
        left.grid(axis="y", alpha=0.3)

        category_counts = [errors[category] for category in ERROR_CATEGORIES]
        annotate(
            right,
            right.bar(
                ERROR_CATEGORIES,
                category_counts,
                color=[palette[c] for c in ERROR_CATEGORIES],
            ),
        )
        right.set_title("403 / 404 / 5XX totals")
        right.set_ylabel("Failed requests")
        right.grid(axis="y", alpha=0.3)

        figure.tight_layout()
        pdf.savefig(figure)
        plt.close(figure)

        # Error breakdown per verb, stacked.
        figure, axis = plt.subplots(figsize=(11.69, 8.27))
        bottom = [0] * len(names)

        for category in ERROR_CATEGORIES:
            values = [verbs[verb][category] for verb in names]
            axis.bar(
                names,
                values,
                bottom=bottom,
                label=category,
                color=palette[category],
            )
            bottom = [total + value for total, value in zip(bottom, values)]

        axis.set_title("Errors by verb (403 / 404 / 5XX / other 4XX)")
        axis.set_ylabel("Failed requests")
        axis.legend()
        axis.grid(axis="y", alpha=0.3)
        figure.tight_layout()
        pdf.savefig(figure)
        plt.close(figure)

        # Status code distribution.
        figure, axis = plt.subplots(figsize=(11.69, 8.27))
        status_items = sorted(analytics["status_counts"].items())
        axis.bar(
            [code for code, _ in status_items],
            [count for _, count in status_items],
            color="#2e86c1",
        )
        axis.set_title("Requests by HTTP status code")
        axis.set_ylabel("Requests")
        axis.set_xlabel("Status")
        axis.grid(axis="y", alpha=0.3)
        figure.tight_layout()
        pdf.savefig(figure)
        plt.close(figure)

        # Top S3 error codes.
        top_codes = analytics["error_codes"].most_common(15)

        if top_codes:
            figure, axis = plt.subplots(figsize=(11.69, 8.27))
            labels = [name for name, _ in reversed(top_codes)]
            values = [count for _, count in reversed(top_codes)]
            axis.barh(labels, values, color="#b03a2e")
            axis.set_title("Top S3 error codes")
            axis.set_xlabel("Occurrences")
            axis.grid(axis="x", alpha=0.3)
            figure.tight_layout()
            pdf.savefig(figure)
            plt.close(figure)

        # Daily trend.
        if days:
            figure, (top, bottom_axis) = plt.subplots(
                2, 1, figsize=(11.69, 8.27), sharex=True
            )

            for verb, color in (
                ("GET", "#1f4e79"),
                ("PUT", "#27ae60"),
                ("DELETE", "#c0392b"),
            ):
                top.plot(
                    days,
                    [daily[day][verb] for day in days],
                    marker="o",
                    label=verb,
                    color=color,
                )

            top.plot(
                days,
                [daily[day]["errors"] for day in days],
                marker="x",
                linestyle="--",
                label="Errors",
                color="#8e44ad",
            )
            top.set_title("Daily GET / PUT / DELETE and errors")
            top.set_ylabel("Requests")
            top.legend()
            top.grid(alpha=0.3)

            bottom_axis.bar(
                days,
                [daily[day]["upload_bytes"] / (1024 ** 3) for day in days],
                label="Upload GiB",
                color="#27ae60",
            )
            bottom_axis.bar(
                days,
                [daily[day]["download_bytes"] / (1024 ** 3) for day in days],
                bottom=[
                    daily[day]["upload_bytes"] / (1024 ** 3) for day in days
                ],
                label="Download GiB",
                color="#2e86c1",
            )
            bottom_axis.set_title("Daily traffic")
            bottom_axis.set_ylabel("GiB")
            bottom_axis.legend()
            bottom_axis.grid(axis="y", alpha=0.3)

            # Keep the date axis readable when the mirror spans many days.
            stride = max(1, len(days) // 30)
            bottom_axis.set_xticks(range(0, len(days), stride))
            bottom_axis.set_xticklabels(
                days[::stride], rotation=90, fontsize=7
            )

            figure.tight_layout()
            pdf.savefig(figure)
            plt.close(figure)

    return target


# ============================================================
# MAIN
# ============================================================

def format_bytes(value: int) -> str:
    if value >= 1024 ** 4:
        return f"{value / (1024 ** 4):.3f} TiB"
    if value >= 1024 ** 3:
        return f"{value / (1024 ** 3):.3f} GiB"
    if value >= 1024 ** 2:
        return f"{value / (1024 ** 2):.3f} MiB"
    if value >= 1024:
        return f"{value / 1024:.3f} KiB"
    return f"{value} bytes"


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Download Wasabi bucket logs and generate detailed CSV reports."
    )

    parser.add_argument(
        "mode",
        choices=["L", "W", "l", "w"],
        help="L = Linux (/home/Wasabi), W = Windows (C:\\Wasabi)",
    )

    parser.add_argument(
        "--skip-sync",
        action="store_true",
        help="Do not run AWS CLI sync; process the current local raw folder only.",
    )

    parser.add_argument(
        "--no-excel",
        action="store_true",
        help="Skip the Excel workbook (wasabi_report.xlsx).",
    )

    parser.add_argument(
        "--no-pdf",
        action="store_true",
        help="Skip the PDF report (wasabi_report.pdf).",
    )

    args = parser.parse_args()
    mode = args.mode.upper()

    paths = get_paths(mode)
    create_directories(paths)

    print("=" * 68)
    print("Wasabi Bucket Log CSV Reporter")
    print("=" * 68)
    print(f"Mode        : {mode}")
    print(f"Root        : {paths['root']}")
    print(f"Raw mirror  : {paths['raw']}")
    print(f"CSV output  : {paths['output']}")
    print(f"S3 source   : {build_s3_source()}")
    print()

    if not args.skip_sync:
        sync_logs(paths["raw"])
        print()

    print("Parsing current local log mirror...")
    result = write_reports(paths["raw"], paths["output"])

    analytics = build_analytics(result["rows"])
    excel_path = None
    pdf_path = None

    if WRITE_EXCEL_REPORT and not args.no_excel:
        print("Building Excel workbook...")
        excel_path = write_excel_report(analytics, paths["output"])

    if WRITE_PDF_REPORT and not args.no_pdf:
        print("Building PDF report...")
        pdf_path = write_pdf_report(analytics, paths["output"])

    print()
    print("=" * 68)
    print("Report summary")
    print("=" * 68)
    print(f"Input files       : {result['input_files']}")
    print(f"Input lines       : {result['input_lines']}")
    print(f"Skipped headers   : {result['skipped_lines']}")
    print(f"Parsed records    : {result['parsed_records']}")
    print(f"Parse errors      : {result['parse_errors']}")
    print()
    print(
        f"Upload traffic    : {result['upload_requests']} requests / "
        f"{format_bytes(result['upload_bytes'])}"
    )
    print(
        f"Download traffic  : {result['download_requests']} requests / "
        f"{format_bytes(result['download_bytes'])}"
    )
    print()

    verbs = analytics["verbs"]
    errors = analytics["error_totals"]

    print(
        "Requests by verb  : "
        + ", ".join(
            f"{verb}={verbs[verb]['requests']}" for verb in REPORT_VERBS
        )
    )
    print(
        "Error totals      : "
        + ", ".join(
            f"{category}={errors[category]}" for category in ERROR_CATEGORIES
        )
    )
    print()
    print(f"Details CSV       : {result['detail_csv']}")
    print(f"Traffic summary   : {result['traffic_csv']}")
    print(f"Operation summary : {result['operation_csv']}")
    print(f"Parse errors CSV  : {result['error_csv']}")

    if excel_path:
        print(f"Excel report      : {excel_path}")

    if pdf_path:
        print(f"PDF report        : {pdf_path}")

    print()

    if result["parse_errors"]:
        print(
            "WARNING: Some records could not be parsed. "
            "Review parse_errors.csv before relying on the report."
        )

    return 0


if __name__ == "__main__":
    try:
        raise SystemExit(main())
    except KeyboardInterrupt:
        print("\nCancelled.", file=sys.stderr)
        raise SystemExit(130)
    except Exception as exc:
        print(f"ERROR: {exc}", file=sys.stderr)
        raise SystemExit(1)