#!/usr/bin/env python3
"""Check this documentary dataset's arithmetic and CSV/JSON consistency.

Run beside the six data files: python verify_dataset.py
Python 3.10+; standard library only. This does not retrieve new source data,
verify a law, determine code compliance, or estimate missing observations.
"""
from __future__ import annotations
import csv
import json
from decimal import Decimal, ROUND_HALF_UP
from pathlib import Path

FILES = {
    'cbecs-2018-us-warehouse-layout-profile.csv': ('cbecs_2018_us_warehouse_layout_profile', 81),
    'cbecs-2018-us-warehouse-subcategories.csv': ('cbecs_2018_us_warehouse_subcategories', 6),
    'warehouse-layout-standards-ledger.csv': ('warehouse_layout_standards_ledger', 55),
    'warehouse-layout-published-rules-of-thumb-survey.csv': ('published_rules_of_thumb_survey', 32),
    'warehouse-workforce-and-michigan-kent-layer.csv': ('workforce_and_michigan_kent_layer', 26),
}

def require(condition: bool, message: str) -> None:
    if not condition:
        raise ValueError(message)

def rounded(value: Decimal, places: int) -> Decimal:
    return value.quantize(Decimal(1).scaleb(-places), rounding=ROUND_HALF_UP)

def main() -> None:
    root = Path(__file__).resolve().parent
    bundle = json.loads((root / 'warehouse-layout-dataset.json').read_text(encoding='utf-8'))
    rows_by_key: dict[str, list[dict[str, str]]] = {}
    ids: set[str] = set()
    total = 0
    for filename, (key, count) in FILES.items():
        with (root / filename).open(encoding='utf-8', newline='') as stream:
            rows = list(csv.DictReader(stream))
        require(len(rows) == count, f'{filename}: expected {count} rows, found {len(rows)}')
        json_rows = bundle['tables'][key]
        require(len(json_rows) == count, f'{key}: JSON row count differs')
        for index, (csv_row, json_row) in enumerate(zip(rows, json_rows), 1):
            require(set(csv_row) == set(json_row), f'{filename} row {index}: fields differ')
            for field, value in csv_row.items():
                expected = '' if json_row[field] is None else str(json_row[field])
                require(value == expected, f'{filename} row {index}: {field} differs between CSV and JSON')
            rid = csv_row['record_id']
            require(rid not in ids, f'Duplicate record ID: {rid}')
            ids.add(rid)
            require(csv_row['verification_date'] == '2026-09-11', f'{rid}: unexpected check date')
            if csv_row['publish_status'] == 'not_published':
                value = csv_row.get('value', csv_row.get('published_figure', ''))
                require(value == '', f'{rid}: excluded numerical record must not contain a value')
        rows_by_key[key] = rows
        total += len(rows)
    require(total == 200, 'The five files must retain all 200 original record slots')

    for key in ('cbecs_2018_us_warehouse_layout_profile', 'cbecs_2018_us_warehouse_subcategories'):
        for row in rows_by_key[key]:
            b_text, f_text = row['buildings_thousand'], row['floorspace_million_sqft']
            if b_text in ('Q', 'N', '') or f_text in ('Q', 'N', ''):
                for field in ('share_of_warehouse_buildings_pct', 'share_of_warehouse_floorspace_pct', 'mean_sqft_per_building_computed'):
                    require(row[field] == '', f'{row["record_id"]}: suppressed input cannot have a derived value')
                continue
            b, f = Decimal(b_text), Decimal(f_text)
            expected_building_share = rounded(100 * b / Decimal(1004), 1)
            expected_floor_share = rounded(100 * f / Decimal(17483), 1)
            require(Decimal(row['share_of_warehouse_buildings_pct']) == expected_building_share, f'{row["record_id"]}: building share')
            require(Decimal(row['share_of_warehouse_floorspace_pct']) == expected_floor_share, f'{row["record_id"]}: floorspace share')
            # Whole-square-foot exports do not imply whole-square-foot measurement precision.
            mean = 1000 * f / b
            require(abs(Decimal(row['mean_sqft_per_building_computed']) - mean) <= Decimal('0.5'), f'{row["record_id"]}: computed mean')

    p = rows_by_key['cbecs_2018_us_warehouse_layout_profile']
    # The existing derived rows are checked against their component size classes.
    for derived_index, components in [(9, [1, 2]), (10, [5, 6, 7, 8]), (11, [6, 7, 8])]:
        for field in ('buildings_thousand', 'floorspace_million_sqft'):
            require(sum(Decimal(p[i][field]) for i in components) == Decimal(p[derived_index][field]), f'Derived size-band row {derived_index}: {field}')
    require(sum(Decimal(p[i]['buildings_thousand']) for i in [62,63,64]) == 25, '50+ worker building aggregation')
    require(sum(Decimal(p[i]['floorspace_million_sqft']) for i in [62,63,64]) == 4681, '50+ worker floorspace aggregation')
    require(86 + 48 + 12 == 146 and divmod(146, 12) == (12, 2), 'Toyota example arithmetic')
    require(41400 - 29100 == 12300, 'Michigan employment difference')
    require(rounded(Decimal(12300) / Decimal(29100) * 100, 0) == 42, 'Michigan rounded growth')
    conventions = rows_by_key['published_rules_of_thumb_survey']
    published = [r for r in conventions if r['publish_status'] == 'published']
    require(len(published) == 29, 'Expected 29 source-checked publisher statements')
    require(len({r['publisher'] for r in published}) == 12, 'Expected 12 distinct publishers')
    require(sum(r['publish_status'] == 'not_published' for rows in rows_by_key.values() for r in rows) == 4, 'Expected three historical and one county exclusion')
    print('PASS: 200 records; CSV/JSON consistency; source-estimate arithmetic; derived bands; publisher counts; four empty numerical exclusions.')

if __name__ == '__main__':
    try:
        main()
    except (OSError, ValueError, KeyError, json.JSONDecodeError) as error:
        raise SystemExit(f'CHECK FAILED: {error}') from error