"""Bounded, paired local strength audit; never reads or writes the leaderboard. The real pinned engine runs each game. Per-seat RNG state is isolated to match the child adapter's random.seed(0), so one policy cannot change referee random draws or another player's random stream. No model/network calls are made. .venvs/werewolf/bin/python games/werewolf/referee/compare.py --rounds 20 """ import argparse from collections import defaultdict import contextlib import json import random import sys import time with contextlib.redirect_stdout(sys.stderr): from kaggle_environments import make from kaggle_environments.envs.werewolf.werewolf import deterministic_agent, random_agent from play import configuration from rule_agent import RuleAgent class Isolated: def __init__(self, fn): self.fn, self.state = fn, random.Random(0).getstate() def __call__(self, obs, config=None): outer = random.getstate() random.setstate(self.state) try: return self.fn(obs) finally: self.state = random.getstate() random.setstate(outer) def main(): parser = argparse.ArgumentParser() parser.add_argument('--rounds', type=int, default=20, choices=range(1, 101), metavar='1..100') parser.add_argument('--policy', choices=['both', 'old', 'evidence'], default='both') parser.add_argument('--opponent', choices=['all', 'deterministic', 'random', 'evidence'], default='all') args = parser.parse_args() started = time.monotonic() for opponent, fn in [('deterministic', deterministic_agent), ('random', random_agent), ('evidence', None)]: if args.opponent not in ('all', opponent): continue for name, factory in [('old', lambda: Isolated(deterministic_agent)), ('evidence', RuleAgent)]: if args.policy not in ('both', name): continue by = defaultdict(list) for repeat in range(args.rounds): for mine in range(7): seed = repeat + 900 random.seed(seed) conf = configuration(seed, mine, rng=random.Random(9000 + repeat * 7 + mine)) env = make('werewolf', configuration=conf) players = [Isolated(fn) if fn else RuleAgent() for _ in range(7)] players[mine] = factory() env.run(players) if not env.done or env.info['GAME_END']['terminated_with_agent_error']: raise RuntimeError(f'Invalid game: opponent={opponent}, policy={name}, round={repeat}, seat={mine}') by[conf['agents'][mine]['role']].append(int((env.state[mine].reward or 0) > 0)) print(json.dumps({'opponent': opponent, 'policy': name, 'wins': sum(map(sum, by.values())), 'games': sum(map(len, by.values())), 'roles': {r: {'wins': sum(v), 'games': len(v)} for r, v in by.items()}}), flush=True) print(json.dumps({'elapsed_seconds': round(time.monotonic() - started, 3)})) if __name__ == '__main__': main()