| 1 | """The project's referee, on the official ARC-AGI-3 toolkit. ARC-AGI-3 is played alone: the season's field is a set of |
| 2 | its public games, each a subtask. check: the route's agent plays each game up to the action limit through the toolkit, |
| 3 | served locally; the score is ARC's own RHAE: for each level cleared, (human baseline actions / actions taken)², |
| 4 | capped at 1.15, weighted by the level's number, as a percentage of the game; the mean over the games is the score. |
| 5 | practice: the first two games, a replay of each in replays/. |
| 6 | python3 referee/play.py check|practice <route folder> |
| 7 | """ |
| 8 | import glob, gzip, json, os, socket, subprocess, sys, tempfile, time |
| 9 | |
| 10 | ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 11 | sys.path.insert(0, os.path.join(ROOT, 'kit')) |
| 12 | import league |
| 13 | |
| 14 | ENVS = os.path.join(ROOT, 'referee', 'environment_files') |
| 15 | MOST = 400 |
| 16 | |
| 17 | |
| 18 | def games(): |
| 19 | with open(os.path.join(ROOT, 'referee', 'games.txt')) as f: |
| 20 | return [l.strip() for l in f if l.strip() and not l.startswith('#')] |
| 21 | |
| 22 | |
| 23 | def meta(game): |
| 24 | return json.load(open(glob.glob(os.path.join(ENVS, game, '*', 'metadata.json'))[0])) |
| 25 | |
| 26 | |
| 27 | def rhae(steps, baseline): |
| 28 | """Per level cleared, (baseline / actions)^2 capped at 1.15, weighted by level number, as a % of the game.""" |
| 29 | total, done, since = 0.0, 0, 0 |
| 30 | for s in steps: |
| 31 | since += 1 |
| 32 | if s['levels_completed'] > done: |
| 33 | level = s['levels_completed'] |
| 34 | b = baseline[level - 1] if level - 1 < len(baseline) else baseline[-1] |
| 35 | total += level * min(1.15, (b / since) ** 2) |
| 36 | done, since = level, 0 |
| 37 | return 100 * total / sum(range(1, len(baseline) + 1)) |
| 38 | |
| 39 | |
| 40 | def replay(game, steps): |
| 41 | m = meta(game) |
| 42 | frame = lambda s: ''.join('%x' % (v & 15) for row in s['frame'][-1] for v in row) if s['frame'] else '' |
| 43 | return {'game': 'arc-agi-3', 'id': m['game_id'], 'title': m.get('title', game), 'fps': m.get('default_fps', 10), 'levels': len(m['baseline_actions']), |
| 44 | 'steps': [{'a': (s.get('action_input') or {}).get('id', 0), 'd': (s.get('action_input') or {}).get('data') or {}, 's': s['state'], 'l': s['levels_completed'], 'f': frame(s)} for s in steps]} |
| 45 | |
| 46 | |
| 47 | def main(mode, route): |
| 48 | bot = os.path.join(league.bot_dir(route), 'agent.py') |
| 49 | if not os.path.isfile(bot): |
| 50 | sys.exit(f'No bot in {route}: add agent.py with a class Agent (choose_action(frames, latest_frame)), as ARC-AGI-3-Agents writes them') |
| 51 | name = league.route_name() |
| 52 | reps = league.Replays() if mode == 'practice' else None |
| 53 | clean = tempfile.mkdtemp() |
| 54 | recs = os.path.join(clean, 'recordings') |
| 55 | with socket.socket() as s: |
| 56 | s.bind(('127.0.0.1', 0)) |
| 57 | port = s.getsockname()[1] |
| 58 | env = {k: v for k, v in os.environ.items() if not k.startswith(('ARC_', 'OPERATION_MODE', 'SS_', 'SLURM_'))} |
| 59 | server = subprocess.Popen([sys.executable, '-s', os.path.join(ROOT, 'referee', 'serve.py'), ENVS, recs, str(port)], cwd=clean, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) |
| 60 | try: |
| 61 | for _ in range(100): |
| 62 | try: |
| 63 | socket.create_connection(('127.0.0.1', port), 0.2).close() |
| 64 | break |
| 65 | except OSError: |
| 66 | time.sleep(0.1) |
| 67 | subtasks = {} |
| 68 | for game in games() if mode == 'check' else games()[:2]: |
| 69 | before = set(glob.glob(os.path.join(recs, '**', '*.jsonl*'), recursive=True)) |
| 70 | try: |
| 71 | subprocess.run([sys.executable, '-s', os.path.join(ROOT, 'referee', 'agent_child.py'), bot, f'http://127.0.0.1:{port}', game, str(MOST)], |
| 72 | cwd=clean, env={**env, 'PYTHONHASHSEED': '0'}, timeout=MOST * 2 + 30, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) |
| 73 | except subprocess.TimeoutExpired: |
| 74 | pass |
| 75 | time.sleep(0.3) |
| 76 | new = sorted(set(glob.glob(os.path.join(recs, '**', f'{game}-*.jsonl*'), recursive=True)) - before, key=os.path.getmtime) |
| 77 | steps = [] |
| 78 | for path in new: |
| 79 | with (gzip.open if path.endswith('.gz') else open)(path, 'rt') as f: |
| 80 | steps += [json.loads(l)['data'] for l in f if l.strip()] |
| 81 | subtasks[game] = rhae(steps[1:], meta(game)['baseline_actions']) if steps else 0.0 |
| 82 | if reps and steps: |
| 83 | r = replay(game, steps) |
| 84 | reps.add(f'{game}.json', r, f'{r["title"]}: {name}', [name], f'{max(s["levels_completed"] for s in steps)} of {r["levels"]} levels in {len(steps) - 1} actions, {subtasks[game]:.1f} RHAE') |
| 85 | league.write_metrics(sum(subtasks.values()) / len(subtasks), subtasks) |
| 86 | finally: |
| 87 | server.kill() |
| 88 | |
| 89 | |
| 90 | if __name__ == '__main__': |
| 91 | main(sys.argv[1], sys.argv[2]) |
shoal