From d367b33b6d7dcf04eaa7c7e89afb4f2349ea89fb Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Mon, 25 Feb 2019 01:48:47 -0800 Subject: [PATCH 1/2] rework some pair quality metrics --- cluster/evaluator/launch_eval.py | 36 +++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/cluster/evaluator/launch_eval.py b/cluster/evaluator/launch_eval.py index 83b15766f..8ef826002 100644 --- a/cluster/evaluator/launch_eval.py +++ b/cluster/evaluator/launch_eval.py @@ -161,11 +161,15 @@ def get_cross_eval_pairs(): print ("Found", sum(game_counts.values()), "games") model_game_counts = Counter() - for (winner, losser), count in game_counts.items(): + for (winner, loser), count in game_counts.items(): model_game_counts[winner] += count - model_game_counts[losser] += count + model_game_counts[loser] += count existing_pairs, previous_pairs = restore_pairs() + max_uncertainty = float(max([r[1] for r in rs.values()])) + + import pdb + pdb.set_trace() # priority is roughly related to expected gain of information pairs = [] @@ -207,32 +211,35 @@ def get_cross_eval_pairs(): # priority based on being highly ranked # Higher = better - rank_num = max(ranks.get(model_a, 0), ranks.get(model_b, 0)) - rank_adjustment = (1 + rank_num / len(rs)) ** 2 / 4 + rank_num = min(ranks.get(model_a, 0), ranks.get(model_b, 0)) - # Do this for a while - if (rank_num + 25 < len(rs)): - continue + if (rank_num + 25) < len(rs): + continue + + rank_adjustment = (1 + rank_num / len(rs)) ** 2 / 4 # priority based on model variances - joint_uncertainty = (r_a[1] ** 2 + r_b[1] ** 2) ** 0.5 - uncertainty_priority = joint_uncertainty / uncertainty_const + avg_u = (r_a[1] + r_b[1]) / 2.0 + uncertainty_priority = avg_u / max_uncertainty # priority based on information gained by playing this pairing - win_prob = 1 / (1 + 10 ** (-(r_a[0] - r_b[0])/400)) - variance = win_prob * (1 - win_prob) - pairing_priority = equality_const * variance / (1 + games) ** games_power + win_prob = abs(1 / (1 + 10 ** (-(r_a[0] - r_b[0])/400))) + game_quality = win_prob * (1 - win_prob) + rank_adjustment # priority based on playing a game with this model model_priority = (1 / (1 + model_a_games) ** games_power + 1 / (1 + model_b_games) ** games_power) - priority = pairing_priority + model_priority + uncertainty_priority + priority = game_quality + model_priority + uncertainty_priority +# print("{}:{} vs {}:{}" +# "\t{:0.2f} {:0.2f} {:0.2f} = {:0.2f}".format( +# run_a, model_a, run_b, model_b, +# game_quality, model_priority, uncertainty_priority, priority)) pairs.append(( [rank_adjustment * priority, - rank_adjustment, win_prob, joint_uncertainty, + rank_adjustment, win_prob, uncertainty_priority, games, model_a_games, model_b_games], pair)) @@ -414,6 +421,7 @@ def make_pairs_for_model(model_num=0): 'same_run_eval': same_run_eval, 'cross_run_eval': cross_run_eval, 'cleanup': cleanup, + 'get_cross_eval_pairs': get_cross_eval_pairs, 'add_top_pairs': add_top_pairs, 'launch_eval_job': launch_eval_job, }, remaining_argv[1:]) From 11e41a429fd83f6220ba29eef0d32c8cb3acb0e1 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Mon, 25 Feb 2019 01:50:42 -0800 Subject: [PATCH 2/2] remove stray debug call --- cluster/evaluator/launch_eval.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cluster/evaluator/launch_eval.py b/cluster/evaluator/launch_eval.py index 8ef826002..c979f9638 100644 --- a/cluster/evaluator/launch_eval.py +++ b/cluster/evaluator/launch_eval.py @@ -168,9 +168,6 @@ def get_cross_eval_pairs(): existing_pairs, previous_pairs = restore_pairs() max_uncertainty = float(max([r[1] for r in rs.values()])) - import pdb - pdb.set_trace() - # priority is roughly related to expected gain of information pairs = [] for run_a, model_a in all_models: