Update visualization codes for NATS-Bench

exps/NATS-Bench/draw-correlations.py

@@ -0,0 +1,90 @@
+###############################################################
+# NATS-Bench (https://arxiv.org/pdf/2009.00437.pdf)           #
+###############################################################
+# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2020.06           #
+###############################################################
+# Usage: python exps/NATS-Bench/draw-correlations.py          #
+###############################################################
+import os, gc, sys, time, scipy, torch, argparse
+import numpy as np
+from typing import List, Text, Dict, Any
+from shutil import copyfile
+from collections import defaultdict, OrderedDict
+from copy    import deepcopy
+from pathlib import Path
+import matplotlib
+import seaborn as sns
+matplotlib.use('agg')
+import matplotlib.pyplot as plt
+import matplotlib.ticker as ticker
+
+lib_dir = (Path(__file__).parent / '..' / '..' / 'lib').resolve()
+if str(lib_dir) not in sys.path: sys.path.insert(0, str(lib_dir))
+from config_utils import dict2config, load_config
+from nats_bench import create
+from log_utils import time_string
+
+
+def get_valid_test_acc(api, arch, dataset):
+  is_size_space = api.search_space_name == 'size'
+  if dataset == 'cifar10':
+      xinfo = api.get_more_info(arch, dataset=dataset, hp=90 if is_size_space else 200, is_random=False)
+      test_acc = xinfo['test-accuracy']
+      xinfo = api.get_more_info(arch, dataset='cifar10-valid', hp=90 if is_size_space else 200, is_random=False)
+      valid_acc = xinfo['valid-accuracy']
+  else:
+      xinfo = api.get_more_info(arch, dataset=dataset, hp=90 if is_size_space else 200, is_random=False)
+      valid_acc = xinfo['valid-accuracy']
+      test_acc = xinfo['test-accuracy']
+  return valid_acc, test_acc, 'validation = {:.2f}, test = {:.2f}\n'.format(valid_acc, test_acc)
+
+
+def compute_kendalltau(vectori, vectorj):
+  # indexes = list(range(len(vectori)))
+  # rank_1 = sorted(indexes, key=lambda i: vectori[i])
+  # rank_2 = sorted(indexes, key=lambda i: vectorj[i])
+  # import pdb; pdb.set_trace()
+  coef, p = scipy.stats.kendalltau(vectori, vectorj)
+  return coef
+
+
+def compute_spearmanr(vectori, vectorj):
+  coef, p = scipy.stats.spearmanr(vectori, vectorj)
+  return coef
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser(description='NATS-Bench: Benchmarking NAS algorithms for Architecture Topology and Size', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+  parser.add_argument('--save_dir',     type=str,   default='output/vis-nas-bench/nas-algos', help='Folder to save checkpoints and log.')
+  parser.add_argument('--search_space', type=str,   choices=['tss', 'sss'], help='Choose the search space.')
+  args = parser.parse_args()
+
+  save_dir = Path(args.save_dir)
+
+  api = create(None, 'tss', fast_mode=True, verbose=False)
+  indexes = list(range(1, 10000, 300))
+  scores_1 = []
+  scores_2 = []
+  for index in indexes:
+    valid_acc, test_acc, _ = get_valid_test_acc(api, index, 'cifar10')
+    scores_1.append(valid_acc)
+    scores_2.append(test_acc)
+  correlation = compute_kendalltau(scores_1, scores_2)
+  print('The kendall tau correlation of {:} samples : {:}'.format(len(indexes), correlation))
+  correlation = compute_spearmanr(scores_1, scores_2)
+  print('The spearmanr correlation of {:} samples : {:}'.format(len(indexes), correlation))
+  # scores_1 = ['{:.2f}'.format(x) for x in scores_1]
+  # scores_2 = ['{:.2f}'.format(x) for x in scores_2]
+  # print(', '.join(scores_1))
+  # print(', '.join(scores_2))
+
+  dpi, width, height = 250, 1000, 1000
+  figsize = width / float(dpi), height / float(dpi)
+  LabelSize, LegendFontsize = 14, 14
+
+  fig, ax = plt.subplots(1, 1, figsize=figsize)
+  ax.scatter(scores_1, scores_2 , marker='^', s=0.5, c='tab:green', alpha=0.8)
+
+  save_path = '/Users/xuanyidong/Desktop/test-temp-rank.png'
+  fig.savefig(save_path, dpi=dpi, bbox_inches='tight', format='png')
+  plt.close('all')