iicd/naswot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Produce table for multiple Ns

Browse Source
This commit is contained in:
jack-willturner
2020-06-03 16:28:45 +01:00
parent 45901dd7ec
commit 204a72f95e
4 changed files with 40 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
process_results.py
View File

@@ -16,7 +16,6 @@ parser.add_argument('--GPU', default='0', type=str)
parser.add_argument('--seed', default=1, type=int)
parser.add_argument('--trainval', action='store_true')
parser.add_argument('--n_samples', default=100, type=int, help='how many samples to take')
parser.add_argument('--n_runs', default=500, type=int)
args = parser.parse_args()
@@ -48,39 +47,41 @@ datasets['ImageNet16-120 (test)'] = ('ImageNet16-120', 'x-test', False)
dataset_top1s = OrderedDict()
dataset_top1s['Method'] = f"Ours (N={args.n_samples})"
dataset_top1s['Search time (s)'] = np.nan
time = 0.
for n_samples in [10, 100]:
method = f"Ours (N={n_samples})"
for dataset, params in datasets.items():
top1s = []
time = 0.
dset = params[0]
acc_type = 'accs' if 'test' in params[1] else 'val_accs'
filename = f"{args.save_loc}/{dset}_{args.n_runs}_{args.n_samples}_{args.seed}.t7"
for dataset, params in datasets.items():
top1s = []
full_scores = torch.load(filename)
if dataset == 'CIFAR-10 (test)':
time = median(full_scores['times'])
dataset_top1s['Search time (s)'] = time
accs = []
for n in range(args.n_runs):
acc = full_scores[acc_type][n]
accs.append(acc)
dataset_top1s[dataset] = accs
dset = params[0]
acc_type = 'accs' if 'test' in params[1] else 'val_accs'
filename = f"{args.save_loc}/{dset}_{args.n_runs}_{n_samples}_{args.seed}.t7"
df = pd.DataFrame(dataset_top1s)
full_scores = torch.load(filename)
if dataset == 'CIFAR-10 (test)':
time = median(full_scores['times'])
dataset_top1s['Search time (s)'] = time
accs = []
for n in range(args.n_runs):
acc = full_scores[acc_type][n]
accs.append(acc)
dataset_top1s[dataset] = accs
df['CIFAR-10 (val)'] = f"{mean(df['CIFAR-10 (val)']):.2f} +- {std(df['CIFAR-10 (val)']):.2f}"
df['CIFAR-10 (test)'] = f"{mean(df['CIFAR-10 (test)']):.2f} +- {std(df['CIFAR-10 (test)']):.2f}"
cifar10_val = f"{mean(dataset_top1s['CIFAR-10 (val)']):.2f} $\pm$ {std(dataset_top1s['CIFAR-10 (val)']):.2f}"
cifar10_test = f"{mean(dataset_top1s['CIFAR-10 (test)']):.2f} $\pm$ {std(dataset_top1s['CIFAR-10 (test)']):.2f}"
df['CIFAR-100 (val)'] = f"{mean(df['CIFAR-100 (val)']):.2f} +- {std(df['CIFAR-100 (val)']):.2f}"
df['CIFAR-100 (test)'] = f"{mean(df['CIFAR-100 (test)']):.2f} +- {std(df['CIFAR-100 (test)']):.2f}"
cifar100_val = f"{mean(dataset_top1s['CIFAR-100 (val)']):.2f} $\pm$ {std(dataset_top1s['CIFAR-100 (val)']):.2f}"
cifar100_test = f"{mean(dataset_top1s['CIFAR-100 (test)']):.2f} $\pm$ {std(dataset_top1s['CIFAR-100 (test)']):.2f}"
df['ImageNet16-120 (val)'] = f"{mean(df['ImageNet16-120 (val)']):.2f} +- {std(df['ImageNet16-120 (val)']):.2f}"
df['ImageNet16-120 (test)'] = f"{mean(df['ImageNet16-120 (test)']):.2f} +- {std(df['ImageNet16-120 (test)']):.2f}"
imagenet_val = f"{mean(dataset_top1s['ImageNet16-120 (val)']):.2f} $\pm$ {std(dataset_top1s['ImageNet16-120 (val)']):.2f}"
imagenet_test = f"{mean(dataset_top1s['ImageNet16-120 (test)']):.2f} $\pm$ {std(dataset_top1s['ImageNet16-120 (test)']):.2f}"
df = df.round(2)
df = df.iloc[:1]
df.append([method, time, cifar10_val, cifar10_test, cifar100_val, cifar100_test, imagenet_val, imagenet_test])
df = pd.DataFrame(df, columns=['Method','Search time (s)','CIFAR-10 (val)','CIFAR-10 (test)','CIFAR-100 (val)','CIFAR-100 (test)','ImageNet16-120 (val)','ImageNet16-120 (test)' ])
df.round(2)
print(tabulate.tabulate(df.values,df.columns, tablefmt="pipe"))